arch/mips/kvm/vz.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * KVM/MIPS: Support for hardware virtualization extensions
   7  *
   8  * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
   9  * Authors: Yann Le Du <[email protected]>
  10  */
  11
  12 #include <linux/errno.h>
  13 #include <linux/err.h>
  14 #include <linux/module.h>
  15 #include <linux/preempt.h>
  16 #include <linux/vmalloc.h>
  17 #include <asm/cacheflush.h>
  18 #include <asm/cacheops.h>
  19 #include <asm/cmpxchg.h>
  20 #include <asm/fpu.h>
  21 #include <asm/hazards.h>
  22 #include <asm/inst.h>
  23 #include <asm/mmu_context.h>
  24 #include <asm/r4kcache.h>
  25 #include <asm/time.h>
  26 #include <asm/tlb.h>
  27 #include <asm/tlbex.h>
  28
  29 #include <linux/kvm_host.h>
  30
  31 #include "interrupt.h"
  32
  33 #include "trace.h"
  34
  35 /* Pointers to last VCPU loaded on each physical CPU */
  36 static struct kvm_vcpu *last_vcpu[NR_CPUS];
  37 /* Pointers to last VCPU executed on each physical CPU */
  38 static struct kvm_vcpu *last_exec_vcpu[NR_CPUS];
  39
  40 /*
  41  * Number of guest VTLB entries to use, so we can catch inconsistency between
  42  * CPUs.
  43  */
  44 static unsigned int kvm_vz_guest_vtlb_size;
  45
  46 static inline long kvm_vz_read_gc0_ebase(void)
  47 {
  48         if (sizeof(long) == 8 && cpu_has_ebase_wg)
  49                 return read_gc0_ebase_64();
  50         else
  51                 return read_gc0_ebase();
  52 }
  53
  54 static inline void kvm_vz_write_gc0_ebase(long v)
  55 {
  56         /*
  57          * First write with WG=1 to write upper bits, then write again in case
  58          * WG should be left at 0.
  59          * write_gc0_ebase_64() is no longer UNDEFINED since R6.
  60          */
  61         if (sizeof(long) == 8 &&
  62             (cpu_has_mips64r6 || cpu_has_ebase_wg)) {
  63                 write_gc0_ebase_64(v | MIPS_EBASE_WG);
  64                 write_gc0_ebase_64(v);
  65         } else {
  66                 write_gc0_ebase(v | MIPS_EBASE_WG);
  67                 write_gc0_ebase(v);
  68         }
  69 }
  70
  71 /*
  72  * These Config bits may be writable by the guest:
  73  * Config:      [K23, KU] (!TLB), K0
  74  * Config1:     (none)
  75  * Config2:     [TU, SU] (impl)
  76  * Config3:     ISAOnExc
  77  * Config4:     FTLBPageSize
  78  * Config5:     K, CV, MSAEn, UFE, FRE, SBRI, UFR
  79  */
  80
  81 static inline unsigned int kvm_vz_config_guest_wrmask(struct kvm_vcpu *vcpu)
  82 {
  83         return CONF_CM_CMASK;
  84 }
  85
  86 static inline unsigned int kvm_vz_config1_guest_wrmask(struct kvm_vcpu *vcpu)
  87 {
  88         return 0;
  89 }
  90
  91 static inline unsigned int kvm_vz_config2_guest_wrmask(struct kvm_vcpu *vcpu)
  92 {
  93         return 0;
  94 }
  95
  96 static inline unsigned int kvm_vz_config3_guest_wrmask(struct kvm_vcpu *vcpu)
  97 {
  98         return MIPS_CONF3_ISA_OE;
  99 }
 100
 101 static inline unsigned int kvm_vz_config4_guest_wrmask(struct kvm_vcpu *vcpu)
 102 {
 103         /* no need to be exact */
 104         return MIPS_CONF4_VFTLBPAGESIZE;
 105 }
 106
 107 static inline unsigned int kvm_vz_config5_guest_wrmask(struct kvm_vcpu *vcpu)
 108 {
 109         unsigned int mask = MIPS_CONF5_K | MIPS_CONF5_CV | MIPS_CONF5_SBRI;
 110
 111         /* Permit MSAEn changes if MSA supported and enabled */
 112         if (kvm_mips_guest_has_msa(&vcpu->arch))
 113                 mask |= MIPS_CONF5_MSAEN;
 114
 115         /*
 116          * Permit guest FPU mode changes if FPU is enabled and the relevant
 117          * feature exists according to FIR register.
 118          */
 119         if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
 120                 if (cpu_has_ufr)
 121                         mask |= MIPS_CONF5_UFR;
 122                 if (cpu_has_fre)
 123                         mask |= MIPS_CONF5_FRE | MIPS_CONF5_UFE;
 124         }
 125
 126         return mask;
 127 }
 128
 129 /*
 130  * VZ optionally allows these additional Config bits to be written by root:
 131  * Config:      M, [MT]
 132  * Config1:     M, [MMUSize-1, C2, MD, PC, WR, CA], FP
 133  * Config2:     M
 134  * Config3:     M, MSAP, [BPG], ULRI, [DSP2P, DSPP], CTXTC, [ITL, LPA, VEIC,
 135  *              VInt, SP, CDMM, MT, SM, TL]
 136  * Config4:     M, [VTLBSizeExt, MMUSizeExt]
 137  * Config5:     MRP
 138  */
 139
 140 static inline unsigned int kvm_vz_config_user_wrmask(struct kvm_vcpu *vcpu)
 141 {
 142         return kvm_vz_config_guest_wrmask(vcpu) | MIPS_CONF_M;
 143 }
 144
 145 static inline unsigned int kvm_vz_config1_user_wrmask(struct kvm_vcpu *vcpu)
 146 {
 147         unsigned int mask = kvm_vz_config1_guest_wrmask(vcpu) | MIPS_CONF_M;
 148
 149         /* Permit FPU to be present if FPU is supported */
 150         if (kvm_mips_guest_can_have_fpu(&vcpu->arch))
 151                 mask |= MIPS_CONF1_FP;
 152
 153         return mask;
 154 }
 155
 156 static inline unsigned int kvm_vz_config2_user_wrmask(struct kvm_vcpu *vcpu)
 157 {
 158         return kvm_vz_config2_guest_wrmask(vcpu) | MIPS_CONF_M;
 159 }
 160
 161 static inline unsigned int kvm_vz_config3_user_wrmask(struct kvm_vcpu *vcpu)
 162 {
 163         unsigned int mask = kvm_vz_config3_guest_wrmask(vcpu) | MIPS_CONF_M |
 164                 MIPS_CONF3_ULRI | MIPS_CONF3_CTXTC;
 165
 166         /* Permit MSA to be present if MSA is supported */
 167         if (kvm_mips_guest_can_have_msa(&vcpu->arch))
 168                 mask |= MIPS_CONF3_MSA;
 169
 170         return mask;
 171 }
 172
 173 static inline unsigned int kvm_vz_config4_user_wrmask(struct kvm_vcpu *vcpu)
 174 {
 175         return kvm_vz_config4_guest_wrmask(vcpu) | MIPS_CONF_M;
 176 }
 177
 178 static inline unsigned int kvm_vz_config5_user_wrmask(struct kvm_vcpu *vcpu)
 179 {
 180         return kvm_vz_config5_guest_wrmask(vcpu) | MIPS_CONF5_MRP;
 181 }
 182
 183 static gpa_t kvm_vz_gva_to_gpa_cb(gva_t gva)
 184 {
 185         /* VZ guest has already converted gva to gpa */
 186         return gva;
 187 }
 188
 189 static void kvm_vz_queue_irq(struct kvm_vcpu *vcpu, unsigned int priority)
 190 {
 191         set_bit(priority, &vcpu->arch.pending_exceptions);
 192         clear_bit(priority, &vcpu->arch.pending_exceptions_clr);
 193 }
 194
 195 static void kvm_vz_dequeue_irq(struct kvm_vcpu *vcpu, unsigned int priority)
 196 {
 197         clear_bit(priority, &vcpu->arch.pending_exceptions);
 198         set_bit(priority, &vcpu->arch.pending_exceptions_clr);
 199 }
 200
 201 static void kvm_vz_queue_timer_int_cb(struct kvm_vcpu *vcpu)
 202 {
 203         /*
 204          * timer expiry is asynchronous to vcpu execution therefore defer guest
 205          * cp0 accesses
 206          */
 207         kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER);
 208 }
 209
 210 static void kvm_vz_dequeue_timer_int_cb(struct kvm_vcpu *vcpu)
 211 {
 212         /*
 213          * timer expiry is asynchronous to vcpu execution therefore defer guest
 214          * cp0 accesses
 215          */
 216         kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_TIMER);
 217 }
 218
 219 static void kvm_vz_queue_io_int_cb(struct kvm_vcpu *vcpu,
 220                                    struct kvm_mips_interrupt *irq)
 221 {
 222         int intr = (int)irq->irq;
 223
 224         /*
 225          * interrupts are asynchronous to vcpu execution therefore defer guest
 226          * cp0 accesses
 227          */
 228         switch (intr) {
 229         case 2:
 230                 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_IO);
 231                 break;
 232
 233         case 3:
 234                 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_IPI_1);
 235                 break;
 236
 237         case 4:
 238                 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_IPI_2);
 239                 break;
 240
 241         default:
 242                 break;
 243         }
 244
 245 }
 246
 247 static void kvm_vz_dequeue_io_int_cb(struct kvm_vcpu *vcpu,
 248                                      struct kvm_mips_interrupt *irq)
 249 {
 250         int intr = (int)irq->irq;
 251
 252         /*
 253          * interrupts are asynchronous to vcpu execution therefore defer guest
 254          * cp0 accesses
 255          */
 256         switch (intr) {
 257         case -2:
 258                 kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_IO);
 259                 break;
 260
 261         case -3:
 262                 kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_IPI_1);
 263                 break;
 264
 265         case -4:
 266                 kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_IPI_2);
 267                 break;
 268
 269         default:
 270                 break;
 271         }
 272
 273 }
 274
 275 static u32 kvm_vz_priority_to_irq[MIPS_EXC_MAX] = {
 276         [MIPS_EXC_INT_TIMER] = C_IRQ5,
 277         [MIPS_EXC_INT_IO]    = C_IRQ0,
 278         [MIPS_EXC_INT_IPI_1] = C_IRQ1,
 279         [MIPS_EXC_INT_IPI_2] = C_IRQ2,
 280 };
 281
 282 static int kvm_vz_irq_deliver_cb(struct kvm_vcpu *vcpu, unsigned int priority,
 283                                  u32 cause)
 284 {
 285         u32 irq = (priority < MIPS_EXC_MAX) ?
 286                 kvm_vz_priority_to_irq[priority] : 0;
 287
 288         switch (priority) {
 289         case MIPS_EXC_INT_TIMER:
 290                 set_gc0_cause(C_TI);
 291                 break;
 292
 293         case MIPS_EXC_INT_IO:
 294         case MIPS_EXC_INT_IPI_1:
 295         case MIPS_EXC_INT_IPI_2:
 296                 if (cpu_has_guestctl2)
 297                         set_c0_guestctl2(irq);
 298                 else
 299                         set_gc0_cause(irq);
 300                 break;
 301
 302         default:
 303                 break;
 304         }
 305
 306         clear_bit(priority, &vcpu->arch.pending_exceptions);
 307         return 1;
 308 }
 309
 310 static int kvm_vz_irq_clear_cb(struct kvm_vcpu *vcpu, unsigned int priority,
 311                                u32 cause)
 312 {
 313         u32 irq = (priority < MIPS_EXC_MAX) ?
 314                 kvm_vz_priority_to_irq[priority] : 0;
 315
 316         switch (priority) {
 317         case MIPS_EXC_INT_TIMER:
 318                 /*
 319                  * Call to kvm_write_c0_guest_compare() clears Cause.TI in
 320                  * kvm_mips_emulate_CP0(). Explicitly clear irq associated with
 321                  * Cause.IP[IPTI] if GuestCtl2 virtual interrupt register not
 322                  * supported or if not using GuestCtl2 Hardware Clear.
 323                  */
 324                 if (cpu_has_guestctl2) {
 325                         if (!(read_c0_guestctl2() & (irq << 14)))
 326                                 clear_c0_guestctl2(irq);
 327                 } else {
 328                         clear_gc0_cause(irq);
 329                 }
 330                 break;
 331
 332         case MIPS_EXC_INT_IO:
 333         case MIPS_EXC_INT_IPI_1:
 334         case MIPS_EXC_INT_IPI_2:
 335                 /* Clear GuestCtl2.VIP irq if not using Hardware Clear */
 336                 if (cpu_has_guestctl2) {
 337                         if (!(read_c0_guestctl2() & (irq << 14)))
 338                                 clear_c0_guestctl2(irq);
 339                 } else {
 340                         clear_gc0_cause(irq);
 341                 }
 342                 break;
 343
 344         default:
 345                 break;
 346         }
 347
 348         clear_bit(priority, &vcpu->arch.pending_exceptions_clr);
 349         return 1;
 350 }
 351
 352 /*
 353  * VZ guest timer handling.
 354  */
 355
 356 /**
 357  * kvm_vz_should_use_htimer() - Find whether to use the VZ hard guest timer.
 358  * @vcpu:       Virtual CPU.
 359  *
 360  * Returns:     true if the VZ GTOffset & real guest CP0_Count should be used
 361  *              instead of software emulation of guest timer.
 362  *              false otherwise.
 363  */
 364 static bool kvm_vz_should_use_htimer(struct kvm_vcpu *vcpu)
 365 {
 366         if (kvm_mips_count_disabled(vcpu))
 367                 return false;
 368
 369         /* Chosen frequency must match real frequency */
 370         if (mips_hpt_frequency != vcpu->arch.count_hz)
 371                 return false;
 372
 373         /* We don't support a CP0_GTOffset with fewer bits than CP0_Count */
 374         if (current_cpu_data.gtoffset_mask != 0xffffffff)
 375                 return false;
 376
 377         return true;
 378 }
 379
 380 /**
 381  * _kvm_vz_restore_stimer() - Restore soft timer state.
 382  * @vcpu:       Virtual CPU.
 383  * @compare:    CP0_Compare register value, restored by caller.
 384  * @cause:      CP0_Cause register to restore.
 385  *
 386  * Restore VZ state relating to the soft timer. The hard timer can be enabled
 387  * later.
 388  */
 389 static void _kvm_vz_restore_stimer(struct kvm_vcpu *vcpu, u32 compare,
 390                                    u32 cause)
 391 {
 392         /*
 393          * Avoid spurious counter interrupts by setting Guest CP0_Count to just
 394          * after Guest CP0_Compare.
 395          */
 396         write_c0_gtoffset(compare - read_c0_count());
 397
 398         back_to_back_c0_hazard();
 399         write_gc0_cause(cause);
 400 }
 401
 402 /**
 403  * _kvm_vz_restore_htimer() - Restore hard timer state.
 404  * @vcpu:       Virtual CPU.
 405  * @compare:    CP0_Compare register value, restored by caller.
 406  * @cause:      CP0_Cause register to restore.
 407  *
 408  * Restore hard timer Guest.Count & Guest.Cause taking care to preserve the
 409  * value of Guest.CP0_Cause.TI while restoring Guest.CP0_Cause.
 410  */
 411 static void _kvm_vz_restore_htimer(struct kvm_vcpu *vcpu,
 412                                    u32 compare, u32 cause)
 413 {
 414         u32 start_count, after_count;
 415         ktime_t freeze_time;
 416         unsigned long flags;
 417
 418         /*
 419          * Freeze the soft-timer and sync the guest CP0_Count with it. We do
 420          * this with interrupts disabled to avoid latency.
 421          */
 422         local_irq_save(flags);
 423         freeze_time = kvm_mips_freeze_hrtimer(vcpu, &start_count);
 424         write_c0_gtoffset(start_count - read_c0_count());
 425         local_irq_restore(flags);
 426
 427         /* restore guest CP0_Cause, as TI may already be set */
 428         back_to_back_c0_hazard();
 429         write_gc0_cause(cause);
 430
 431         /*
 432          * The above sequence isn't atomic and would result in lost timer
 433          * interrupts if we're not careful. Detect if a timer interrupt is due
 434          * and assert it.
 435          */
 436         back_to_back_c0_hazard();
 437         after_count = read_gc0_count();
 438         if (after_count - start_count > compare - start_count - 1)
 439                 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER);
 440 }
 441
 442 /**
 443  * kvm_vz_restore_timer() - Restore timer state.
 444  * @vcpu:       Virtual CPU.
 445  *
 446  * Restore soft timer state from saved context.
 447  */
 448 static void kvm_vz_restore_timer(struct kvm_vcpu *vcpu)
 449 {
 450         struct mips_coproc *cop0 = vcpu->arch.cop0;
 451         u32 cause, compare;
 452
 453         compare = kvm_read_sw_gc0_compare(cop0);
 454         cause = kvm_read_sw_gc0_cause(cop0);
 455
 456         write_gc0_compare(compare);
 457         _kvm_vz_restore_stimer(vcpu, compare, cause);
 458 }
 459
 460 /**
 461  * kvm_vz_acquire_htimer() - Switch to hard timer state.
 462  * @vcpu:       Virtual CPU.
 463  *
 464  * Restore hard timer state on top of existing soft timer state if possible.
 465  *
 466  * Since hard timer won't remain active over preemption, preemption should be
 467  * disabled by the caller.
 468  */
 469 void kvm_vz_acquire_htimer(struct kvm_vcpu *vcpu)
 470 {
 471         u32 gctl0;
 472
 473         gctl0 = read_c0_guestctl0();
 474         if (!(gctl0 & MIPS_GCTL0_GT) && kvm_vz_should_use_htimer(vcpu)) {
 475                 /* enable guest access to hard timer */
 476                 write_c0_guestctl0(gctl0 | MIPS_GCTL0_GT);
 477
 478                 _kvm_vz_restore_htimer(vcpu, read_gc0_compare(),
 479                                        read_gc0_cause());
 480         }
 481 }
 482
 483 /**
 484  * _kvm_vz_save_htimer() - Switch to software emulation of guest timer.
 485  * @vcpu:       Virtual CPU.
 486  * @compare:    Pointer to write compare value to.
 487  * @cause:      Pointer to write cause value to.
 488  *
 489  * Save VZ guest timer state and switch to software emulation of guest CP0
 490  * timer. The hard timer must already be in use, so preemption should be
 491  * disabled.
 492  */
 493 static void _kvm_vz_save_htimer(struct kvm_vcpu *vcpu,
 494                                 u32 *out_compare, u32 *out_cause)
 495 {
 496         u32 cause, compare, before_count, end_count;
 497         ktime_t before_time;
 498
 499         compare = read_gc0_compare();
 500         *out_compare = compare;
 501
 502         before_time = ktime_get();
 503
 504         /*
 505          * Record the CP0_Count *prior* to saving CP0_Cause, so we have a time
 506          * at which no pending timer interrupt is missing.
 507          */
 508         before_count = read_gc0_count();
 509         back_to_back_c0_hazard();
 510         cause = read_gc0_cause();
 511         *out_cause = cause;
 512
 513         /*
 514          * Record a final CP0_Count which we will transfer to the soft-timer.
 515          * This is recorded *after* saving CP0_Cause, so we don't get any timer
 516          * interrupts from just after the final CP0_Count point.
 517          */
 518         back_to_back_c0_hazard();
 519         end_count = read_gc0_count();
 520
 521         /*
 522          * The above sequence isn't atomic, so we could miss a timer interrupt
 523          * between reading CP0_Cause and end_count. Detect and record any timer
 524          * interrupt due between before_count and end_count.
 525          */
 526         if (end_count - before_count > compare - before_count - 1)
 527                 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER);
 528
 529         /*
 530          * Restore soft-timer, ignoring a small amount of negative drift due to
 531          * delay between freeze_hrtimer and setting CP0_GTOffset.
 532          */
 533         kvm_mips_restore_hrtimer(vcpu, before_time, end_count, -0x10000);
 534 }
 535
 536 /**
 537  * kvm_vz_save_timer() - Save guest timer state.
 538  * @vcpu:       Virtual CPU.
 539  *
 540  * Save VZ guest timer state and switch to soft guest timer if hard timer was in
 541  * use.
 542  */
 543 static void kvm_vz_save_timer(struct kvm_vcpu *vcpu)
 544 {
 545         struct mips_coproc *cop0 = vcpu->arch.cop0;
 546         u32 gctl0, compare, cause;
 547
 548         gctl0 = read_c0_guestctl0();
 549         if (gctl0 & MIPS_GCTL0_GT) {
 550                 /* disable guest use of hard timer */
 551                 write_c0_guestctl0(gctl0 & ~MIPS_GCTL0_GT);
 552
 553                 /* save hard timer state */
 554                 _kvm_vz_save_htimer(vcpu, &compare, &cause);
 555         } else {
 556                 compare = read_gc0_compare();
 557                 cause = read_gc0_cause();
 558         }
 559
 560         /* save timer-related state to VCPU context */
 561         kvm_write_sw_gc0_cause(cop0, cause);
 562         kvm_write_sw_gc0_compare(cop0, compare);
 563 }
 564
 565 /**
 566  * kvm_vz_lose_htimer() - Ensure hard guest timer is not in use.
 567  * @vcpu:       Virtual CPU.
 568  *
 569  * Transfers the state of the hard guest timer to the soft guest timer, leaving
 570  * guest state intact so it can continue to be used with the soft timer.
 571  */
 572 void kvm_vz_lose_htimer(struct kvm_vcpu *vcpu)
 573 {
 574         u32 gctl0, compare, cause;
 575
 576         preempt_disable();
 577         gctl0 = read_c0_guestctl0();
 578         if (gctl0 & MIPS_GCTL0_GT) {
 579                 /* disable guest use of timer */
 580                 write_c0_guestctl0(gctl0 & ~MIPS_GCTL0_GT);
 581
 582                 /* switch to soft timer */
 583                 _kvm_vz_save_htimer(vcpu, &compare, &cause);
 584
 585                 /* leave soft timer in usable state */
 586                 _kvm_vz_restore_stimer(vcpu, compare, cause);
 587         }
 588         preempt_enable();
 589 }
 590
 591 /**
 592  * is_eva_access() - Find whether an instruction is an EVA memory accessor.
 593  * @inst:       32-bit instruction encoding.
 594  *
 595  * Finds whether @inst encodes an EVA memory access instruction, which would
 596  * indicate that emulation of it should access the user mode address space
 597  * instead of the kernel mode address space. This matters for MUSUK segments
 598  * which are TLB mapped for user mode but unmapped for kernel mode.
 599  *
 600  * Returns:     Whether @inst encodes an EVA accessor instruction.
 601  */
 602 static bool is_eva_access(union mips_instruction inst)
 603 {
 604         if (inst.spec3_format.opcode != spec3_op)
 605                 return false;
 606
 607         switch (inst.spec3_format.func) {
 608         case lwle_op:
 609         case lwre_op:
 610         case cachee_op:
 611         case sbe_op:
 612         case she_op:
 613         case sce_op:
 614         case swe_op:
 615         case swle_op:
 616         case swre_op:
 617         case prefe_op:
 618         case lbue_op:
 619         case lhue_op:
 620         case lbe_op:
 621         case lhe_op:
 622         case lle_op:
 623         case lwe_op:
 624                 return true;
 625         default:
 626                 return false;
 627         }
 628 }
 629
 630 /**
 631  * is_eva_am_mapped() - Find whether an access mode is mapped.
 632  * @vcpu:       KVM VCPU state.
 633  * @am:         3-bit encoded access mode.
 634  * @eu:         Segment becomes unmapped and uncached when Status.ERL=1.
 635  *
 636  * Decode @am to find whether it encodes a mapped segment for the current VCPU
 637  * state. Where necessary @eu and the actual instruction causing the fault are
 638  * taken into account to make the decision.
 639  *
 640  * Returns:     Whether the VCPU faulted on a TLB mapped address.
 641  */
 642 static bool is_eva_am_mapped(struct kvm_vcpu *vcpu, unsigned int am, bool eu)
 643 {
 644         u32 am_lookup;
 645         int err;
 646
 647         /*
 648          * Interpret access control mode. We assume address errors will already
 649          * have been caught by the guest, leaving us with:
 650          *      AM      UM  SM  KM  31..24 23..16
 651          * UK    0 000          Unm   0      0
 652          * MK    1 001          TLB   1
 653          * MSK   2 010      TLB TLB   1
 654          * MUSK  3 011  TLB TLB TLB   1
 655          * MUSUK 4 100  TLB TLB Unm   0      1
 656          * USK   5 101      Unm Unm   0      0
 657          * -     6 110                0      0
 658          * UUSK  7 111  Unm Unm Unm   0      0
 659          *
 660          * We shift a magic value by AM across the sign bit to find if always
 661          * TLB mapped, and if not shift by 8 again to find if it depends on KM.
 662          */
 663         am_lookup = 0x70080000 << am;
 664         if ((s32)am_lookup < 0) {
 665                 /*
 666                  * MK, MSK, MUSK
 667                  * Always TLB mapped, unless SegCtl.EU && ERL
 668                  */
 669                 if (!eu || !(read_gc0_status() & ST0_ERL))
 670                         return true;
 671         } else {
 672                 am_lookup <<= 8;
 673                 if ((s32)am_lookup < 0) {
 674                         union mips_instruction inst;
 675                         unsigned int status;
 676                         u32 *opc;
 677
 678                         /*
 679                          * MUSUK
 680                          * TLB mapped if not in kernel mode
 681                          */
 682                         status = read_gc0_status();
 683                         if (!(status & (ST0_EXL | ST0_ERL)) &&
 684                             (status & ST0_KSU))
 685                                 return true;
 686                         /*
 687                          * EVA access instructions in kernel
 688                          * mode access user address space.
 689                          */
 690                         opc = (u32 *)vcpu->arch.pc;
 691                         if (vcpu->arch.host_cp0_cause & CAUSEF_BD)
 692                                 opc += 1;
 693                         err = kvm_get_badinstr(opc, vcpu, &inst.word);
 694                         if (!err && is_eva_access(inst))
 695                                 return true;
 696                 }
 697         }
 698
 699         return false;
 700 }
 701
 702 /**
 703  * kvm_vz_gva_to_gpa() - Convert valid GVA to GPA.
 704  * @vcpu:       KVM VCPU state.
 705  * @gva:        Guest virtual address to convert.
 706  * @gpa:        Output guest physical address.
 707  *
 708  * Convert a guest virtual address (GVA) which is valid according to the guest
 709  * context, to a guest physical address (GPA).
 710  *
 711  * Returns:     0 on success.
 712  *              -errno on failure.
 713  */
 714 static int kvm_vz_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
 715                              unsigned long *gpa)
 716 {
 717         u32 gva32 = gva;
 718         unsigned long segctl;
 719
 720         if ((long)gva == (s32)gva32) {
 721                 /* Handle canonical 32-bit virtual address */
 722                 if (cpu_guest_has_segments) {
 723                         unsigned long mask, pa;
 724
 725                         switch (gva32 >> 29) {
 726                         case 0:
 727                         case 1: /* CFG5 (1GB) */
 728                                 segctl = read_gc0_segctl2() >> 16;
 729                                 mask = (unsigned long)0xfc0000000ull;
 730                                 break;
 731                         case 2:
 732                         case 3: /* CFG4 (1GB) */
 733                                 segctl = read_gc0_segctl2();
 734                                 mask = (unsigned long)0xfc0000000ull;
 735                                 break;
 736                         case 4: /* CFG3 (512MB) */
 737                                 segctl = read_gc0_segctl1() >> 16;
 738                                 mask = (unsigned long)0xfe0000000ull;
 739                                 break;
 740                         case 5: /* CFG2 (512MB) */
 741                                 segctl = read_gc0_segctl1();
 742                                 mask = (unsigned long)0xfe0000000ull;
 743                                 break;
 744                         case 6: /* CFG1 (512MB) */
 745                                 segctl = read_gc0_segctl0() >> 16;
 746                                 mask = (unsigned long)0xfe0000000ull;
 747                                 break;
 748                         case 7: /* CFG0 (512MB) */
 749                                 segctl = read_gc0_segctl0();
 750                                 mask = (unsigned long)0xfe0000000ull;
 751                                 break;
 752                         default:
 753                                 /*
 754                                  * GCC 4.9 isn't smart enough to figure out that
 755                                  * segctl and mask are always initialised.
 756                                  */
 757                                 unreachable();
 758                         }
 759
 760                         if (is_eva_am_mapped(vcpu, (segctl >> 4) & 0x7,
 761                                              segctl & 0x0008))
 762                                 goto tlb_mapped;
 763
 764                         /* Unmapped, find guest physical address */
 765                         pa = (segctl << 20) & mask;
 766                         pa |= gva32 & ~mask;
 767                         *gpa = pa;
 768                         return 0;
 769                 } else if ((s32)gva32 < (s32)0xc0000000) {
 770                         /* legacy unmapped KSeg0 or KSeg1 */
 771                         *gpa = gva32 & 0x1fffffff;
 772                         return 0;
 773                 }
 774 #ifdef CONFIG_64BIT
 775         } else if ((gva & 0xc000000000000000) == 0x8000000000000000) {
 776                 /* XKPHYS */
 777                 if (cpu_guest_has_segments) {
 778                         /*
 779                          * Each of the 8 regions can be overridden by SegCtl2.XR
 780                          * to use SegCtl1.XAM.
 781                          */
 782                         segctl = read_gc0_segctl2();
 783                         if (segctl & (1ull << (56 + ((gva >> 59) & 0x7)))) {
 784                                 segctl = read_gc0_segctl1();
 785                                 if (is_eva_am_mapped(vcpu, (segctl >> 59) & 0x7,
 786                                                      0))
 787                                         goto tlb_mapped;
 788                         }
 789
 790                 }
 791                 /*
 792                  * Traditionally fully unmapped.
 793                  * Bits 61:59 specify the CCA, which we can just mask off here.
 794                  * Bits 58:PABITS should be zero, but we shouldn't have got here
 795                  * if it wasn't.
 796                  */
 797                 *gpa = gva & 0x07ffffffffffffff;
 798                 return 0;
 799 #endif
 800         }
 801
 802 tlb_mapped:
 803         return kvm_vz_guest_tlb_lookup(vcpu, gva, gpa);
 804 }
 805
 806 /**
 807  * kvm_vz_badvaddr_to_gpa() - Convert GVA BadVAddr from root exception to GPA.
 808  * @vcpu:       KVM VCPU state.
 809  * @badvaddr:   Root BadVAddr.
 810  * @gpa:        Output guest physical address.
 811  *
 812  * VZ implementations are permitted to report guest virtual addresses (GVA) in
 813  * BadVAddr on a root exception during guest execution, instead of the more
 814  * convenient guest physical addresses (GPA). When we get a GVA, this function
 815  * converts it to a GPA, taking into account guest segmentation and guest TLB
 816  * state.
 817  *
 818  * Returns:     0 on success.
 819  *              -errno on failure.
 820  */
 821 static int kvm_vz_badvaddr_to_gpa(struct kvm_vcpu *vcpu, unsigned long badvaddr,
 822                                   unsigned long *gpa)
 823 {
 824         unsigned int gexccode = (vcpu->arch.host_cp0_guestctl0 &
 825                                  MIPS_GCTL0_GEXC) >> MIPS_GCTL0_GEXC_SHIFT;
 826
 827         /* If BadVAddr is GPA, then all is well in the world */
 828         if (likely(gexccode == MIPS_GCTL0_GEXC_GPA)) {
 829                 *gpa = badvaddr;
 830                 return 0;
 831         }
 832
 833         /* Otherwise we'd expect it to be GVA ... */
 834         if (WARN(gexccode != MIPS_GCTL0_GEXC_GVA,
 835                  "Unexpected gexccode %#x\n", gexccode))
 836                 return -EINVAL;
 837
 838         /* ... and we need to perform the GVA->GPA translation in software */
 839         return kvm_vz_gva_to_gpa(vcpu, badvaddr, gpa);
 840 }
 841
 842 static int kvm_trap_vz_no_handler(struct kvm_vcpu *vcpu)
 843 {
 844         u32 *opc = (u32 *) vcpu->arch.pc;
 845         u32 cause = vcpu->arch.host_cp0_cause;
 846         u32 exccode = (cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE;
 847         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
 848         u32 inst = 0;
 849
 850         /*
 851          *  Fetch the instruction.
 852          */
 853         if (cause & CAUSEF_BD)
 854                 opc += 1;
 855         kvm_get_badinstr(opc, vcpu, &inst);
 856
 857         kvm_err("Exception Code: %d not handled @ PC: %p, inst: 0x%08x BadVaddr: %#lx Status: %#x\n",
 858                 exccode, opc, inst, badvaddr,
 859                 read_gc0_status());
 860         kvm_arch_vcpu_dump_regs(vcpu);
 861         vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 862         return RESUME_HOST;
 863 }
 864
 865 static unsigned long mips_process_maar(unsigned int op, unsigned long val)
 866 {
 867         /* Mask off unused bits */
 868         unsigned long mask = 0xfffff000 | MIPS_MAAR_S | MIPS_MAAR_VL;
 869
 870         if (read_gc0_pagegrain() & PG_ELPA)
 871                 mask |= 0x00ffffff00000000ull;
 872         if (cpu_guest_has_mvh)
 873                 mask |= MIPS_MAAR_VH;
 874
 875         /* Set or clear VH */
 876         if (op == mtc_op) {
 877                 /* clear VH */
 878                 val &= ~MIPS_MAAR_VH;
 879         } else if (op == dmtc_op) {
 880                 /* set VH to match VL */
 881                 val &= ~MIPS_MAAR_VH;
 882                 if (val & MIPS_MAAR_VL)
 883                         val |= MIPS_MAAR_VH;
 884         }
 885
 886         return val & mask;
 887 }
 888
 889 static void kvm_write_maari(struct kvm_vcpu *vcpu, unsigned long val)
 890 {
 891         struct mips_coproc *cop0 = vcpu->arch.cop0;
 892
 893         val &= MIPS_MAARI_INDEX;
 894         if (val == MIPS_MAARI_INDEX)
 895                 kvm_write_sw_gc0_maari(cop0, ARRAY_SIZE(vcpu->arch.maar) - 1);
 896         else if (val < ARRAY_SIZE(vcpu->arch.maar))
 897                 kvm_write_sw_gc0_maari(cop0, val);
 898 }
 899
 900 static enum emulation_result kvm_vz_gpsi_cop0(union mips_instruction inst,
 901                                               u32 *opc, u32 cause,
 902                                               struct kvm_run *run,
 903                                               struct kvm_vcpu *vcpu)
 904 {
 905         struct mips_coproc *cop0 = vcpu->arch.cop0;
 906         enum emulation_result er = EMULATE_DONE;
 907         u32 rt, rd, sel;
 908         unsigned long curr_pc;
 909         unsigned long val;
 910
 911         /*
 912          * Update PC and hold onto current PC in case there is
 913          * an error and we want to rollback the PC
 914          */
 915         curr_pc = vcpu->arch.pc;
 916         er = update_pc(vcpu, cause);
 917         if (er == EMULATE_FAIL)
 918                 return er;
 919
 920         if (inst.co_format.co) {
 921                 switch (inst.co_format.func) {
 922                 case wait_op:
 923                         er = kvm_mips_emul_wait(vcpu);
 924                         break;
 925                 default:
 926                         er = EMULATE_FAIL;
 927                 }
 928         } else {
 929                 rt = inst.c0r_format.rt;
 930                 rd = inst.c0r_format.rd;
 931                 sel = inst.c0r_format.sel;
 932
 933                 switch (inst.c0r_format.rs) {
 934                 case dmfc_op:
 935                 case mfc_op:
 936 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
 937                         cop0->stat[rd][sel]++;
 938 #endif
 939                         if (rd == MIPS_CP0_COUNT &&
 940                             sel == 0) {                 /* Count */
 941                                 val = kvm_mips_read_count(vcpu);
 942                         } else if (rd == MIPS_CP0_COMPARE &&
 943                                    sel == 0) {          /* Compare */
 944                                 val = read_gc0_compare();
 945                         } else if (rd == MIPS_CP0_LLADDR &&
 946                                    sel == 0) {          /* LLAddr */
 947                                 if (cpu_guest_has_rw_llb)
 948                                         val = read_gc0_lladdr() &
 949                                                 MIPS_LLADDR_LLB;
 950                                 else
 951                                         val = 0;
 952                         } else if (rd == MIPS_CP0_LLADDR &&
 953                                    sel == 1 &&          /* MAAR */
 954                                    cpu_guest_has_maar &&
 955                                    !cpu_guest_has_dyn_maar) {
 956                                 /* MAARI must be in range */
 957                                 BUG_ON(kvm_read_sw_gc0_maari(cop0) >=
 958                                                 ARRAY_SIZE(vcpu->arch.maar));
 959                                 val = vcpu->arch.maar[
 960                                         kvm_read_sw_gc0_maari(cop0)];
 961                         } else if ((rd == MIPS_CP0_PRID &&
 962                                     (sel == 0 ||        /* PRid */
 963                                      sel == 2 ||        /* CDMMBase */
 964                                      sel == 3)) ||      /* CMGCRBase */
 965                                    (rd == MIPS_CP0_STATUS &&
 966                                     (sel == 2 ||        /* SRSCtl */
 967                                      sel == 3)) ||      /* SRSMap */
 968                                    (rd == MIPS_CP0_CONFIG &&
 969                                     (sel == 7)) ||      /* Config7 */
 970                                    (rd == MIPS_CP0_LLADDR &&
 971                                     (sel == 2) &&       /* MAARI */
 972                                     cpu_guest_has_maar &&
 973                                     !cpu_guest_has_dyn_maar) ||
 974                                    (rd == MIPS_CP0_ERRCTL &&
 975                                     (sel == 0))) {      /* ErrCtl */
 976                                 val = cop0->reg[rd][sel];
 977                         } else {
 978                                 val = 0;
 979                                 er = EMULATE_FAIL;
 980                         }
 981
 982                         if (er != EMULATE_FAIL) {
 983                                 /* Sign extend */
 984                                 if (inst.c0r_format.rs == mfc_op)
 985                                         val = (int)val;
 986                                 vcpu->arch.gprs[rt] = val;
 987                         }
 988
 989                         trace_kvm_hwr(vcpu, (inst.c0r_format.rs == mfc_op) ?
 990                                         KVM_TRACE_MFC0 : KVM_TRACE_DMFC0,
 991                                       KVM_TRACE_COP0(rd, sel), val);
 992                         break;
 993
 994                 case dmtc_op:
 995                 case mtc_op:
 996 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
 997                         cop0->stat[rd][sel]++;
 998 #endif
 999                         val = vcpu->arch.gprs[rt];
1000                         trace_kvm_hwr(vcpu, (inst.c0r_format.rs == mtc_op) ?
1001                                         KVM_TRACE_MTC0 : KVM_TRACE_DMTC0,
1002                                       KVM_TRACE_COP0(rd, sel), val);
1003
1004                         if (rd == MIPS_CP0_COUNT &&
1005                             sel == 0) {                 /* Count */
1006                                 kvm_vz_lose_htimer(vcpu);
1007                                 kvm_mips_write_count(vcpu, vcpu->arch.gprs[rt]);
1008                         } else if (rd == MIPS_CP0_COMPARE &&
1009                                    sel == 0) {          /* Compare */
1010                                 kvm_mips_write_compare(vcpu,
1011                                                        vcpu->arch.gprs[rt],
1012                                                        true);
1013                         } else if (rd == MIPS_CP0_LLADDR &&
1014                                    sel == 0) {          /* LLAddr */
1015                                 /*
1016                                  * P5600 generates GPSI on guest MTC0 LLAddr.
1017                                  * Only allow the guest to clear LLB.
1018                                  */
1019                                 if (cpu_guest_has_rw_llb &&
1020                                     !(val & MIPS_LLADDR_LLB))
1021                                         write_gc0_lladdr(0);
1022                         } else if (rd == MIPS_CP0_LLADDR &&
1023                                    sel == 1 &&          /* MAAR */
1024                                    cpu_guest_has_maar &&
1025                                    !cpu_guest_has_dyn_maar) {
1026                                 val = mips_process_maar(inst.c0r_format.rs,
1027                                                         val);
1028
1029                                 /* MAARI must be in range */
1030                                 BUG_ON(kvm_read_sw_gc0_maari(cop0) >=
1031                                                 ARRAY_SIZE(vcpu->arch.maar));
1032                                 vcpu->arch.maar[kvm_read_sw_gc0_maari(cop0)] =
1033                                                                         val;
1034                         } else if (rd == MIPS_CP0_LLADDR &&
1035                                    (sel == 2) &&        /* MAARI */
1036                                    cpu_guest_has_maar &&
1037                                    !cpu_guest_has_dyn_maar) {
1038                                 kvm_write_maari(vcpu, val);
1039                         } else if (rd == MIPS_CP0_ERRCTL &&
1040                                    (sel == 0)) {        /* ErrCtl */
1041                                 /* ignore the written value */
1042                         } else {
1043                                 er = EMULATE_FAIL;
1044                         }
1045                         break;
1046
1047                 default:
1048                         er = EMULATE_FAIL;
1049                         break;
1050                 }
1051         }
1052         /* Rollback PC only if emulation was unsuccessful */
1053         if (er == EMULATE_FAIL) {
1054                 kvm_err("[%#lx]%s: unsupported cop0 instruction 0x%08x\n",
1055                         curr_pc, __func__, inst.word);
1056
1057                 vcpu->arch.pc = curr_pc;
1058         }
1059
1060         return er;
1061 }
1062
1063 static enum emulation_result kvm_vz_gpsi_cache(union mips_instruction inst,
1064                                                u32 *opc, u32 cause,
1065                                                struct kvm_run *run,
1066                                                struct kvm_vcpu *vcpu)
1067 {
1068         enum emulation_result er = EMULATE_DONE;
1069         u32 cache, op_inst, op, base;
1070         s16 offset;
1071         struct kvm_vcpu_arch *arch = &vcpu->arch;
1072         unsigned long va, curr_pc;
1073
1074         /*
1075          * Update PC and hold onto current PC in case there is
1076          * an error and we want to rollback the PC
1077          */
1078         curr_pc = vcpu->arch.pc;
1079         er = update_pc(vcpu, cause);
1080         if (er == EMULATE_FAIL)
1081                 return er;
1082
1083         base = inst.i_format.rs;
1084         op_inst = inst.i_format.rt;
1085         if (cpu_has_mips_r6)
1086                 offset = inst.spec3_format.simmediate;
1087         else
1088                 offset = inst.i_format.simmediate;
1089         cache = op_inst & CacheOp_Cache;
1090         op = op_inst & CacheOp_Op;
1091
1092         va = arch->gprs[base] + offset;
1093
1094         kvm_debug("CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1095                   cache, op, base, arch->gprs[base], offset);
1096
1097         /* Secondary or tirtiary cache ops ignored */
1098         if (cache != Cache_I && cache != Cache_D)
1099                 return EMULATE_DONE;
1100
1101         switch (op_inst) {
1102         case Index_Invalidate_I:
1103                 flush_icache_line_indexed(va);
1104                 return EMULATE_DONE;
1105         case Index_Writeback_Inv_D:
1106                 flush_dcache_line_indexed(va);
1107                 return EMULATE_DONE;
1108         case Hit_Invalidate_I:
1109         case Hit_Invalidate_D:
1110         case Hit_Writeback_Inv_D:
1111                 if (boot_cpu_type() == CPU_CAVIUM_OCTEON3) {
1112                         /* We can just flush entire icache */
1113                         local_flush_icache_range(0, 0);
1114                         return EMULATE_DONE;
1115                 }
1116
1117                 /* So far, other platforms support guest hit cache ops */
1118                 break;
1119         default:
1120                 break;
1121         };
1122
1123         kvm_err("@ %#lx/%#lx CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1124                 curr_pc, vcpu->arch.gprs[31], cache, op, base, arch->gprs[base],
1125                 offset);
1126         /* Rollback PC */
1127         vcpu->arch.pc = curr_pc;
1128
1129         return EMULATE_FAIL;
1130 }
1131
1132 static enum emulation_result kvm_trap_vz_handle_gpsi(u32 cause, u32 *opc,
1133                                                      struct kvm_vcpu *vcpu)
1134 {
1135         enum emulation_result er = EMULATE_DONE;
1136         struct kvm_vcpu_arch *arch = &vcpu->arch;
1137         struct kvm_run *run = vcpu->run;
1138         union mips_instruction inst;
1139         int rd, rt, sel;
1140         int err;
1141
1142         /*
1143          *  Fetch the instruction.
1144          */
1145         if (cause & CAUSEF_BD)
1146                 opc += 1;
1147         err = kvm_get_badinstr(opc, vcpu, &inst.word);
1148         if (err)
1149                 return EMULATE_FAIL;
1150
1151         switch (inst.r_format.opcode) {
1152         case cop0_op:
1153                 er = kvm_vz_gpsi_cop0(inst, opc, cause, run, vcpu);
1154                 break;
1155 #ifndef CONFIG_CPU_MIPSR6
1156         case cache_op:
1157                 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
1158                 er = kvm_vz_gpsi_cache(inst, opc, cause, run, vcpu);
1159                 break;
1160 #endif
1161         case spec3_op:
1162                 switch (inst.spec3_format.func) {
1163 #ifdef CONFIG_CPU_MIPSR6
1164                 case cache6_op:
1165                         trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
1166                         er = kvm_vz_gpsi_cache(inst, opc, cause, run, vcpu);
1167                         break;
1168 #endif
1169                 case rdhwr_op:
1170                         if (inst.r_format.rs || (inst.r_format.re >> 3))
1171                                 goto unknown;
1172
1173                         rd = inst.r_format.rd;
1174                         rt = inst.r_format.rt;
1175                         sel = inst.r_format.re & 0x7;
1176
1177                         switch (rd) {
1178                         case MIPS_HWR_CC:       /* Read count register */
1179                                 arch->gprs[rt] =
1180                                         (long)(int)kvm_mips_read_count(vcpu);
1181                                 break;
1182                         default:
1183                                 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR,
1184                                               KVM_TRACE_HWR(rd, sel), 0);
1185                                 goto unknown;
1186                         };
1187
1188                         trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR,
1189                                       KVM_TRACE_HWR(rd, sel), arch->gprs[rt]);
1190
1191                         er = update_pc(vcpu, cause);
1192                         break;
1193                 default:
1194                         goto unknown;
1195                 };
1196                 break;
1197 unknown:
1198
1199         default:
1200                 kvm_err("GPSI exception not supported (%p/%#x)\n",
1201                                 opc, inst.word);
1202                 kvm_arch_vcpu_dump_regs(vcpu);
1203                 er = EMULATE_FAIL;
1204                 break;
1205         }
1206
1207         return er;
1208 }
1209
1210 static enum emulation_result kvm_trap_vz_handle_gsfc(u32 cause, u32 *opc,
1211                                                      struct kvm_vcpu *vcpu)
1212 {
1213         enum emulation_result er = EMULATE_DONE;
1214         struct kvm_vcpu_arch *arch = &vcpu->arch;
1215         union mips_instruction inst;
1216         int err;
1217
1218         /*
1219          *  Fetch the instruction.
1220          */
1221         if (cause & CAUSEF_BD)
1222                 opc += 1;
1223         err = kvm_get_badinstr(opc, vcpu, &inst.word);
1224         if (err)
1225                 return EMULATE_FAIL;
1226
1227         /* complete MTC0 on behalf of guest and advance EPC */
1228         if (inst.c0r_format.opcode == cop0_op &&
1229             inst.c0r_format.rs == mtc_op &&
1230             inst.c0r_format.z == 0) {
1231                 int rt = inst.c0r_format.rt;
1232                 int rd = inst.c0r_format.rd;
1233                 int sel = inst.c0r_format.sel;
1234                 unsigned int val = arch->gprs[rt];
1235                 unsigned int old_val, change;
1236
1237                 trace_kvm_hwr(vcpu, KVM_TRACE_MTC0, KVM_TRACE_COP0(rd, sel),
1238                               val);
1239
1240                 if ((rd == MIPS_CP0_STATUS) && (sel == 0)) {
1241                         /* FR bit should read as zero if no FPU */
1242                         if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1243                                 val &= ~(ST0_CU1 | ST0_FR);
1244
1245                         /*
1246                          * Also don't allow FR to be set if host doesn't support
1247                          * it.
1248                          */
1249                         if (!(boot_cpu_data.fpu_id & MIPS_FPIR_F64))
1250                                 val &= ~ST0_FR;
1251
1252                         old_val = read_gc0_status();
1253                         change = val ^ old_val;
1254
1255                         if (change & ST0_FR) {
1256                                 /*
1257                                  * FPU and Vector register state is made
1258                                  * UNPREDICTABLE by a change of FR, so don't
1259                                  * even bother saving it.
1260                                  */
1261                                 kvm_drop_fpu(vcpu);
1262                         }
1263
1264                         /*
1265                          * If MSA state is already live, it is undefined how it
1266                          * interacts with FR=0 FPU state, and we don't want to
1267                          * hit reserved instruction exceptions trying to save
1268                          * the MSA state later when CU=1 && FR=1, so play it
1269                          * safe and save it first.
1270                          */
1271                         if (change & ST0_CU1 && !(val & ST0_FR) &&
1272                             vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
1273                                 kvm_lose_fpu(vcpu);
1274
1275                         write_gc0_status(val);
1276                 } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) {
1277                         u32 old_cause = read_gc0_cause();
1278                         u32 change = old_cause ^ val;
1279
1280                         /* DC bit enabling/disabling timer? */
1281                         if (change & CAUSEF_DC) {
1282                                 if (val & CAUSEF_DC) {
1283                                         kvm_vz_lose_htimer(vcpu);
1284                                         kvm_mips_count_disable_cause(vcpu);
1285                                 } else {
1286                                         kvm_mips_count_enable_cause(vcpu);
1287                                 }
1288                         }
1289
1290                         /* Only certain bits are RW to the guest */
1291                         change &= (CAUSEF_DC | CAUSEF_IV | CAUSEF_WP |
1292                                    CAUSEF_IP0 | CAUSEF_IP1);
1293
1294                         /* WP can only be cleared */
1295                         change &= ~CAUSEF_WP | old_cause;
1296
1297                         write_gc0_cause(old_cause ^ change);
1298                 } else if ((rd == MIPS_CP0_STATUS) && (sel == 1)) { /* IntCtl */
1299                         write_gc0_intctl(val);
1300                 } else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) {
1301                         old_val = read_gc0_config5();
1302                         change = val ^ old_val;
1303                         /* Handle changes in FPU/MSA modes */
1304                         preempt_disable();
1305
1306                         /*
1307                          * Propagate FRE changes immediately if the FPU
1308                          * context is already loaded.
1309                          */
1310                         if (change & MIPS_CONF5_FRE &&
1311                             vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)
1312                                 change_c0_config5(MIPS_CONF5_FRE, val);
1313
1314                         preempt_enable();
1315
1316                         val = old_val ^
1317                                 (change & kvm_vz_config5_guest_wrmask(vcpu));
1318                         write_gc0_config5(val);
1319                 } else {
1320                         kvm_err("Handle GSFC, unsupported field change @ %p: %#x\n",
1321                             opc, inst.word);
1322                         er = EMULATE_FAIL;
1323                 }
1324
1325                 if (er != EMULATE_FAIL)
1326                         er = update_pc(vcpu, cause);
1327         } else {
1328                 kvm_err("Handle GSFC, unrecognized instruction @ %p: %#x\n",
1329                         opc, inst.word);
1330                 er = EMULATE_FAIL;
1331         }
1332
1333         return er;
1334 }
1335
1336 static enum emulation_result kvm_trap_vz_handle_ghfc(u32 cause, u32 *opc,
1337                                                      struct kvm_vcpu *vcpu)
1338 {
1339         /*
1340          * Presumably this is due to MC (guest mode change), so lets trace some
1341          * relevant info.
1342          */
1343         trace_kvm_guest_mode_change(vcpu);
1344
1345         return EMULATE_DONE;
1346 }
1347
1348 static enum emulation_result kvm_trap_vz_handle_hc(u32 cause, u32 *opc,
1349                                                    struct kvm_vcpu *vcpu)
1350 {
1351         enum emulation_result er;
1352         union mips_instruction inst;
1353         unsigned long curr_pc;
1354         int err;
1355
1356         if (cause & CAUSEF_BD)
1357                 opc += 1;
1358         err = kvm_get_badinstr(opc, vcpu, &inst.word);
1359         if (err)
1360                 return EMULATE_FAIL;
1361
1362         /*
1363          * Update PC and hold onto current PC in case there is
1364          * an error and we want to rollback the PC
1365          */
1366         curr_pc = vcpu->arch.pc;
1367         er = update_pc(vcpu, cause);
1368         if (er == EMULATE_FAIL)
1369                 return er;
1370
1371         er = kvm_mips_emul_hypcall(vcpu, inst);
1372         if (er == EMULATE_FAIL)
1373                 vcpu->arch.pc = curr_pc;
1374
1375         return er;
1376 }
1377
1378 static enum emulation_result kvm_trap_vz_no_handler_guest_exit(u32 gexccode,
1379                                                         u32 cause,
1380                                                         u32 *opc,
1381                                                         struct kvm_vcpu *vcpu)
1382 {
1383         u32 inst;
1384
1385         /*
1386          *  Fetch the instruction.
1387          */
1388         if (cause & CAUSEF_BD)
1389                 opc += 1;
1390         kvm_get_badinstr(opc, vcpu, &inst);
1391
1392         kvm_err("Guest Exception Code: %d not yet handled @ PC: %p, inst: 0x%08x  Status: %#x\n",
1393                 gexccode, opc, inst, read_gc0_status());
1394
1395         return EMULATE_FAIL;
1396 }
1397
1398 static int kvm_trap_vz_handle_guest_exit(struct kvm_vcpu *vcpu)
1399 {
1400         u32 *opc = (u32 *) vcpu->arch.pc;
1401         u32 cause = vcpu->arch.host_cp0_cause;
1402         enum emulation_result er = EMULATE_DONE;
1403         u32 gexccode = (vcpu->arch.host_cp0_guestctl0 &
1404                         MIPS_GCTL0_GEXC) >> MIPS_GCTL0_GEXC_SHIFT;
1405         int ret = RESUME_GUEST;
1406
1407         trace_kvm_exit(vcpu, KVM_TRACE_EXIT_GEXCCODE_BASE + gexccode);
1408         switch (gexccode) {
1409         case MIPS_GCTL0_GEXC_GPSI:
1410                 ++vcpu->stat.vz_gpsi_exits;
1411                 er = kvm_trap_vz_handle_gpsi(cause, opc, vcpu);
1412                 break;
1413         case MIPS_GCTL0_GEXC_GSFC:
1414                 ++vcpu->stat.vz_gsfc_exits;
1415                 er = kvm_trap_vz_handle_gsfc(cause, opc, vcpu);
1416                 break;
1417         case MIPS_GCTL0_GEXC_HC:
1418                 ++vcpu->stat.vz_hc_exits;
1419                 er = kvm_trap_vz_handle_hc(cause, opc, vcpu);
1420                 break;
1421         case MIPS_GCTL0_GEXC_GRR:
1422                 ++vcpu->stat.vz_grr_exits;
1423                 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc,
1424                                                        vcpu);
1425                 break;
1426         case MIPS_GCTL0_GEXC_GVA:
1427                 ++vcpu->stat.vz_gva_exits;
1428                 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc,
1429                                                        vcpu);
1430                 break;
1431         case MIPS_GCTL0_GEXC_GHFC:
1432                 ++vcpu->stat.vz_ghfc_exits;
1433                 er = kvm_trap_vz_handle_ghfc(cause, opc, vcpu);
1434                 break;
1435         case MIPS_GCTL0_GEXC_GPA:
1436                 ++vcpu->stat.vz_gpa_exits;
1437                 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc,
1438                                                        vcpu);
1439                 break;
1440         default:
1441                 ++vcpu->stat.vz_resvd_exits;
1442                 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc,
1443                                                        vcpu);
1444                 break;
1445
1446         }
1447
1448         if (er == EMULATE_DONE) {
1449                 ret = RESUME_GUEST;
1450         } else if (er == EMULATE_HYPERCALL) {
1451                 ret = kvm_mips_handle_hypcall(vcpu);
1452         } else {
1453                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1454                 ret = RESUME_HOST;
1455         }
1456         return ret;
1457 }
1458
1459 /**
1460  * kvm_trap_vz_handle_cop_unusuable() - Guest used unusable coprocessor.
1461  * @vcpu:       Virtual CPU context.
1462  *
1463  * Handle when the guest attempts to use a coprocessor which hasn't been allowed
1464  * by the root context.
1465  */
1466 static int kvm_trap_vz_handle_cop_unusable(struct kvm_vcpu *vcpu)
1467 {
1468         struct kvm_run *run = vcpu->run;
1469         u32 cause = vcpu->arch.host_cp0_cause;
1470         enum emulation_result er = EMULATE_FAIL;
1471         int ret = RESUME_GUEST;
1472
1473         if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 1) {
1474                 /*
1475                  * If guest FPU not present, the FPU operation should have been
1476                  * treated as a reserved instruction!
1477                  * If FPU already in use, we shouldn't get this at all.
1478                  */
1479                 if (WARN_ON(!kvm_mips_guest_has_fpu(&vcpu->arch) ||
1480                             vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)) {
1481                         preempt_enable();
1482                         return EMULATE_FAIL;
1483                 }
1484
1485                 kvm_own_fpu(vcpu);
1486                 er = EMULATE_DONE;
1487         }
1488         /* other coprocessors not handled */
1489
1490         switch (er) {
1491         case EMULATE_DONE:
1492                 ret = RESUME_GUEST;
1493                 break;
1494
1495         case EMULATE_FAIL:
1496                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1497                 ret = RESUME_HOST;
1498                 break;
1499
1500         default:
1501                 BUG();
1502         }
1503         return ret;
1504 }
1505
1506 /**
1507  * kvm_trap_vz_handle_msa_disabled() - Guest used MSA while disabled in root.
1508  * @vcpu:       Virtual CPU context.
1509  *
1510  * Handle when the guest attempts to use MSA when it is disabled in the root
1511  * context.
1512  */
1513 static int kvm_trap_vz_handle_msa_disabled(struct kvm_vcpu *vcpu)
1514 {
1515         struct kvm_run *run = vcpu->run;
1516
1517         /*
1518          * If MSA not present or not exposed to guest or FR=0, the MSA operation
1519          * should have been treated as a reserved instruction!
1520          * Same if CU1=1, FR=0.
1521          * If MSA already in use, we shouldn't get this at all.
1522          */
1523         if (!kvm_mips_guest_has_msa(&vcpu->arch) ||
1524             (read_gc0_status() & (ST0_CU1 | ST0_FR)) == ST0_CU1 ||
1525             !(read_gc0_config5() & MIPS_CONF5_MSAEN) ||
1526             vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) {
1527                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1528                 return RESUME_HOST;
1529         }
1530
1531         kvm_own_msa(vcpu);
1532
1533         return RESUME_GUEST;
1534 }
1535
1536 static int kvm_trap_vz_handle_tlb_ld_miss(struct kvm_vcpu *vcpu)
1537 {
1538         struct kvm_run *run = vcpu->run;
1539         u32 *opc = (u32 *) vcpu->arch.pc;
1540         u32 cause = vcpu->arch.host_cp0_cause;
1541         ulong badvaddr = vcpu->arch.host_cp0_badvaddr;
1542         union mips_instruction inst;
1543         enum emulation_result er = EMULATE_DONE;
1544         int err, ret = RESUME_GUEST;
1545
1546         if (kvm_mips_handle_vz_root_tlb_fault(badvaddr, vcpu, false)) {
1547                 /* A code fetch fault doesn't count as an MMIO */
1548                 if (kvm_is_ifetch_fault(&vcpu->arch)) {
1549                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1550                         return RESUME_HOST;
1551                 }
1552
1553                 /* Fetch the instruction */
1554                 if (cause & CAUSEF_BD)
1555                         opc += 1;
1556                 err = kvm_get_badinstr(opc, vcpu, &inst.word);
1557                 if (err) {
1558                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1559                         return RESUME_HOST;
1560                 }
1561
1562                 /* Treat as MMIO */
1563                 er = kvm_mips_emulate_load(inst, cause, run, vcpu);
1564                 if (er == EMULATE_FAIL) {
1565                         kvm_err("Guest Emulate Load from MMIO space failed: PC: %p, BadVaddr: %#lx\n",
1566                                 opc, badvaddr);
1567                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1568                 }
1569         }
1570
1571         if (er == EMULATE_DONE) {
1572                 ret = RESUME_GUEST;
1573         } else if (er == EMULATE_DO_MMIO) {
1574                 run->exit_reason = KVM_EXIT_MMIO;
1575                 ret = RESUME_HOST;
1576         } else {
1577                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1578                 ret = RESUME_HOST;
1579         }
1580         return ret;
1581 }
1582
1583 static int kvm_trap_vz_handle_tlb_st_miss(struct kvm_vcpu *vcpu)
1584 {
1585         struct kvm_run *run = vcpu->run;
1586         u32 *opc = (u32 *) vcpu->arch.pc;
1587         u32 cause = vcpu->arch.host_cp0_cause;
1588         ulong badvaddr = vcpu->arch.host_cp0_badvaddr;
1589         union mips_instruction inst;
1590         enum emulation_result er = EMULATE_DONE;
1591         int err;
1592         int ret = RESUME_GUEST;
1593
1594         /* Just try the access again if we couldn't do the translation */
1595         if (kvm_vz_badvaddr_to_gpa(vcpu, badvaddr, &badvaddr))
1596                 return RESUME_GUEST;
1597         vcpu->arch.host_cp0_badvaddr = badvaddr;
1598
1599         if (kvm_mips_handle_vz_root_tlb_fault(badvaddr, vcpu, true)) {
1600                 /* Fetch the instruction */
1601                 if (cause & CAUSEF_BD)
1602                         opc += 1;
1603                 err = kvm_get_badinstr(opc, vcpu, &inst.word);
1604                 if (err) {
1605                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1606                         return RESUME_HOST;
1607                 }
1608
1609                 /* Treat as MMIO */
1610                 er = kvm_mips_emulate_store(inst, cause, run, vcpu);
1611                 if (er == EMULATE_FAIL) {
1612                         kvm_err("Guest Emulate Store to MMIO space failed: PC: %p, BadVaddr: %#lx\n",
1613                                 opc, badvaddr);
1614                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1615                 }
1616         }
1617
1618         if (er == EMULATE_DONE) {
1619                 ret = RESUME_GUEST;
1620         } else if (er == EMULATE_DO_MMIO) {
1621                 run->exit_reason = KVM_EXIT_MMIO;
1622                 ret = RESUME_HOST;
1623         } else {
1624                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1625                 ret = RESUME_HOST;
1626         }
1627         return ret;
1628 }
1629
1630 static u64 kvm_vz_get_one_regs[] = {
1631         KVM_REG_MIPS_CP0_INDEX,
1632         KVM_REG_MIPS_CP0_ENTRYLO0,
1633         KVM_REG_MIPS_CP0_ENTRYLO1,
1634         KVM_REG_MIPS_CP0_CONTEXT,
1635         KVM_REG_MIPS_CP0_PAGEMASK,
1636         KVM_REG_MIPS_CP0_PAGEGRAIN,
1637         KVM_REG_MIPS_CP0_WIRED,
1638         KVM_REG_MIPS_CP0_HWRENA,
1639         KVM_REG_MIPS_CP0_BADVADDR,
1640         KVM_REG_MIPS_CP0_COUNT,
1641         KVM_REG_MIPS_CP0_ENTRYHI,
1642         KVM_REG_MIPS_CP0_COMPARE,
1643         KVM_REG_MIPS_CP0_STATUS,
1644         KVM_REG_MIPS_CP0_INTCTL,
1645         KVM_REG_MIPS_CP0_CAUSE,
1646         KVM_REG_MIPS_CP0_EPC,
1647         KVM_REG_MIPS_CP0_PRID,
1648         KVM_REG_MIPS_CP0_EBASE,
1649         KVM_REG_MIPS_CP0_CONFIG,
1650         KVM_REG_MIPS_CP0_CONFIG1,
1651         KVM_REG_MIPS_CP0_CONFIG2,
1652         KVM_REG_MIPS_CP0_CONFIG3,
1653         KVM_REG_MIPS_CP0_CONFIG4,
1654         KVM_REG_MIPS_CP0_CONFIG5,
1655 #ifdef CONFIG_64BIT
1656         KVM_REG_MIPS_CP0_XCONTEXT,
1657 #endif
1658         KVM_REG_MIPS_CP0_ERROREPC,
1659
1660         KVM_REG_MIPS_COUNT_CTL,
1661         KVM_REG_MIPS_COUNT_RESUME,
1662         KVM_REG_MIPS_COUNT_HZ,
1663 };
1664
1665 static u64 kvm_vz_get_one_regs_contextconfig[] = {
1666         KVM_REG_MIPS_CP0_CONTEXTCONFIG,
1667 #ifdef CONFIG_64BIT
1668         KVM_REG_MIPS_CP0_XCONTEXTCONFIG,
1669 #endif
1670 };
1671
1672 static u64 kvm_vz_get_one_regs_segments[] = {
1673         KVM_REG_MIPS_CP0_SEGCTL0,
1674         KVM_REG_MIPS_CP0_SEGCTL1,
1675         KVM_REG_MIPS_CP0_SEGCTL2,
1676 };
1677
1678 static u64 kvm_vz_get_one_regs_htw[] = {
1679         KVM_REG_MIPS_CP0_PWBASE,
1680         KVM_REG_MIPS_CP0_PWFIELD,
1681         KVM_REG_MIPS_CP0_PWSIZE,
1682         KVM_REG_MIPS_CP0_PWCTL,
1683 };
1684
1685 static u64 kvm_vz_get_one_regs_kscratch[] = {
1686         KVM_REG_MIPS_CP0_KSCRATCH1,
1687         KVM_REG_MIPS_CP0_KSCRATCH2,
1688         KVM_REG_MIPS_CP0_KSCRATCH3,
1689         KVM_REG_MIPS_CP0_KSCRATCH4,
1690         KVM_REG_MIPS_CP0_KSCRATCH5,
1691         KVM_REG_MIPS_CP0_KSCRATCH6,
1692 };
1693
1694 static unsigned long kvm_vz_num_regs(struct kvm_vcpu *vcpu)
1695 {
1696         unsigned long ret;
1697
1698         ret = ARRAY_SIZE(kvm_vz_get_one_regs);
1699         if (cpu_guest_has_userlocal)
1700                 ++ret;
1701         if (cpu_guest_has_badinstr)
1702                 ++ret;
1703         if (cpu_guest_has_badinstrp)
1704                 ++ret;
1705         if (cpu_guest_has_contextconfig)
1706                 ret += ARRAY_SIZE(kvm_vz_get_one_regs_contextconfig);
1707         if (cpu_guest_has_segments)
1708                 ret += ARRAY_SIZE(kvm_vz_get_one_regs_segments);
1709         if (cpu_guest_has_htw)
1710                 ret += ARRAY_SIZE(kvm_vz_get_one_regs_htw);
1711         if (cpu_guest_has_maar && !cpu_guest_has_dyn_maar)
1712                 ret += 1 + ARRAY_SIZE(vcpu->arch.maar);
1713         ret += __arch_hweight8(cpu_data[0].guest.kscratch_mask);
1714
1715         return ret;
1716 }
1717
1718 static int kvm_vz_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices)
1719 {
1720         u64 index;
1721         unsigned int i;
1722
1723         if (copy_to_user(indices, kvm_vz_get_one_regs,
1724                          sizeof(kvm_vz_get_one_regs)))
1725                 return -EFAULT;
1726         indices += ARRAY_SIZE(kvm_vz_get_one_regs);
1727
1728         if (cpu_guest_has_userlocal) {
1729                 index = KVM_REG_MIPS_CP0_USERLOCAL;
1730                 if (copy_to_user(indices, &index, sizeof(index)))
1731                         return -EFAULT;
1732                 ++indices;
1733         }
1734         if (cpu_guest_has_badinstr) {
1735                 index = KVM_REG_MIPS_CP0_BADINSTR;
1736                 if (copy_to_user(indices, &index, sizeof(index)))
1737                         return -EFAULT;
1738                 ++indices;
1739         }
1740         if (cpu_guest_has_badinstrp) {
1741                 index = KVM_REG_MIPS_CP0_BADINSTRP;
1742                 if (copy_to_user(indices, &index, sizeof(index)))
1743                         return -EFAULT;
1744                 ++indices;
1745         }
1746         if (cpu_guest_has_contextconfig) {
1747                 if (copy_to_user(indices, kvm_vz_get_one_regs_contextconfig,
1748                                  sizeof(kvm_vz_get_one_regs_contextconfig)))
1749                         return -EFAULT;
1750                 indices += ARRAY_SIZE(kvm_vz_get_one_regs_contextconfig);
1751         }
1752         if (cpu_guest_has_segments) {
1753                 if (copy_to_user(indices, kvm_vz_get_one_regs_segments,
1754                                  sizeof(kvm_vz_get_one_regs_segments)))
1755                         return -EFAULT;
1756                 indices += ARRAY_SIZE(kvm_vz_get_one_regs_segments);
1757         }
1758         if (cpu_guest_has_htw) {
1759                 if (copy_to_user(indices, kvm_vz_get_one_regs_htw,
1760                                  sizeof(kvm_vz_get_one_regs_htw)))
1761                         return -EFAULT;
1762                 indices += ARRAY_SIZE(kvm_vz_get_one_regs_htw);
1763         }
1764         if (cpu_guest_has_maar && !cpu_guest_has_dyn_maar) {
1765                 for (i = 0; i < ARRAY_SIZE(vcpu->arch.maar); ++i) {
1766                         index = KVM_REG_MIPS_CP0_MAAR(i);
1767                         if (copy_to_user(indices, &index, sizeof(index)))
1768                                 return -EFAULT;
1769                         ++indices;
1770                 }
1771
1772                 index = KVM_REG_MIPS_CP0_MAARI;
1773                 if (copy_to_user(indices, &index, sizeof(index)))
1774                         return -EFAULT;
1775                 ++indices;
1776         }
1777         for (i = 0; i < 6; ++i) {
1778                 if (!cpu_guest_has_kscr(i + 2))
1779                         continue;
1780
1781                 if (copy_to_user(indices, &kvm_vz_get_one_regs_kscratch[i],
1782                                  sizeof(kvm_vz_get_one_regs_kscratch[i])))
1783                         return -EFAULT;
1784                 ++indices;
1785         }
1786
1787         return 0;
1788 }
1789
1790 static inline s64 entrylo_kvm_to_user(unsigned long v)
1791 {
1792         s64 mask, ret = v;
1793
1794         if (BITS_PER_LONG == 32) {
1795                 /*
1796                  * KVM API exposes 64-bit version of the register, so move the
1797                  * RI/XI bits up into place.
1798                  */
1799                 mask = MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI;
1800                 ret &= ~mask;
1801                 ret |= ((s64)v & mask) << 32;
1802         }
1803         return ret;
1804 }
1805
1806 static inline unsigned long entrylo_user_to_kvm(s64 v)
1807 {
1808         unsigned long mask, ret = v;
1809
1810         if (BITS_PER_LONG == 32) {
1811                 /*
1812                  * KVM API exposes 64-bit versiono of the register, so move the
1813                  * RI/XI bits down into place.
1814                  */
1815                 mask = MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI;
1816                 ret &= ~mask;
1817                 ret |= (v >> 32) & mask;
1818         }
1819         return ret;
1820 }
1821
1822 static int kvm_vz_get_one_reg(struct kvm_vcpu *vcpu,
1823                               const struct kvm_one_reg *reg,
1824                               s64 *v)
1825 {
1826         struct mips_coproc *cop0 = vcpu->arch.cop0;
1827         unsigned int idx;
1828
1829         switch (reg->id) {
1830         case KVM_REG_MIPS_CP0_INDEX:
1831                 *v = (long)read_gc0_index();
1832                 break;
1833         case KVM_REG_MIPS_CP0_ENTRYLO0:
1834                 *v = entrylo_kvm_to_user(read_gc0_entrylo0());
1835                 break;
1836         case KVM_REG_MIPS_CP0_ENTRYLO1:
1837                 *v = entrylo_kvm_to_user(read_gc0_entrylo1());
1838                 break;
1839         case KVM_REG_MIPS_CP0_CONTEXT:
1840                 *v = (long)read_gc0_context();
1841                 break;
1842         case KVM_REG_MIPS_CP0_CONTEXTCONFIG:
1843                 if (!cpu_guest_has_contextconfig)
1844                         return -EINVAL;
1845                 *v = read_gc0_contextconfig();
1846                 break;
1847         case KVM_REG_MIPS_CP0_USERLOCAL:
1848                 if (!cpu_guest_has_userlocal)
1849                         return -EINVAL;
1850                 *v = read_gc0_userlocal();
1851                 break;
1852 #ifdef CONFIG_64BIT
1853         case KVM_REG_MIPS_CP0_XCONTEXTCONFIG:
1854                 if (!cpu_guest_has_contextconfig)
1855                         return -EINVAL;
1856                 *v = read_gc0_xcontextconfig();
1857                 break;
1858 #endif
1859         case KVM_REG_MIPS_CP0_PAGEMASK:
1860                 *v = (long)read_gc0_pagemask();
1861                 break;
1862         case KVM_REG_MIPS_CP0_PAGEGRAIN:
1863                 *v = (long)read_gc0_pagegrain();
1864                 break;
1865         case KVM_REG_MIPS_CP0_SEGCTL0:
1866                 if (!cpu_guest_has_segments)
1867                         return -EINVAL;
1868                 *v = read_gc0_segctl0();
1869                 break;
1870         case KVM_REG_MIPS_CP0_SEGCTL1:
1871                 if (!cpu_guest_has_segments)
1872                         return -EINVAL;
1873                 *v = read_gc0_segctl1();
1874                 break;
1875         case KVM_REG_MIPS_CP0_SEGCTL2:
1876                 if (!cpu_guest_has_segments)
1877                         return -EINVAL;
1878                 *v = read_gc0_segctl2();
1879                 break;
1880         case KVM_REG_MIPS_CP0_PWBASE:
1881                 if (!cpu_guest_has_htw)
1882                         return -EINVAL;
1883                 *v = read_gc0_pwbase();
1884                 break;
1885         case KVM_REG_MIPS_CP0_PWFIELD:
1886                 if (!cpu_guest_has_htw)
1887                         return -EINVAL;
1888                 *v = read_gc0_pwfield();
1889                 break;
1890         case KVM_REG_MIPS_CP0_PWSIZE:
1891                 if (!cpu_guest_has_htw)
1892                         return -EINVAL;
1893                 *v = read_gc0_pwsize();
1894                 break;
1895         case KVM_REG_MIPS_CP0_WIRED:
1896                 *v = (long)read_gc0_wired();
1897                 break;
1898         case KVM_REG_MIPS_CP0_PWCTL:
1899                 if (!cpu_guest_has_htw)
1900                         return -EINVAL;
1901                 *v = read_gc0_pwctl();
1902                 break;
1903         case KVM_REG_MIPS_CP0_HWRENA:
1904                 *v = (long)read_gc0_hwrena();
1905                 break;
1906         case KVM_REG_MIPS_CP0_BADVADDR:
1907                 *v = (long)read_gc0_badvaddr();
1908                 break;
1909         case KVM_REG_MIPS_CP0_BADINSTR:
1910                 if (!cpu_guest_has_badinstr)
1911                         return -EINVAL;
1912                 *v = read_gc0_badinstr();
1913                 break;
1914         case KVM_REG_MIPS_CP0_BADINSTRP:
1915                 if (!cpu_guest_has_badinstrp)
1916                         return -EINVAL;
1917                 *v = read_gc0_badinstrp();
1918                 break;
1919         case KVM_REG_MIPS_CP0_COUNT:
1920                 *v = kvm_mips_read_count(vcpu);
1921                 break;
1922         case KVM_REG_MIPS_CP0_ENTRYHI:
1923                 *v = (long)read_gc0_entryhi();
1924                 break;
1925         case KVM_REG_MIPS_CP0_COMPARE:
1926                 *v = (long)read_gc0_compare();
1927                 break;
1928         case KVM_REG_MIPS_CP0_STATUS:
1929                 *v = (long)read_gc0_status();
1930                 break;
1931         case KVM_REG_MIPS_CP0_INTCTL:
1932                 *v = read_gc0_intctl();
1933                 break;
1934         case KVM_REG_MIPS_CP0_CAUSE:
1935                 *v = (long)read_gc0_cause();
1936                 break;
1937         case KVM_REG_MIPS_CP0_EPC:
1938                 *v = (long)read_gc0_epc();
1939                 break;
1940         case KVM_REG_MIPS_CP0_PRID:
1941                 switch (boot_cpu_type()) {
1942                 case CPU_CAVIUM_OCTEON3:
1943                         /* Octeon III has a read-only guest.PRid */
1944                         *v = read_gc0_prid();
1945                         break;
1946                 default:
1947                         *v = (long)kvm_read_c0_guest_prid(cop0);
1948                         break;
1949                 };
1950                 break;
1951         case KVM_REG_MIPS_CP0_EBASE:
1952                 *v = kvm_vz_read_gc0_ebase();
1953                 break;
1954         case KVM_REG_MIPS_CP0_CONFIG:
1955                 *v = read_gc0_config();
1956                 break;
1957         case KVM_REG_MIPS_CP0_CONFIG1:
1958                 if (!cpu_guest_has_conf1)
1959                         return -EINVAL;
1960                 *v = read_gc0_config1();
1961                 break;
1962         case KVM_REG_MIPS_CP0_CONFIG2:
1963                 if (!cpu_guest_has_conf2)
1964                         return -EINVAL;
1965                 *v = read_gc0_config2();
1966                 break;
1967         case KVM_REG_MIPS_CP0_CONFIG3:
1968                 if (!cpu_guest_has_conf3)
1969                         return -EINVAL;
1970                 *v = read_gc0_config3();
1971                 break;
1972         case KVM_REG_MIPS_CP0_CONFIG4:
1973                 if (!cpu_guest_has_conf4)
1974                         return -EINVAL;
1975                 *v = read_gc0_config4();
1976                 break;
1977         case KVM_REG_MIPS_CP0_CONFIG5:
1978                 if (!cpu_guest_has_conf5)
1979                         return -EINVAL;
1980                 *v = read_gc0_config5();
1981                 break;
1982         case KVM_REG_MIPS_CP0_MAAR(0) ... KVM_REG_MIPS_CP0_MAAR(0x3f):
1983                 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar)
1984                         return -EINVAL;
1985                 idx = reg->id - KVM_REG_MIPS_CP0_MAAR(0);
1986                 if (idx >= ARRAY_SIZE(vcpu->arch.maar))
1987                         return -EINVAL;
1988                 *v = vcpu->arch.maar[idx];
1989                 break;
1990         case KVM_REG_MIPS_CP0_MAARI:
1991                 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar)
1992                         return -EINVAL;
1993                 *v = kvm_read_sw_gc0_maari(vcpu->arch.cop0);
1994                 break;
1995 #ifdef CONFIG_64BIT
1996         case KVM_REG_MIPS_CP0_XCONTEXT:
1997                 *v = read_gc0_xcontext();
1998                 break;
1999 #endif
2000         case KVM_REG_MIPS_CP0_ERROREPC:
2001                 *v = (long)read_gc0_errorepc();
2002                 break;
2003         case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6:
2004                 idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2;
2005                 if (!cpu_guest_has_kscr(idx))
2006                         return -EINVAL;
2007                 switch (idx) {
2008                 case 2:
2009                         *v = (long)read_gc0_kscratch1();
2010                         break;
2011                 case 3:
2012                         *v = (long)read_gc0_kscratch2();
2013                         break;
2014                 case 4:
2015                         *v = (long)read_gc0_kscratch3();
2016                         break;
2017                 case 5:
2018                         *v = (long)read_gc0_kscratch4();
2019                         break;
2020                 case 6:
2021                         *v = (long)read_gc0_kscratch5();
2022                         break;
2023                 case 7:
2024                         *v = (long)read_gc0_kscratch6();
2025                         break;
2026                 }
2027                 break;
2028         case KVM_REG_MIPS_COUNT_CTL:
2029                 *v = vcpu->arch.count_ctl;
2030                 break;
2031         case KVM_REG_MIPS_COUNT_RESUME:
2032                 *v = ktime_to_ns(vcpu->arch.count_resume);
2033                 break;
2034         case KVM_REG_MIPS_COUNT_HZ:
2035                 *v = vcpu->arch.count_hz;
2036                 break;
2037         default:
2038                 return -EINVAL;
2039         }
2040         return 0;
2041 }
2042
2043 static int kvm_vz_set_one_reg(struct kvm_vcpu *vcpu,
2044                               const struct kvm_one_reg *reg,
2045                               s64 v)
2046 {
2047         struct mips_coproc *cop0 = vcpu->arch.cop0;
2048         unsigned int idx;
2049         int ret = 0;
2050         unsigned int cur, change;
2051
2052         switch (reg->id) {
2053         case KVM_REG_MIPS_CP0_INDEX:
2054                 write_gc0_index(v);
2055                 break;
2056         case KVM_REG_MIPS_CP0_ENTRYLO0:
2057                 write_gc0_entrylo0(entrylo_user_to_kvm(v));
2058                 break;
2059         case KVM_REG_MIPS_CP0_ENTRYLO1:
2060                 write_gc0_entrylo1(entrylo_user_to_kvm(v));
2061                 break;
2062         case KVM_REG_MIPS_CP0_CONTEXT:
2063                 write_gc0_context(v);
2064                 break;
2065         case KVM_REG_MIPS_CP0_CONTEXTCONFIG:
2066                 if (!cpu_guest_has_contextconfig)
2067                         return -EINVAL;
2068                 write_gc0_contextconfig(v);
2069                 break;
2070         case KVM_REG_MIPS_CP0_USERLOCAL:
2071                 if (!cpu_guest_has_userlocal)
2072                         return -EINVAL;
2073                 write_gc0_userlocal(v);
2074                 break;
2075 #ifdef CONFIG_64BIT
2076         case KVM_REG_MIPS_CP0_XCONTEXTCONFIG:
2077                 if (!cpu_guest_has_contextconfig)
2078                         return -EINVAL;
2079                 write_gc0_xcontextconfig(v);
2080                 break;
2081 #endif
2082         case KVM_REG_MIPS_CP0_PAGEMASK:
2083                 write_gc0_pagemask(v);
2084                 break;
2085         case KVM_REG_MIPS_CP0_PAGEGRAIN:
2086                 write_gc0_pagegrain(v);
2087                 break;
2088         case KVM_REG_MIPS_CP0_SEGCTL0:
2089                 if (!cpu_guest_has_segments)
2090                         return -EINVAL;
2091                 write_gc0_segctl0(v);
2092                 break;
2093         case KVM_REG_MIPS_CP0_SEGCTL1:
2094                 if (!cpu_guest_has_segments)
2095                         return -EINVAL;
2096                 write_gc0_segctl1(v);
2097                 break;
2098         case KVM_REG_MIPS_CP0_SEGCTL2:
2099                 if (!cpu_guest_has_segments)
2100                         return -EINVAL;
2101                 write_gc0_segctl2(v);
2102                 break;
2103         case KVM_REG_MIPS_CP0_PWBASE:
2104                 if (!cpu_guest_has_htw)
2105                         return -EINVAL;
2106                 write_gc0_pwbase(v);
2107                 break;
2108         case KVM_REG_MIPS_CP0_PWFIELD:
2109                 if (!cpu_guest_has_htw)
2110                         return -EINVAL;
2111                 write_gc0_pwfield(v);
2112                 break;
2113         case KVM_REG_MIPS_CP0_PWSIZE:
2114                 if (!cpu_guest_has_htw)
2115                         return -EINVAL;
2116                 write_gc0_pwsize(v);
2117                 break;
2118         case KVM_REG_MIPS_CP0_WIRED:
2119                 change_gc0_wired(MIPSR6_WIRED_WIRED, v);
2120                 break;
2121         case KVM_REG_MIPS_CP0_PWCTL:
2122                 if (!cpu_guest_has_htw)
2123                         return -EINVAL;
2124                 write_gc0_pwctl(v);
2125                 break;
2126         case KVM_REG_MIPS_CP0_HWRENA:
2127                 write_gc0_hwrena(v);
2128                 break;
2129         case KVM_REG_MIPS_CP0_BADVADDR:
2130                 write_gc0_badvaddr(v);
2131                 break;
2132         case KVM_REG_MIPS_CP0_BADINSTR:
2133                 if (!cpu_guest_has_badinstr)
2134                         return -EINVAL;
2135                 write_gc0_badinstr(v);
2136                 break;
2137         case KVM_REG_MIPS_CP0_BADINSTRP:
2138                 if (!cpu_guest_has_badinstrp)
2139                         return -EINVAL;
2140                 write_gc0_badinstrp(v);
2141                 break;
2142         case KVM_REG_MIPS_CP0_COUNT:
2143                 kvm_mips_write_count(vcpu, v);
2144                 break;
2145         case KVM_REG_MIPS_CP0_ENTRYHI:
2146                 write_gc0_entryhi(v);
2147                 break;
2148         case KVM_REG_MIPS_CP0_COMPARE:
2149                 kvm_mips_write_compare(vcpu, v, false);
2150                 break;
2151         case KVM_REG_MIPS_CP0_STATUS:
2152                 write_gc0_status(v);
2153                 break;
2154         case KVM_REG_MIPS_CP0_INTCTL:
2155                 write_gc0_intctl(v);
2156                 break;
2157         case KVM_REG_MIPS_CP0_CAUSE:
2158                 /*
2159                  * If the timer is stopped or started (DC bit) it must look
2160                  * atomic with changes to the timer interrupt pending bit (TI).
2161                  * A timer interrupt should not happen in between.
2162                  */
2163                 if ((read_gc0_cause() ^ v) & CAUSEF_DC) {
2164                         if (v & CAUSEF_DC) {
2165                                 /* disable timer first */
2166                                 kvm_mips_count_disable_cause(vcpu);
2167                                 change_gc0_cause((u32)~CAUSEF_DC, v);
2168                         } else {
2169                                 /* enable timer last */
2170                                 change_gc0_cause((u32)~CAUSEF_DC, v);
2171                                 kvm_mips_count_enable_cause(vcpu);
2172                         }
2173                 } else {
2174                         write_gc0_cause(v);
2175                 }
2176                 break;
2177         case KVM_REG_MIPS_CP0_EPC:
2178                 write_gc0_epc(v);
2179                 break;
2180         case KVM_REG_MIPS_CP0_PRID:
2181                 switch (boot_cpu_type()) {
2182                 case CPU_CAVIUM_OCTEON3:
2183                         /* Octeon III has a guest.PRid, but its read-only */
2184                         break;
2185                 default:
2186                         kvm_write_c0_guest_prid(cop0, v);
2187                         break;
2188                 };
2189                 break;
2190         case KVM_REG_MIPS_CP0_EBASE:
2191                 kvm_vz_write_gc0_ebase(v);
2192                 break;
2193         case KVM_REG_MIPS_CP0_CONFIG:
2194                 cur = read_gc0_config();
2195                 change = (cur ^ v) & kvm_vz_config_user_wrmask(vcpu);
2196                 if (change) {
2197                         v = cur ^ change;
2198                         write_gc0_config(v);
2199                 }
2200                 break;
2201         case KVM_REG_MIPS_CP0_CONFIG1:
2202                 if (!cpu_guest_has_conf1)
2203                         break;
2204                 cur = read_gc0_config1();
2205                 change = (cur ^ v) & kvm_vz_config1_user_wrmask(vcpu);
2206                 if (change) {
2207                         v = cur ^ change;
2208                         write_gc0_config1(v);
2209                 }
2210                 break;
2211         case KVM_REG_MIPS_CP0_CONFIG2:
2212                 if (!cpu_guest_has_conf2)
2213                         break;
2214                 cur = read_gc0_config2();
2215                 change = (cur ^ v) & kvm_vz_config2_user_wrmask(vcpu);
2216                 if (change) {
2217                         v = cur ^ change;
2218                         write_gc0_config2(v);
2219                 }
2220                 break;
2221         case KVM_REG_MIPS_CP0_CONFIG3:
2222                 if (!cpu_guest_has_conf3)
2223                         break;
2224                 cur = read_gc0_config3();
2225                 change = (cur ^ v) & kvm_vz_config3_user_wrmask(vcpu);
2226                 if (change) {
2227                         v = cur ^ change;
2228                         write_gc0_config3(v);
2229                 }
2230                 break;
2231         case KVM_REG_MIPS_CP0_CONFIG4:
2232                 if (!cpu_guest_has_conf4)
2233                         break;
2234                 cur = read_gc0_config4();
2235                 change = (cur ^ v) & kvm_vz_config4_user_wrmask(vcpu);
2236                 if (change) {
2237                         v = cur ^ change;
2238                         write_gc0_config4(v);
2239                 }
2240                 break;
2241         case KVM_REG_MIPS_CP0_CONFIG5:
2242                 if (!cpu_guest_has_conf5)
2243                         break;
2244                 cur = read_gc0_config5();
2245                 change = (cur ^ v) & kvm_vz_config5_user_wrmask(vcpu);
2246                 if (change) {
2247                         v = cur ^ change;
2248                         write_gc0_config5(v);
2249                 }
2250                 break;
2251         case KVM_REG_MIPS_CP0_MAAR(0) ... KVM_REG_MIPS_CP0_MAAR(0x3f):
2252                 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar)
2253                         return -EINVAL;
2254                 idx = reg->id - KVM_REG_MIPS_CP0_MAAR(0);
2255                 if (idx >= ARRAY_SIZE(vcpu->arch.maar))
2256                         return -EINVAL;
2257                 vcpu->arch.maar[idx] = mips_process_maar(dmtc_op, v);
2258                 break;
2259         case KVM_REG_MIPS_CP0_MAARI:
2260                 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar)
2261                         return -EINVAL;
2262                 kvm_write_maari(vcpu, v);
2263                 break;
2264 #ifdef CONFIG_64BIT
2265         case KVM_REG_MIPS_CP0_XCONTEXT:
2266                 write_gc0_xcontext(v);
2267                 break;
2268 #endif
2269         case KVM_REG_MIPS_CP0_ERROREPC:
2270                 write_gc0_errorepc(v);
2271                 break;
2272         case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6:
2273                 idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2;
2274                 if (!cpu_guest_has_kscr(idx))
2275                         return -EINVAL;
2276                 switch (idx) {
2277                 case 2:
2278                         write_gc0_kscratch1(v);
2279                         break;
2280                 case 3:
2281                         write_gc0_kscratch2(v);
2282                         break;
2283                 case 4:
2284                         write_gc0_kscratch3(v);
2285                         break;
2286                 case 5:
2287                         write_gc0_kscratch4(v);
2288                         break;
2289                 case 6:
2290                         write_gc0_kscratch5(v);
2291                         break;
2292                 case 7:
2293                         write_gc0_kscratch6(v);
2294                         break;
2295                 }
2296                 break;
2297         case KVM_REG_MIPS_COUNT_CTL:
2298                 ret = kvm_mips_set_count_ctl(vcpu, v);
2299                 break;
2300         case KVM_REG_MIPS_COUNT_RESUME:
2301                 ret = kvm_mips_set_count_resume(vcpu, v);
2302                 break;
2303         case KVM_REG_MIPS_COUNT_HZ:
2304                 ret = kvm_mips_set_count_hz(vcpu, v);
2305                 break;
2306         default:
2307                 return -EINVAL;
2308         }
2309         return ret;
2310 }
2311
2312 #define guestid_cache(cpu)      (cpu_data[cpu].guestid_cache)
2313 static void kvm_vz_get_new_guestid(unsigned long cpu, struct kvm_vcpu *vcpu)
2314 {
2315         unsigned long guestid = guestid_cache(cpu);
2316
2317         if (!(++guestid & GUESTID_MASK)) {
2318                 if (cpu_has_vtag_icache)
2319                         flush_icache_all();
2320
2321                 if (!guestid)           /* fix version if needed */
2322                         guestid = GUESTID_FIRST_VERSION;
2323
2324                 ++guestid;              /* guestid 0 reserved for root */
2325
2326                 /* start new guestid cycle */
2327                 kvm_vz_local_flush_roottlb_all_guests();
2328                 kvm_vz_local_flush_guesttlb_all();
2329         }
2330
2331         guestid_cache(cpu) = guestid;
2332 }
2333
2334 /* Returns 1 if the guest TLB may be clobbered */
2335 static int kvm_vz_check_requests(struct kvm_vcpu *vcpu, int cpu)
2336 {
2337         int ret = 0;
2338         int i;
2339
2340         if (!kvm_request_pending(vcpu))
2341                 return 0;
2342
2343         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2344                 if (cpu_has_guestid) {
2345                         /* Drop all GuestIDs for this VCPU */
2346                         for_each_possible_cpu(i)
2347                                 vcpu->arch.vzguestid[i] = 0;
2348                         /* This will clobber guest TLB contents too */
2349                         ret = 1;
2350                 }
2351                 /*
2352                  * For Root ASID Dealias (RAD) we don't do anything here, but we
2353                  * still need the request to ensure we recheck asid_flush_mask.
2354                  * We can still return 0 as only the root TLB will be affected
2355                  * by a root ASID flush.
2356                  */
2357         }
2358
2359         return ret;
2360 }
2361
2362 static void kvm_vz_vcpu_save_wired(struct kvm_vcpu *vcpu)
2363 {
2364         unsigned int wired = read_gc0_wired();
2365         struct kvm_mips_tlb *tlbs;
2366         int i;
2367
2368         /* Expand the wired TLB array if necessary */
2369         wired &= MIPSR6_WIRED_WIRED;
2370         if (wired > vcpu->arch.wired_tlb_limit) {
2371                 tlbs = krealloc(vcpu->arch.wired_tlb, wired *
2372                                 sizeof(*vcpu->arch.wired_tlb), GFP_ATOMIC);
2373                 if (WARN_ON(!tlbs)) {
2374                         /* Save whatever we can */
2375                         wired = vcpu->arch.wired_tlb_limit;
2376                 } else {
2377                         vcpu->arch.wired_tlb = tlbs;
2378                         vcpu->arch.wired_tlb_limit = wired;
2379                 }
2380         }
2381
2382         if (wired)
2383                 /* Save wired entries from the guest TLB */
2384                 kvm_vz_save_guesttlb(vcpu->arch.wired_tlb, 0, wired);
2385         /* Invalidate any dropped entries since last time */
2386         for (i = wired; i < vcpu->arch.wired_tlb_used; ++i) {
2387                 vcpu->arch.wired_tlb[i].tlb_hi = UNIQUE_GUEST_ENTRYHI(i);
2388                 vcpu->arch.wired_tlb[i].tlb_lo[0] = 0;
2389                 vcpu->arch.wired_tlb[i].tlb_lo[1] = 0;
2390                 vcpu->arch.wired_tlb[i].tlb_mask = 0;
2391         }
2392         vcpu->arch.wired_tlb_used = wired;
2393 }
2394
2395 static void kvm_vz_vcpu_load_wired(struct kvm_vcpu *vcpu)
2396 {
2397         /* Load wired entries into the guest TLB */
2398         if (vcpu->arch.wired_tlb)
2399                 kvm_vz_load_guesttlb(vcpu->arch.wired_tlb, 0,
2400                                      vcpu->arch.wired_tlb_used);
2401 }
2402
2403 static void kvm_vz_vcpu_load_tlb(struct kvm_vcpu *vcpu, int cpu)
2404 {
2405         struct kvm *kvm = vcpu->kvm;
2406         struct mm_struct *gpa_mm = &kvm->arch.gpa_mm;
2407         bool migrated;
2408
2409         /*
2410          * Are we entering guest context on a different CPU to last time?
2411          * If so, the VCPU's guest TLB state on this CPU may be stale.
2412          */
2413         migrated = (vcpu->arch.last_exec_cpu != cpu);
2414         vcpu->arch.last_exec_cpu = cpu;
2415
2416         /*
2417          * A vcpu's GuestID is set in GuestCtl1.ID when the vcpu is loaded and
2418          * remains set until another vcpu is loaded in.  As a rule GuestRID
2419          * remains zeroed when in root context unless the kernel is busy
2420          * manipulating guest tlb entries.
2421          */
2422         if (cpu_has_guestid) {
2423                 /*
2424                  * Check if our GuestID is of an older version and thus invalid.
2425                  *
2426                  * We also discard the stored GuestID if we've executed on
2427                  * another CPU, as the guest mappings may have changed without
2428                  * hypervisor knowledge.
2429                  */
2430                 if (migrated ||
2431                     (vcpu->arch.vzguestid[cpu] ^ guestid_cache(cpu)) &
2432                                         GUESTID_VERSION_MASK) {
2433                         kvm_vz_get_new_guestid(cpu, vcpu);
2434                         vcpu->arch.vzguestid[cpu] = guestid_cache(cpu);
2435                         trace_kvm_guestid_change(vcpu,
2436                                                  vcpu->arch.vzguestid[cpu]);
2437                 }
2438
2439                 /* Restore GuestID */
2440                 change_c0_guestctl1(GUESTID_MASK, vcpu->arch.vzguestid[cpu]);
2441         } else {
2442                 /*
2443                  * The Guest TLB only stores a single guest's TLB state, so
2444                  * flush it if another VCPU has executed on this CPU.
2445                  *
2446                  * We also flush if we've executed on another CPU, as the guest
2447                  * mappings may have changed without hypervisor knowledge.
2448                  */
2449                 if (migrated || last_exec_vcpu[cpu] != vcpu)
2450                         kvm_vz_local_flush_guesttlb_all();
2451                 last_exec_vcpu[cpu] = vcpu;
2452
2453                 /*
2454                  * Root ASID dealiases guest GPA mappings in the root TLB.
2455                  * Allocate new root ASID if needed.
2456                  */
2457                 if (cpumask_test_and_clear_cpu(cpu, &kvm->arch.asid_flush_mask))
2458                         get_new_mmu_context(gpa_mm);
2459                 else
2460                         check_mmu_context(gpa_mm);
2461         }
2462 }
2463
2464 static int kvm_vz_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2465 {
2466         struct mips_coproc *cop0 = vcpu->arch.cop0;
2467         bool migrated, all;
2468
2469         /*
2470          * Have we migrated to a different CPU?
2471          * If so, any old guest TLB state may be stale.
2472          */
2473         migrated = (vcpu->arch.last_sched_cpu != cpu);
2474
2475         /*
2476          * Was this the last VCPU to run on this CPU?
2477          * If not, any old guest state from this VCPU will have been clobbered.
2478          */
2479         all = migrated || (last_vcpu[cpu] != vcpu);
2480         last_vcpu[cpu] = vcpu;
2481
2482         /*
2483          * Restore CP0_Wired unconditionally as we clear it after use, and
2484          * restore wired guest TLB entries (while in guest context).
2485          */
2486         kvm_restore_gc0_wired(cop0);
2487         if (current->flags & PF_VCPU) {
2488                 tlbw_use_hazard();
2489                 kvm_vz_vcpu_load_tlb(vcpu, cpu);
2490                 kvm_vz_vcpu_load_wired(vcpu);
2491         }
2492
2493         /*
2494          * Restore timer state regardless, as e.g. Cause.TI can change over time
2495          * if left unmaintained.
2496          */
2497         kvm_vz_restore_timer(vcpu);
2498
2499         /* Set MC bit if we want to trace guest mode changes */
2500         if (kvm_trace_guest_mode_change)
2501                 set_c0_guestctl0(MIPS_GCTL0_MC);
2502         else
2503                 clear_c0_guestctl0(MIPS_GCTL0_MC);
2504
2505         /* Don't bother restoring registers multiple times unless necessary */
2506         if (!all)
2507                 return 0;
2508
2509         /*
2510          * Restore config registers first, as some implementations restrict
2511          * writes to other registers when the corresponding feature bits aren't
2512          * set. For example Status.CU1 cannot be set unless Config1.FP is set.
2513          */
2514         kvm_restore_gc0_config(cop0);
2515         if (cpu_guest_has_conf1)
2516                 kvm_restore_gc0_config1(cop0);
2517         if (cpu_guest_has_conf2)
2518                 kvm_restore_gc0_config2(cop0);
2519         if (cpu_guest_has_conf3)
2520                 kvm_restore_gc0_config3(cop0);
2521         if (cpu_guest_has_conf4)
2522                 kvm_restore_gc0_config4(cop0);
2523         if (cpu_guest_has_conf5)
2524                 kvm_restore_gc0_config5(cop0);
2525         if (cpu_guest_has_conf6)
2526                 kvm_restore_gc0_config6(cop0);
2527         if (cpu_guest_has_conf7)
2528                 kvm_restore_gc0_config7(cop0);
2529
2530         kvm_restore_gc0_index(cop0);
2531         kvm_restore_gc0_entrylo0(cop0);
2532         kvm_restore_gc0_entrylo1(cop0);
2533         kvm_restore_gc0_context(cop0);
2534         if (cpu_guest_has_contextconfig)
2535                 kvm_restore_gc0_contextconfig(cop0);
2536 #ifdef CONFIG_64BIT
2537         kvm_restore_gc0_xcontext(cop0);
2538         if (cpu_guest_has_contextconfig)
2539                 kvm_restore_gc0_xcontextconfig(cop0);
2540 #endif
2541         kvm_restore_gc0_pagemask(cop0);
2542         kvm_restore_gc0_pagegrain(cop0);
2543         kvm_restore_gc0_hwrena(cop0);
2544         kvm_restore_gc0_badvaddr(cop0);
2545         kvm_restore_gc0_entryhi(cop0);
2546         kvm_restore_gc0_status(cop0);
2547         kvm_restore_gc0_intctl(cop0);
2548         kvm_restore_gc0_epc(cop0);
2549         kvm_vz_write_gc0_ebase(kvm_read_sw_gc0_ebase(cop0));
2550         if (cpu_guest_has_userlocal)
2551                 kvm_restore_gc0_userlocal(cop0);
2552
2553         kvm_restore_gc0_errorepc(cop0);
2554
2555         /* restore KScratch registers if enabled in guest */
2556         if (cpu_guest_has_conf4) {
2557                 if (cpu_guest_has_kscr(2))
2558                         kvm_restore_gc0_kscratch1(cop0);
2559                 if (cpu_guest_has_kscr(3))
2560                         kvm_restore_gc0_kscratch2(cop0);
2561                 if (cpu_guest_has_kscr(4))
2562                         kvm_restore_gc0_kscratch3(cop0);
2563                 if (cpu_guest_has_kscr(5))
2564                         kvm_restore_gc0_kscratch4(cop0);
2565                 if (cpu_guest_has_kscr(6))
2566                         kvm_restore_gc0_kscratch5(cop0);
2567                 if (cpu_guest_has_kscr(7))
2568                         kvm_restore_gc0_kscratch6(cop0);
2569         }
2570
2571         if (cpu_guest_has_badinstr)
2572                 kvm_restore_gc0_badinstr(cop0);
2573         if (cpu_guest_has_badinstrp)
2574                 kvm_restore_gc0_badinstrp(cop0);
2575
2576         if (cpu_guest_has_segments) {
2577                 kvm_restore_gc0_segctl0(cop0);
2578                 kvm_restore_gc0_segctl1(cop0);
2579                 kvm_restore_gc0_segctl2(cop0);
2580         }
2581
2582         /* restore HTW registers */
2583         if (cpu_guest_has_htw) {
2584                 kvm_restore_gc0_pwbase(cop0);
2585                 kvm_restore_gc0_pwfield(cop0);
2586                 kvm_restore_gc0_pwsize(cop0);
2587                 kvm_restore_gc0_pwctl(cop0);
2588         }
2589
2590         /* restore Root.GuestCtl2 from unused Guest guestctl2 register */
2591         if (cpu_has_guestctl2)
2592                 write_c0_guestctl2(
2593                         cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL]);
2594
2595         /*
2596          * We should clear linked load bit to break interrupted atomics. This
2597          * prevents a SC on the next VCPU from succeeding by matching a LL on
2598          * the previous VCPU.
2599          */
2600         if (cpu_guest_has_rw_llb)
2601                 write_gc0_lladdr(0);
2602
2603         return 0;
2604 }
2605
2606 static int kvm_vz_vcpu_put(struct kvm_vcpu *vcpu, int cpu)
2607 {
2608         struct mips_coproc *cop0 = vcpu->arch.cop0;
2609
2610         if (current->flags & PF_VCPU)
2611                 kvm_vz_vcpu_save_wired(vcpu);
2612
2613         kvm_lose_fpu(vcpu);
2614
2615         kvm_save_gc0_index(cop0);
2616         kvm_save_gc0_entrylo0(cop0);
2617         kvm_save_gc0_entrylo1(cop0);
2618         kvm_save_gc0_context(cop0);
2619         if (cpu_guest_has_contextconfig)
2620                 kvm_save_gc0_contextconfig(cop0);
2621 #ifdef CONFIG_64BIT
2622         kvm_save_gc0_xcontext(cop0);
2623         if (cpu_guest_has_contextconfig)
2624                 kvm_save_gc0_xcontextconfig(cop0);
2625 #endif
2626         kvm_save_gc0_pagemask(cop0);
2627         kvm_save_gc0_pagegrain(cop0);
2628         kvm_save_gc0_wired(cop0);
2629         /* allow wired TLB entries to be overwritten */
2630         clear_gc0_wired(MIPSR6_WIRED_WIRED);
2631         kvm_save_gc0_hwrena(cop0);
2632         kvm_save_gc0_badvaddr(cop0);
2633         kvm_save_gc0_entryhi(cop0);
2634         kvm_save_gc0_status(cop0);
2635         kvm_save_gc0_intctl(cop0);
2636         kvm_save_gc0_epc(cop0);
2637         kvm_write_sw_gc0_ebase(cop0, kvm_vz_read_gc0_ebase());
2638         if (cpu_guest_has_userlocal)
2639                 kvm_save_gc0_userlocal(cop0);
2640
2641         /* only save implemented config registers */
2642         kvm_save_gc0_config(cop0);
2643         if (cpu_guest_has_conf1)
2644                 kvm_save_gc0_config1(cop0);
2645         if (cpu_guest_has_conf2)
2646                 kvm_save_gc0_config2(cop0);
2647         if (cpu_guest_has_conf3)
2648                 kvm_save_gc0_config3(cop0);
2649         if (cpu_guest_has_conf4)
2650                 kvm_save_gc0_config4(cop0);
2651         if (cpu_guest_has_conf5)
2652                 kvm_save_gc0_config5(cop0);
2653         if (cpu_guest_has_conf6)
2654                 kvm_save_gc0_config6(cop0);
2655         if (cpu_guest_has_conf7)
2656                 kvm_save_gc0_config7(cop0);
2657
2658         kvm_save_gc0_errorepc(cop0);
2659
2660         /* save KScratch registers if enabled in guest */
2661         if (cpu_guest_has_conf4) {
2662                 if (cpu_guest_has_kscr(2))
2663                         kvm_save_gc0_kscratch1(cop0);
2664                 if (cpu_guest_has_kscr(3))
2665                         kvm_save_gc0_kscratch2(cop0);
2666                 if (cpu_guest_has_kscr(4))
2667                         kvm_save_gc0_kscratch3(cop0);
2668                 if (cpu_guest_has_kscr(5))
2669                         kvm_save_gc0_kscratch4(cop0);
2670                 if (cpu_guest_has_kscr(6))
2671                         kvm_save_gc0_kscratch5(cop0);
2672                 if (cpu_guest_has_kscr(7))
2673                         kvm_save_gc0_kscratch6(cop0);
2674         }
2675
2676         if (cpu_guest_has_badinstr)
2677                 kvm_save_gc0_badinstr(cop0);
2678         if (cpu_guest_has_badinstrp)
2679                 kvm_save_gc0_badinstrp(cop0);
2680
2681         if (cpu_guest_has_segments) {
2682                 kvm_save_gc0_segctl0(cop0);
2683                 kvm_save_gc0_segctl1(cop0);
2684                 kvm_save_gc0_segctl2(cop0);
2685         }
2686
2687         /* save HTW registers if enabled in guest */
2688         if (cpu_guest_has_htw &&
2689             kvm_read_sw_gc0_config3(cop0) & MIPS_CONF3_PW) {
2690                 kvm_save_gc0_pwbase(cop0);
2691                 kvm_save_gc0_pwfield(cop0);
2692                 kvm_save_gc0_pwsize(cop0);
2693                 kvm_save_gc0_pwctl(cop0);
2694         }
2695
2696         kvm_vz_save_timer(vcpu);
2697
2698         /* save Root.GuestCtl2 in unused Guest guestctl2 register */
2699         if (cpu_has_guestctl2)
2700                 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL] =
2701                         read_c0_guestctl2();
2702
2703         return 0;
2704 }
2705
2706 /**
2707  * kvm_vz_resize_guest_vtlb() - Attempt to resize guest VTLB.
2708  * @size:       Number of guest VTLB entries (0 < @size <= root VTLB entries).
2709  *
2710  * Attempt to resize the guest VTLB by writing guest Config registers. This is
2711  * necessary for cores with a shared root/guest TLB to avoid overlap with wired
2712  * entries in the root VTLB.
2713  *
2714  * Returns:     The resulting guest VTLB size.
2715  */
2716 static unsigned int kvm_vz_resize_guest_vtlb(unsigned int size)
2717 {
2718         unsigned int config4 = 0, ret = 0, limit;
2719
2720         /* Write MMUSize - 1 into guest Config registers */
2721         if (cpu_guest_has_conf1)
2722                 change_gc0_config1(MIPS_CONF1_TLBS,
2723                                    (size - 1) << MIPS_CONF1_TLBS_SHIFT);
2724         if (cpu_guest_has_conf4) {
2725                 config4 = read_gc0_config4();
2726                 if (cpu_has_mips_r6 || (config4 & MIPS_CONF4_MMUEXTDEF) ==
2727                     MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT) {
2728                         config4 &= ~MIPS_CONF4_VTLBSIZEEXT;
2729                         config4 |= ((size - 1) >> MIPS_CONF1_TLBS_SIZE) <<
2730                                 MIPS_CONF4_VTLBSIZEEXT_SHIFT;
2731                 } else if ((config4 & MIPS_CONF4_MMUEXTDEF) ==
2732                            MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT) {
2733                         config4 &= ~MIPS_CONF4_MMUSIZEEXT;
2734                         config4 |= ((size - 1) >> MIPS_CONF1_TLBS_SIZE) <<
2735                                 MIPS_CONF4_MMUSIZEEXT_SHIFT;
2736                 }
2737                 write_gc0_config4(config4);
2738         }
2739
2740         /*
2741          * Set Guest.Wired.Limit = 0 (no limit up to Guest.MMUSize-1), unless it
2742          * would exceed Root.Wired.Limit (clearing Guest.Wired.Wired so write
2743          * not dropped)
2744          */
2745         if (cpu_has_mips_r6) {
2746                 limit = (read_c0_wired() & MIPSR6_WIRED_LIMIT) >>
2747                                                 MIPSR6_WIRED_LIMIT_SHIFT;
2748                 if (size - 1 <= limit)
2749                         limit = 0;
2750                 write_gc0_wired(limit << MIPSR6_WIRED_LIMIT_SHIFT);
2751         }
2752
2753         /* Read back MMUSize - 1 */
2754         back_to_back_c0_hazard();
2755         if (cpu_guest_has_conf1)
2756                 ret = (read_gc0_config1() & MIPS_CONF1_TLBS) >>
2757                                                 MIPS_CONF1_TLBS_SHIFT;
2758         if (config4) {
2759                 if (cpu_has_mips_r6 || (config4 & MIPS_CONF4_MMUEXTDEF) ==
2760                     MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT)
2761                         ret |= ((config4 & MIPS_CONF4_VTLBSIZEEXT) >>
2762                                 MIPS_CONF4_VTLBSIZEEXT_SHIFT) <<
2763                                 MIPS_CONF1_TLBS_SIZE;
2764                 else if ((config4 & MIPS_CONF4_MMUEXTDEF) ==
2765                          MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT)
2766                         ret |= ((config4 & MIPS_CONF4_MMUSIZEEXT) >>
2767                                 MIPS_CONF4_MMUSIZEEXT_SHIFT) <<
2768                                 MIPS_CONF1_TLBS_SIZE;
2769         }
2770         return ret + 1;
2771 }
2772
2773 static int kvm_vz_hardware_enable(void)
2774 {
2775         unsigned int mmu_size, guest_mmu_size, ftlb_size;
2776         u64 guest_cvmctl, cvmvmconfig;
2777
2778         switch (current_cpu_type()) {
2779         case CPU_CAVIUM_OCTEON3:
2780                 /* Set up guest timer/perfcount IRQ lines */
2781                 guest_cvmctl = read_gc0_cvmctl();
2782                 guest_cvmctl &= ~CVMCTL_IPTI;
2783                 guest_cvmctl |= 7ull << CVMCTL_IPTI_SHIFT;
2784                 guest_cvmctl &= ~CVMCTL_IPPCI;
2785                 guest_cvmctl |= 6ull << CVMCTL_IPPCI_SHIFT;
2786                 write_gc0_cvmctl(guest_cvmctl);
2787
2788                 cvmvmconfig = read_c0_cvmvmconfig();
2789                 /* No I/O hole translation. */
2790                 cvmvmconfig |= CVMVMCONF_DGHT;
2791                 /* Halve the root MMU size */
2792                 mmu_size = ((cvmvmconfig & CVMVMCONF_MMUSIZEM1)
2793                             >> CVMVMCONF_MMUSIZEM1_S) + 1;
2794                 guest_mmu_size = mmu_size / 2;
2795                 mmu_size -= guest_mmu_size;
2796                 cvmvmconfig &= ~CVMVMCONF_RMMUSIZEM1;
2797                 cvmvmconfig |= mmu_size - 1;
2798                 write_c0_cvmvmconfig(cvmvmconfig);
2799
2800                 /* Update our records */
2801                 current_cpu_data.tlbsize = mmu_size;
2802                 current_cpu_data.tlbsizevtlb = mmu_size;
2803                 current_cpu_data.guest.tlbsize = guest_mmu_size;
2804
2805                 /* Flush moved entries in new (guest) context */
2806                 kvm_vz_local_flush_guesttlb_all();
2807                 break;
2808         default:
2809                 /*
2810                  * ImgTec cores tend to use a shared root/guest TLB. To avoid
2811                  * overlap of root wired and guest entries, the guest TLB may
2812                  * need resizing.
2813                  */
2814                 mmu_size = current_cpu_data.tlbsizevtlb;
2815                 ftlb_size = current_cpu_data.tlbsize - mmu_size;
2816
2817                 /* Try switching to maximum guest VTLB size for flush */
2818                 guest_mmu_size = kvm_vz_resize_guest_vtlb(mmu_size);
2819                 current_cpu_data.guest.tlbsize = guest_mmu_size + ftlb_size;
2820                 kvm_vz_local_flush_guesttlb_all();
2821
2822                 /*
2823                  * Reduce to make space for root wired entries and at least 2
2824                  * root non-wired entries. This does assume that long-term wired
2825                  * entries won't be added later.
2826                  */
2827                 guest_mmu_size = mmu_size - num_wired_entries() - 2;
2828                 guest_mmu_size = kvm_vz_resize_guest_vtlb(guest_mmu_size);
2829                 current_cpu_data.guest.tlbsize = guest_mmu_size + ftlb_size;
2830
2831                 /*
2832                  * Write the VTLB size, but if another CPU has already written,
2833                  * check it matches or we won't provide a consistent view to the
2834                  * guest. If this ever happens it suggests an asymmetric number
2835                  * of wired entries.
2836                  */
2837                 if (cmpxchg(&kvm_vz_guest_vtlb_size, 0, guest_mmu_size) &&
2838                     WARN(guest_mmu_size != kvm_vz_guest_vtlb_size,
2839                          "Available guest VTLB size mismatch"))
2840                         return -EINVAL;
2841                 break;
2842         }
2843
2844         /*
2845          * Enable virtualization features granting guest direct control of
2846          * certain features:
2847          * CP0=1:       Guest coprocessor 0 context.
2848          * AT=Guest:    Guest MMU.
2849          * CG=1:        Hit (virtual address) CACHE operations (optional).
2850          * CF=1:        Guest Config registers.
2851          * CGI=1:       Indexed flush CACHE operations (optional).
2852          */
2853         write_c0_guestctl0(MIPS_GCTL0_CP0 |
2854                            (MIPS_GCTL0_AT_GUEST << MIPS_GCTL0_AT_SHIFT) |
2855                            MIPS_GCTL0_CG | MIPS_GCTL0_CF);
2856         if (cpu_has_guestctl0ext)
2857                 set_c0_guestctl0ext(MIPS_GCTL0EXT_CGI);
2858
2859         if (cpu_has_guestid) {
2860                 write_c0_guestctl1(0);
2861                 kvm_vz_local_flush_roottlb_all_guests();
2862
2863                 GUESTID_MASK = current_cpu_data.guestid_mask;
2864                 GUESTID_FIRST_VERSION = GUESTID_MASK + 1;
2865                 GUESTID_VERSION_MASK = ~GUESTID_MASK;
2866
2867                 current_cpu_data.guestid_cache = GUESTID_FIRST_VERSION;
2868         }
2869
2870         /* clear any pending injected virtual guest interrupts */
2871         if (cpu_has_guestctl2)
2872                 clear_c0_guestctl2(0x3f << 10);
2873
2874         return 0;
2875 }
2876
2877 static void kvm_vz_hardware_disable(void)
2878 {
2879         u64 cvmvmconfig;
2880         unsigned int mmu_size;
2881
2882         /* Flush any remaining guest TLB entries */
2883         kvm_vz_local_flush_guesttlb_all();
2884
2885         switch (current_cpu_type()) {
2886         case CPU_CAVIUM_OCTEON3:
2887                 /*
2888                  * Allocate whole TLB for root. Existing guest TLB entries will
2889                  * change ownership to the root TLB. We should be safe though as
2890                  * they've already been flushed above while in guest TLB.
2891                  */
2892                 cvmvmconfig = read_c0_cvmvmconfig();
2893                 mmu_size = ((cvmvmconfig & CVMVMCONF_MMUSIZEM1)
2894                             >> CVMVMCONF_MMUSIZEM1_S) + 1;
2895                 cvmvmconfig &= ~CVMVMCONF_RMMUSIZEM1;
2896                 cvmvmconfig |= mmu_size - 1;
2897                 write_c0_cvmvmconfig(cvmvmconfig);
2898
2899                 /* Update our records */
2900                 current_cpu_data.tlbsize = mmu_size;
2901                 current_cpu_data.tlbsizevtlb = mmu_size;
2902                 current_cpu_data.guest.tlbsize = 0;
2903
2904                 /* Flush moved entries in new (root) context */
2905                 local_flush_tlb_all();
2906                 break;
2907         }
2908
2909         if (cpu_has_guestid) {
2910                 write_c0_guestctl1(0);
2911                 kvm_vz_local_flush_roottlb_all_guests();
2912         }
2913 }
2914
2915 static int kvm_vz_check_extension(struct kvm *kvm, long ext)
2916 {
2917         int r;
2918
2919         switch (ext) {
2920         case KVM_CAP_MIPS_VZ:
2921                 /* we wouldn't be here unless cpu_has_vz */
2922                 r = 1;
2923                 break;
2924 #ifdef CONFIG_64BIT
2925         case KVM_CAP_MIPS_64BIT:
2926                 /* We support 64-bit registers/operations and addresses */
2927                 r = 2;
2928                 break;
2929 #endif
2930         default:
2931                 r = 0;
2932                 break;
2933         }
2934
2935         return r;
2936 }
2937
2938 static int kvm_vz_vcpu_init(struct kvm_vcpu *vcpu)
2939 {
2940         int i;
2941
2942         for_each_possible_cpu(i)
2943                 vcpu->arch.vzguestid[i] = 0;
2944
2945         return 0;
2946 }
2947
2948 static void kvm_vz_vcpu_uninit(struct kvm_vcpu *vcpu)
2949 {
2950         int cpu;
2951
2952         /*
2953          * If the VCPU is freed and reused as another VCPU, we don't want the
2954          * matching pointer wrongly hanging around in last_vcpu[] or
2955          * last_exec_vcpu[].
2956          */
2957         for_each_possible_cpu(cpu) {
2958                 if (last_vcpu[cpu] == vcpu)
2959                         last_vcpu[cpu] = NULL;
2960                 if (last_exec_vcpu[cpu] == vcpu)
2961                         last_exec_vcpu[cpu] = NULL;
2962         }
2963 }
2964
2965 static int kvm_vz_vcpu_setup(struct kvm_vcpu *vcpu)
2966 {
2967         struct mips_coproc *cop0 = vcpu->arch.cop0;
2968         unsigned long count_hz = 100*1000*1000; /* default to 100 MHz */
2969
2970         /*
2971          * Start off the timer at the same frequency as the host timer, but the
2972          * soft timer doesn't handle frequencies greater than 1GHz yet.
2973          */
2974         if (mips_hpt_frequency && mips_hpt_frequency <= NSEC_PER_SEC)
2975                 count_hz = mips_hpt_frequency;
2976         kvm_mips_init_count(vcpu, count_hz);
2977
2978         /*
2979          * Initialize guest register state to valid architectural reset state.
2980          */
2981
2982         /* PageGrain */
2983         if (cpu_has_mips_r6)
2984                 kvm_write_sw_gc0_pagegrain(cop0, PG_RIE | PG_XIE | PG_IEC);
2985         /* Wired */
2986         if (cpu_has_mips_r6)
2987                 kvm_write_sw_gc0_wired(cop0,
2988                                        read_gc0_wired() & MIPSR6_WIRED_LIMIT);
2989         /* Status */
2990         kvm_write_sw_gc0_status(cop0, ST0_BEV | ST0_ERL);
2991         if (cpu_has_mips_r6)
2992                 kvm_change_sw_gc0_status(cop0, ST0_FR, read_gc0_status());
2993         /* IntCtl */
2994         kvm_write_sw_gc0_intctl(cop0, read_gc0_intctl() &
2995                                 (INTCTLF_IPFDC | INTCTLF_IPPCI | INTCTLF_IPTI));
2996         /* PRId */
2997         kvm_write_sw_gc0_prid(cop0, boot_cpu_data.processor_id);
2998         /* EBase */
2999         kvm_write_sw_gc0_ebase(cop0, (s32)0x80000000 | vcpu->vcpu_id);
3000         /* Config */
3001         kvm_save_gc0_config(cop0);
3002         /* architecturally writable (e.g. from guest) */
3003         kvm_change_sw_gc0_config(cop0, CONF_CM_CMASK,
3004                                  _page_cachable_default >> _CACHE_SHIFT);
3005         /* architecturally read only, but maybe writable from root */
3006         kvm_change_sw_gc0_config(cop0, MIPS_CONF_MT, read_c0_config());
3007         if (cpu_guest_has_conf1) {
3008                 kvm_set_sw_gc0_config(cop0, MIPS_CONF_M);
3009                 /* Config1 */
3010                 kvm_save_gc0_config1(cop0);
3011                 /* architecturally read only, but maybe writable from root */
3012                 kvm_clear_sw_gc0_config1(cop0, MIPS_CONF1_C2    |
3013                                                MIPS_CONF1_MD    |
3014                                                MIPS_CONF1_PC    |
3015                                                MIPS_CONF1_WR    |
3016                                                MIPS_CONF1_CA    |
3017                                                MIPS_CONF1_FP);
3018         }
3019         if (cpu_guest_has_conf2) {
3020                 kvm_set_sw_gc0_config1(cop0, MIPS_CONF_M);
3021                 /* Config2 */
3022                 kvm_save_gc0_config2(cop0);
3023         }
3024         if (cpu_guest_has_conf3) {
3025                 kvm_set_sw_gc0_config2(cop0, MIPS_CONF_M);
3026                 /* Config3 */
3027                 kvm_save_gc0_config3(cop0);
3028                 /* architecturally writable (e.g. from guest) */
3029                 kvm_clear_sw_gc0_config3(cop0, MIPS_CONF3_ISA_OE);
3030                 /* architecturally read only, but maybe writable from root */
3031                 kvm_clear_sw_gc0_config3(cop0, MIPS_CONF3_MSA   |
3032                                                MIPS_CONF3_BPG   |
3033                                                MIPS_CONF3_ULRI  |
3034                                                MIPS_CONF3_DSP   |
3035                                                MIPS_CONF3_CTXTC |
3036                                                MIPS_CONF3_ITL   |
3037                                                MIPS_CONF3_LPA   |
3038                                                MIPS_CONF3_VEIC  |
3039                                                MIPS_CONF3_VINT  |
3040                                                MIPS_CONF3_SP    |
3041                                                MIPS_CONF3_CDMM  |
3042                                                MIPS_CONF3_MT    |
3043                                                MIPS_CONF3_SM    |
3044                                                MIPS_CONF3_TL);
3045         }
3046         if (cpu_guest_has_conf4) {
3047                 kvm_set_sw_gc0_config3(cop0, MIPS_CONF_M);
3048                 /* Config4 */
3049                 kvm_save_gc0_config4(cop0);
3050         }
3051         if (cpu_guest_has_conf5) {
3052                 kvm_set_sw_gc0_config4(cop0, MIPS_CONF_M);
3053                 /* Config5 */
3054                 kvm_save_gc0_config5(cop0);
3055                 /* architecturally writable (e.g. from guest) */
3056                 kvm_clear_sw_gc0_config5(cop0, MIPS_CONF5_K     |
3057                                                MIPS_CONF5_CV    |
3058                                                MIPS_CONF5_MSAEN |
3059                                                MIPS_CONF5_UFE   |
3060                                                MIPS_CONF5_FRE   |
3061                                                MIPS_CONF5_SBRI  |
3062                                                MIPS_CONF5_UFR);
3063                 /* architecturally read only, but maybe writable from root */
3064                 kvm_clear_sw_gc0_config5(cop0, MIPS_CONF5_MRP);
3065         }
3066
3067         if (cpu_guest_has_contextconfig) {
3068                 /* ContextConfig */
3069                 kvm_write_sw_gc0_contextconfig(cop0, 0x007ffff0);
3070 #ifdef CONFIG_64BIT
3071                 /* XContextConfig */
3072                 /* bits SEGBITS-13+3:4 set */
3073                 kvm_write_sw_gc0_xcontextconfig(cop0,
3074                                         ((1ull << (cpu_vmbits - 13)) - 1) << 4);
3075 #endif
3076         }
3077
3078         /* Implementation dependent, use the legacy layout */
3079         if (cpu_guest_has_segments) {
3080                 /* SegCtl0, SegCtl1, SegCtl2 */
3081                 kvm_write_sw_gc0_segctl0(cop0, 0x00200010);
3082                 kvm_write_sw_gc0_segctl1(cop0, 0x00000002 |
3083                                 (_page_cachable_default >> _CACHE_SHIFT) <<
3084                                                 (16 + MIPS_SEGCFG_C_SHIFT));
3085                 kvm_write_sw_gc0_segctl2(cop0, 0x00380438);
3086         }
3087
3088         /* reset HTW registers */
3089         if (cpu_guest_has_htw && cpu_has_mips_r6) {
3090                 /* PWField */
3091                 kvm_write_sw_gc0_pwfield(cop0, 0x0c30c302);
3092                 /* PWSize */
3093                 kvm_write_sw_gc0_pwsize(cop0, 1 << MIPS_PWSIZE_PTW_SHIFT);
3094         }
3095
3096         /* start with no pending virtual guest interrupts */
3097         if (cpu_has_guestctl2)
3098                 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL] = 0;
3099
3100         /* Put PC at reset vector */
3101         vcpu->arch.pc = CKSEG1ADDR(0x1fc00000);
3102
3103         return 0;
3104 }
3105
3106 static void kvm_vz_flush_shadow_all(struct kvm *kvm)
3107 {
3108         if (cpu_has_guestid) {
3109                 /* Flush GuestID for each VCPU individually */
3110                 kvm_flush_remote_tlbs(kvm);
3111         } else {
3112                 /*
3113                  * For each CPU there is a single GPA ASID used by all VCPUs in
3114                  * the VM, so it doesn't make sense for the VCPUs to handle
3115                  * invalidation of these ASIDs individually.
3116                  *
3117                  * Instead mark all CPUs as needing ASID invalidation in
3118                  * asid_flush_mask, and just use kvm_flush_remote_tlbs(kvm) to
3119                  * kick any running VCPUs so they check asid_flush_mask.
3120                  */
3121                 cpumask_setall(&kvm->arch.asid_flush_mask);
3122                 kvm_flush_remote_tlbs(kvm);
3123         }
3124 }
3125
3126 static void kvm_vz_flush_shadow_memslot(struct kvm *kvm,
3127                                         const struct kvm_memory_slot *slot)
3128 {
3129         kvm_vz_flush_shadow_all(kvm);
3130 }
3131
3132 static void kvm_vz_vcpu_reenter(struct kvm_run *run, struct kvm_vcpu *vcpu)
3133 {
3134         int cpu = smp_processor_id();
3135         int preserve_guest_tlb;
3136
3137         preserve_guest_tlb = kvm_vz_check_requests(vcpu, cpu);
3138
3139         if (preserve_guest_tlb)
3140                 kvm_vz_vcpu_save_wired(vcpu);
3141
3142         kvm_vz_vcpu_load_tlb(vcpu, cpu);
3143
3144         if (preserve_guest_tlb)
3145                 kvm_vz_vcpu_load_wired(vcpu);
3146 }
3147
3148 static int kvm_vz_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu)
3149 {
3150         int cpu = smp_processor_id();
3151         int r;
3152
3153         kvm_vz_acquire_htimer(vcpu);
3154         /* Check if we have any exceptions/interrupts pending */
3155         kvm_mips_deliver_interrupts(vcpu, read_gc0_cause());
3156
3157         kvm_vz_check_requests(vcpu, cpu);
3158         kvm_vz_vcpu_load_tlb(vcpu, cpu);
3159         kvm_vz_vcpu_load_wired(vcpu);
3160
3161         r = vcpu->arch.vcpu_run(run, vcpu);
3162
3163         kvm_vz_vcpu_save_wired(vcpu);
3164
3165         return r;
3166 }
3167
3168 static struct kvm_mips_callbacks kvm_vz_callbacks = {
3169         .handle_cop_unusable = kvm_trap_vz_handle_cop_unusable,
3170         .handle_tlb_mod = kvm_trap_vz_handle_tlb_st_miss,
3171         .handle_tlb_ld_miss = kvm_trap_vz_handle_tlb_ld_miss,
3172         .handle_tlb_st_miss = kvm_trap_vz_handle_tlb_st_miss,
3173         .handle_addr_err_st = kvm_trap_vz_no_handler,
3174         .handle_addr_err_ld = kvm_trap_vz_no_handler,
3175         .handle_syscall = kvm_trap_vz_no_handler,
3176         .handle_res_inst = kvm_trap_vz_no_handler,
3177         .handle_break = kvm_trap_vz_no_handler,
3178         .handle_msa_disabled = kvm_trap_vz_handle_msa_disabled,
3179         .handle_guest_exit = kvm_trap_vz_handle_guest_exit,
3180
3181         .hardware_enable = kvm_vz_hardware_enable,
3182         .hardware_disable = kvm_vz_hardware_disable,
3183         .check_extension = kvm_vz_check_extension,
3184         .vcpu_init = kvm_vz_vcpu_init,
3185         .vcpu_uninit = kvm_vz_vcpu_uninit,
3186         .vcpu_setup = kvm_vz_vcpu_setup,
3187         .flush_shadow_all = kvm_vz_flush_shadow_all,
3188         .flush_shadow_memslot = kvm_vz_flush_shadow_memslot,
3189         .gva_to_gpa = kvm_vz_gva_to_gpa_cb,
3190         .queue_timer_int = kvm_vz_queue_timer_int_cb,
3191         .dequeue_timer_int = kvm_vz_dequeue_timer_int_cb,
3192         .queue_io_int = kvm_vz_queue_io_int_cb,
3193         .dequeue_io_int = kvm_vz_dequeue_io_int_cb,
3194         .irq_deliver = kvm_vz_irq_deliver_cb,
3195         .irq_clear = kvm_vz_irq_clear_cb,
3196         .num_regs = kvm_vz_num_regs,
3197         .copy_reg_indices = kvm_vz_copy_reg_indices,
3198         .get_one_reg = kvm_vz_get_one_reg,
3199         .set_one_reg = kvm_vz_set_one_reg,
3200         .vcpu_load = kvm_vz_vcpu_load,
3201         .vcpu_put = kvm_vz_vcpu_put,
3202         .vcpu_run = kvm_vz_vcpu_run,
3203         .vcpu_reenter = kvm_vz_vcpu_reenter,
3204 };
3205
3206 int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks)
3207 {
3208         if (!cpu_has_vz)
3209                 return -ENODEV;
3210
3211         /*
3212          * VZ requires at least 2 KScratch registers, so it should have been
3213          * possible to allocate pgd_reg.
3214          */
3215         if (WARN(pgd_reg == -1,
3216                  "pgd_reg not allocated even though cpu_has_vz\n"))
3217                 return -ENODEV;
3218
3219         pr_info("Starting KVM with MIPS VZ extensions\n");
3220
3221         *install_callbacks = &kvm_vz_callbacks;
3222         return 0;
3223 }