arch/s390/kvm/kvm-s390.c

   1 /*
   2  * hosting zSeries kernel virtual machines
   3  *
   4  * Copyright IBM Corp. 2008, 2009
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2 only)
   8  * as published by the Free Software Foundation.
   9  *
  10  *    Author(s): Carsten Otte <[email protected]>
  11  *               Christian Borntraeger <[email protected]>
  12  *               Heiko Carstens <[email protected]>
  13  *               Christian Ehrhardt <[email protected]>
  14  */
  15
  16 #include <linux/compiler.h>
  17 #include <linux/err.h>
  18 #include <linux/fs.h>
  19 #include <linux/hrtimer.h>
  20 #include <linux/init.h>
  21 #include <linux/kvm.h>
  22 #include <linux/kvm_host.h>
  23 #include <linux/module.h>
  24 #include <linux/slab.h>
  25 #include <linux/timer.h>
  26 #include <asm/asm-offsets.h>
  27 #include <asm/lowcore.h>
  28 #include <asm/pgtable.h>
  29 #include <asm/nmi.h>
  30 #include <asm/switch_to.h>
  31 #include <asm/facility.h>
  32 #include <asm/sclp.h>
  33 #include "kvm-s390.h"
  34 #include "gaccess.h"
  35
  36 #define CREATE_TRACE_POINTS
  37 #include "trace.h"
  38 #include "trace-s390.h"
  39
  40 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  41
  42 struct kvm_stats_debugfs_item debugfs_entries[] = {
  43         { "userspace_handled", VCPU_STAT(exit_userspace) },
  44         { "exit_null", VCPU_STAT(exit_null) },
  45         { "exit_validity", VCPU_STAT(exit_validity) },
  46         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  47         { "exit_external_request", VCPU_STAT(exit_external_request) },
  48         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  49         { "exit_instruction", VCPU_STAT(exit_instruction) },
  50         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  51         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  52         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  53         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  54         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  55         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  56         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  57         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
  58         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  59         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  60         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  61         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
  62         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  63         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
  64         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
  65         { "instruction_spx", VCPU_STAT(instruction_spx) },
  66         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
  67         { "instruction_stap", VCPU_STAT(instruction_stap) },
  68         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
  69         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
  70         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
  71         { "instruction_essa", VCPU_STAT(instruction_essa) },
  72         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
  73         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
  74         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
  75         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
  76         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
  77         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
  78         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
  79         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
  80         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
  81         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
  82         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
  83         { "diagnose_10", VCPU_STAT(diagnose_10) },
  84         { "diagnose_44", VCPU_STAT(diagnose_44) },
  85         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
  86         { NULL }
  87 };
  88
  89 unsigned long *vfacilities;
  90 static struct gmap_notifier gmap_notifier;
  91
  92 /* test availability of vfacility */
  93 static inline int test_vfacility(unsigned long nr)
  94 {
  95         return __test_facility(nr, (void *) vfacilities);
  96 }
  97
  98 /* Section: not file related */
  99 int kvm_arch_hardware_enable(void *garbage)
 100 {
 101         /* every s390 is virtualization enabled ;-) */
 102         return 0;
 103 }
 104
 105 void kvm_arch_hardware_disable(void *garbage)
 106 {
 107 }
 108
 109 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
 110
 111 int kvm_arch_hardware_setup(void)
 112 {
 113         gmap_notifier.notifier_call = kvm_gmap_notifier;
 114         gmap_register_ipte_notifier(&gmap_notifier);
 115         return 0;
 116 }
 117
 118 void kvm_arch_hardware_unsetup(void)
 119 {
 120         gmap_unregister_ipte_notifier(&gmap_notifier);
 121 }
 122
 123 void kvm_arch_check_processor_compat(void *rtn)
 124 {
 125 }
 126
 127 int kvm_arch_init(void *opaque)
 128 {
 129         return 0;
 130 }
 131
 132 void kvm_arch_exit(void)
 133 {
 134 }
 135
 136 /* Section: device related */
 137 long kvm_arch_dev_ioctl(struct file *filp,
 138                         unsigned int ioctl, unsigned long arg)
 139 {
 140         if (ioctl == KVM_S390_ENABLE_SIE)
 141                 return s390_enable_sie();
 142         return -EINVAL;
 143 }
 144
 145 int kvm_dev_ioctl_check_extension(long ext)
 146 {
 147         int r;
 148
 149         switch (ext) {
 150         case KVM_CAP_S390_PSW:
 151         case KVM_CAP_S390_GMAP:
 152         case KVM_CAP_SYNC_MMU:
 153 #ifdef CONFIG_KVM_S390_UCONTROL
 154         case KVM_CAP_S390_UCONTROL:
 155 #endif
 156         case KVM_CAP_ASYNC_PF:
 157         case KVM_CAP_SYNC_REGS:
 158         case KVM_CAP_ONE_REG:
 159         case KVM_CAP_ENABLE_CAP:
 160         case KVM_CAP_S390_CSS_SUPPORT:
 161         case KVM_CAP_IOEVENTFD:
 162         case KVM_CAP_DEVICE_CTRL:
 163         case KVM_CAP_ENABLE_CAP_VM:
 164                 r = 1;
 165                 break;
 166         case KVM_CAP_NR_VCPUS:
 167         case KVM_CAP_MAX_VCPUS:
 168                 r = KVM_MAX_VCPUS;
 169                 break;
 170         case KVM_CAP_NR_MEMSLOTS:
 171                 r = KVM_USER_MEM_SLOTS;
 172                 break;
 173         case KVM_CAP_S390_COW:
 174                 r = MACHINE_HAS_ESOP;
 175                 break;
 176         default:
 177                 r = 0;
 178         }
 179         return r;
 180 }
 181
 182 /* Section: vm related */
 183 /*
 184  * Get (and clear) the dirty memory log for a memory slot.
 185  */
 186 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 187                                struct kvm_dirty_log *log)
 188 {
 189         return 0;
 190 }
 191
 192 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 193 {
 194         int r;
 195
 196         if (cap->flags)
 197                 return -EINVAL;
 198
 199         switch (cap->cap) {
 200         case KVM_CAP_S390_IRQCHIP:
 201                 kvm->arch.use_irqchip = 1;
 202                 r = 0;
 203                 break;
 204         default:
 205                 r = -EINVAL;
 206                 break;
 207         }
 208         return r;
 209 }
 210
 211 long kvm_arch_vm_ioctl(struct file *filp,
 212                        unsigned int ioctl, unsigned long arg)
 213 {
 214         struct kvm *kvm = filp->private_data;
 215         void __user *argp = (void __user *)arg;
 216         int r;
 217
 218         switch (ioctl) {
 219         case KVM_S390_INTERRUPT: {
 220                 struct kvm_s390_interrupt s390int;
 221
 222                 r = -EFAULT;
 223                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
 224                         break;
 225                 r = kvm_s390_inject_vm(kvm, &s390int);
 226                 break;
 227         }
 228         case KVM_ENABLE_CAP: {
 229                 struct kvm_enable_cap cap;
 230                 r = -EFAULT;
 231                 if (copy_from_user(&cap, argp, sizeof(cap)))
 232                         break;
 233                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
 234                 break;
 235         }
 236         case KVM_CREATE_IRQCHIP: {
 237                 struct kvm_irq_routing_entry routing;
 238
 239                 r = -EINVAL;
 240                 if (kvm->arch.use_irqchip) {
 241                         /* Set up dummy routing. */
 242                         memset(&routing, 0, sizeof(routing));
 243                         kvm_set_irq_routing(kvm, &routing, 0, 0);
 244                         r = 0;
 245                 }
 246                 break;
 247         }
 248         default:
 249                 r = -ENOTTY;
 250         }
 251
 252         return r;
 253 }
 254
 255 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 256 {
 257         int rc;
 258         char debug_name[16];
 259         static unsigned long sca_offset;
 260
 261         rc = -EINVAL;
 262 #ifdef CONFIG_KVM_S390_UCONTROL
 263         if (type & ~KVM_VM_S390_UCONTROL)
 264                 goto out_err;
 265         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
 266                 goto out_err;
 267 #else
 268         if (type)
 269                 goto out_err;
 270 #endif
 271
 272         rc = s390_enable_sie();
 273         if (rc)
 274                 goto out_err;
 275
 276         rc = -ENOMEM;
 277
 278         kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
 279         if (!kvm->arch.sca)
 280                 goto out_err;
 281         spin_lock(&kvm_lock);
 282         sca_offset = (sca_offset + 16) & 0x7f0;
 283         kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
 284         spin_unlock(&kvm_lock);
 285
 286         sprintf(debug_name, "kvm-%u", current->pid);
 287
 288         kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
 289         if (!kvm->arch.dbf)
 290                 goto out_nodbf;
 291
 292         spin_lock_init(&kvm->arch.float_int.lock);
 293         INIT_LIST_HEAD(&kvm->arch.float_int.list);
 294
 295         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
 296         VM_EVENT(kvm, 3, "%s", "vm created");
 297
 298         if (type & KVM_VM_S390_UCONTROL) {
 299                 kvm->arch.gmap = NULL;
 300         } else {
 301                 kvm->arch.gmap = gmap_alloc(current->mm);
 302                 if (!kvm->arch.gmap)
 303                         goto out_nogmap;
 304                 kvm->arch.gmap->private = kvm;
 305                 kvm->arch.gmap->pfault_enabled = 0;
 306         }
 307
 308         kvm->arch.css_support = 0;
 309         kvm->arch.use_irqchip = 0;
 310
 311         return 0;
 312 out_nogmap:
 313         debug_unregister(kvm->arch.dbf);
 314 out_nodbf:
 315         free_page((unsigned long)(kvm->arch.sca));
 316 out_err:
 317         return rc;
 318 }
 319
 320 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 321 {
 322         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
 323         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
 324         kvm_clear_async_pf_completion_queue(vcpu);
 325         if (!kvm_is_ucontrol(vcpu->kvm)) {
 326                 clear_bit(63 - vcpu->vcpu_id,
 327                           (unsigned long *) &vcpu->kvm->arch.sca->mcn);
 328                 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
 329                     (__u64) vcpu->arch.sie_block)
 330                         vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
 331         }
 332         smp_mb();
 333
 334         if (kvm_is_ucontrol(vcpu->kvm))
 335                 gmap_free(vcpu->arch.gmap);
 336
 337         if (vcpu->arch.sie_block->cbrlo)
 338                 __free_page(__pfn_to_page(
 339                                 vcpu->arch.sie_block->cbrlo >> PAGE_SHIFT));
 340         free_page((unsigned long)(vcpu->arch.sie_block));
 341
 342         kvm_vcpu_uninit(vcpu);
 343         kmem_cache_free(kvm_vcpu_cache, vcpu);
 344 }
 345
 346 static void kvm_free_vcpus(struct kvm *kvm)
 347 {
 348         unsigned int i;
 349         struct kvm_vcpu *vcpu;
 350
 351         kvm_for_each_vcpu(i, vcpu, kvm)
 352                 kvm_arch_vcpu_destroy(vcpu);
 353
 354         mutex_lock(&kvm->lock);
 355         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
 356                 kvm->vcpus[i] = NULL;
 357
 358         atomic_set(&kvm->online_vcpus, 0);
 359         mutex_unlock(&kvm->lock);
 360 }
 361
 362 void kvm_arch_sync_events(struct kvm *kvm)
 363 {
 364 }
 365
 366 void kvm_arch_destroy_vm(struct kvm *kvm)
 367 {
 368         kvm_free_vcpus(kvm);
 369         free_page((unsigned long)(kvm->arch.sca));
 370         debug_unregister(kvm->arch.dbf);
 371         if (!kvm_is_ucontrol(kvm))
 372                 gmap_free(kvm->arch.gmap);
 373         kvm_s390_destroy_adapters(kvm);
 374 }
 375
 376 /* Section: vcpu related */
 377 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 378 {
 379         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
 380         kvm_clear_async_pf_completion_queue(vcpu);
 381         if (kvm_is_ucontrol(vcpu->kvm)) {
 382                 vcpu->arch.gmap = gmap_alloc(current->mm);
 383                 if (!vcpu->arch.gmap)
 384                         return -ENOMEM;
 385                 vcpu->arch.gmap->private = vcpu->kvm;
 386                 return 0;
 387         }
 388
 389         vcpu->arch.gmap = vcpu->kvm->arch.gmap;
 390         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
 391                                     KVM_SYNC_GPRS |
 392                                     KVM_SYNC_ACRS |
 393                                     KVM_SYNC_CRS;
 394         return 0;
 395 }
 396
 397 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
 398 {
 399         /* Nothing todo */
 400 }
 401
 402 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 403 {
 404         save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
 405         save_fp_regs(vcpu->arch.host_fpregs.fprs);
 406         save_access_regs(vcpu->arch.host_acrs);
 407         restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
 408         restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
 409         restore_access_regs(vcpu->run->s.regs.acrs);
 410         gmap_enable(vcpu->arch.gmap);
 411         atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
 412 }
 413
 414 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 415 {
 416         atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
 417         gmap_disable(vcpu->arch.gmap);
 418         save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
 419         save_fp_regs(vcpu->arch.guest_fpregs.fprs);
 420         save_access_regs(vcpu->run->s.regs.acrs);
 421         restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
 422         restore_fp_regs(vcpu->arch.host_fpregs.fprs);
 423         restore_access_regs(vcpu->arch.host_acrs);
 424 }
 425
 426 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
 427 {
 428         /* this equals initial cpu reset in pop, but we don't switch to ESA */
 429         vcpu->arch.sie_block->gpsw.mask = 0UL;
 430         vcpu->arch.sie_block->gpsw.addr = 0UL;
 431         kvm_s390_set_prefix(vcpu, 0);
 432         vcpu->arch.sie_block->cputm     = 0UL;
 433         vcpu->arch.sie_block->ckc       = 0UL;
 434         vcpu->arch.sie_block->todpr     = 0;
 435         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
 436         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
 437         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
 438         vcpu->arch.guest_fpregs.fpc = 0;
 439         asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
 440         vcpu->arch.sie_block->gbea = 1;
 441         vcpu->arch.sie_block->pp = 0;
 442         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
 443         kvm_clear_async_pf_completion_queue(vcpu);
 444         atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
 445         kvm_s390_clear_local_irqs(vcpu);
 446 }
 447
 448 int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 449 {
 450         return 0;
 451 }
 452
 453 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 454 {
 455         struct page *cbrl;
 456
 457         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
 458                                                     CPUSTAT_SM |
 459                                                     CPUSTAT_STOPPED |
 460                                                     CPUSTAT_GED);
 461         vcpu->arch.sie_block->ecb   = 6;
 462         if (test_vfacility(50) && test_vfacility(73))
 463                 vcpu->arch.sie_block->ecb |= 0x10;
 464
 465         vcpu->arch.sie_block->ecb2  = 8;
 466         vcpu->arch.sie_block->eca   = 0xC1002001U;
 467         vcpu->arch.sie_block->fac   = (int) (long) vfacilities;
 468         if (kvm_enabled_cmma()) {
 469                 cbrl = alloc_page(GFP_KERNEL | __GFP_ZERO);
 470                 if (cbrl) {
 471                         vcpu->arch.sie_block->ecb2 |= 0x80;
 472                         vcpu->arch.sie_block->ecb2 &= ~0x08;
 473                         vcpu->arch.sie_block->cbrlo = page_to_phys(cbrl);
 474                 }
 475         }
 476         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
 477         tasklet_init(&vcpu->arch.tasklet, kvm_s390_tasklet,
 478                      (unsigned long) vcpu);
 479         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
 480         get_cpu_id(&vcpu->arch.cpu_id);
 481         vcpu->arch.cpu_id.version = 0xff;
 482         return 0;
 483 }
 484
 485 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
 486                                       unsigned int id)
 487 {
 488         struct kvm_vcpu *vcpu;
 489         struct sie_page *sie_page;
 490         int rc = -EINVAL;
 491
 492         if (id >= KVM_MAX_VCPUS)
 493                 goto out;
 494
 495         rc = -ENOMEM;
 496
 497         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
 498         if (!vcpu)
 499                 goto out;
 500
 501         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
 502         if (!sie_page)
 503                 goto out_free_cpu;
 504
 505         vcpu->arch.sie_block = &sie_page->sie_block;
 506         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
 507
 508         vcpu->arch.sie_block->icpua = id;
 509         if (!kvm_is_ucontrol(kvm)) {
 510                 if (!kvm->arch.sca) {
 511                         WARN_ON_ONCE(1);
 512                         goto out_free_cpu;
 513                 }
 514                 if (!kvm->arch.sca->cpu[id].sda)
 515                         kvm->arch.sca->cpu[id].sda =
 516                                 (__u64) vcpu->arch.sie_block;
 517                 vcpu->arch.sie_block->scaoh =
 518                         (__u32)(((__u64)kvm->arch.sca) >> 32);
 519                 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
 520                 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
 521         }
 522
 523         spin_lock_init(&vcpu->arch.local_int.lock);
 524         INIT_LIST_HEAD(&vcpu->arch.local_int.list);
 525         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
 526         vcpu->arch.local_int.wq = &vcpu->wq;
 527         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
 528
 529         rc = kvm_vcpu_init(vcpu, kvm, id);
 530         if (rc)
 531                 goto out_free_sie_block;
 532         VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
 533                  vcpu->arch.sie_block);
 534         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
 535
 536         return vcpu;
 537 out_free_sie_block:
 538         free_page((unsigned long)(vcpu->arch.sie_block));
 539 out_free_cpu:
 540         kmem_cache_free(kvm_vcpu_cache, vcpu);
 541 out:
 542         return ERR_PTR(rc);
 543 }
 544
 545 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 546 {
 547         return kvm_cpu_has_interrupt(vcpu);
 548 }
 549
 550 void s390_vcpu_block(struct kvm_vcpu *vcpu)
 551 {
 552         atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
 553 }
 554
 555 void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
 556 {
 557         atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
 558 }
 559
 560 /*
 561  * Kick a guest cpu out of SIE and wait until SIE is not running.
 562  * If the CPU is not running (e.g. waiting as idle) the function will
 563  * return immediately. */
 564 void exit_sie(struct kvm_vcpu *vcpu)
 565 {
 566         atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
 567         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
 568                 cpu_relax();
 569 }
 570
 571 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
 572 void exit_sie_sync(struct kvm_vcpu *vcpu)
 573 {
 574         s390_vcpu_block(vcpu);
 575         exit_sie(vcpu);
 576 }
 577
 578 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
 579 {
 580         int i;
 581         struct kvm *kvm = gmap->private;
 582         struct kvm_vcpu *vcpu;
 583
 584         kvm_for_each_vcpu(i, vcpu, kvm) {
 585                 /* match against both prefix pages */
 586                 if (vcpu->arch.sie_block->prefix == (address & ~0x1000UL)) {
 587                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
 588                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
 589                         exit_sie_sync(vcpu);
 590                 }
 591         }
 592 }
 593
 594 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
 595 {
 596         /* kvm common code refers to this, but never calls it */
 597         BUG();
 598         return 0;
 599 }
 600
 601 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
 602                                            struct kvm_one_reg *reg)
 603 {
 604         int r = -EINVAL;
 605
 606         switch (reg->id) {
 607         case KVM_REG_S390_TODPR:
 608                 r = put_user(vcpu->arch.sie_block->todpr,
 609                              (u32 __user *)reg->addr);
 610                 break;
 611         case KVM_REG_S390_EPOCHDIFF:
 612                 r = put_user(vcpu->arch.sie_block->epoch,
 613                              (u64 __user *)reg->addr);
 614                 break;
 615         case KVM_REG_S390_CPU_TIMER:
 616                 r = put_user(vcpu->arch.sie_block->cputm,
 617                              (u64 __user *)reg->addr);
 618                 break;
 619         case KVM_REG_S390_CLOCK_COMP:
 620                 r = put_user(vcpu->arch.sie_block->ckc,
 621                              (u64 __user *)reg->addr);
 622                 break;
 623         case KVM_REG_S390_PFTOKEN:
 624                 r = put_user(vcpu->arch.pfault_token,
 625                              (u64 __user *)reg->addr);
 626                 break;
 627         case KVM_REG_S390_PFCOMPARE:
 628                 r = put_user(vcpu->arch.pfault_compare,
 629                              (u64 __user *)reg->addr);
 630                 break;
 631         case KVM_REG_S390_PFSELECT:
 632                 r = put_user(vcpu->arch.pfault_select,
 633                              (u64 __user *)reg->addr);
 634                 break;
 635         case KVM_REG_S390_PP:
 636                 r = put_user(vcpu->arch.sie_block->pp,
 637                              (u64 __user *)reg->addr);
 638                 break;
 639         case KVM_REG_S390_GBEA:
 640                 r = put_user(vcpu->arch.sie_block->gbea,
 641                              (u64 __user *)reg->addr);
 642                 break;
 643         default:
 644                 break;
 645         }
 646
 647         return r;
 648 }
 649
 650 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
 651                                            struct kvm_one_reg *reg)
 652 {
 653         int r = -EINVAL;
 654
 655         switch (reg->id) {
 656         case KVM_REG_S390_TODPR:
 657                 r = get_user(vcpu->arch.sie_block->todpr,
 658                              (u32 __user *)reg->addr);
 659                 break;
 660         case KVM_REG_S390_EPOCHDIFF:
 661                 r = get_user(vcpu->arch.sie_block->epoch,
 662                              (u64 __user *)reg->addr);
 663                 break;
 664         case KVM_REG_S390_CPU_TIMER:
 665                 r = get_user(vcpu->arch.sie_block->cputm,
 666                              (u64 __user *)reg->addr);
 667                 break;
 668         case KVM_REG_S390_CLOCK_COMP:
 669                 r = get_user(vcpu->arch.sie_block->ckc,
 670                              (u64 __user *)reg->addr);
 671                 break;
 672         case KVM_REG_S390_PFTOKEN:
 673                 r = get_user(vcpu->arch.pfault_token,
 674                              (u64 __user *)reg->addr);
 675                 break;
 676         case KVM_REG_S390_PFCOMPARE:
 677                 r = get_user(vcpu->arch.pfault_compare,
 678                              (u64 __user *)reg->addr);
 679                 break;
 680         case KVM_REG_S390_PFSELECT:
 681                 r = get_user(vcpu->arch.pfault_select,
 682                              (u64 __user *)reg->addr);
 683                 break;
 684         case KVM_REG_S390_PP:
 685                 r = get_user(vcpu->arch.sie_block->pp,
 686                              (u64 __user *)reg->addr);
 687                 break;
 688         case KVM_REG_S390_GBEA:
 689                 r = get_user(vcpu->arch.sie_block->gbea,
 690                              (u64 __user *)reg->addr);
 691                 break;
 692         default:
 693                 break;
 694         }
 695
 696         return r;
 697 }
 698
 699 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
 700 {
 701         kvm_s390_vcpu_initial_reset(vcpu);
 702         return 0;
 703 }
 704
 705 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 706 {
 707         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
 708         return 0;
 709 }
 710
 711 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 712 {
 713         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
 714         return 0;
 715 }
 716
 717 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 718                                   struct kvm_sregs *sregs)
 719 {
 720         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
 721         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
 722         restore_access_regs(vcpu->run->s.regs.acrs);
 723         return 0;
 724 }
 725
 726 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 727                                   struct kvm_sregs *sregs)
 728 {
 729         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
 730         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
 731         return 0;
 732 }
 733
 734 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 735 {
 736         if (test_fp_ctl(fpu->fpc))
 737                 return -EINVAL;
 738         memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
 739         vcpu->arch.guest_fpregs.fpc = fpu->fpc;
 740         restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
 741         restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
 742         return 0;
 743 }
 744
 745 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 746 {
 747         memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
 748         fpu->fpc = vcpu->arch.guest_fpregs.fpc;
 749         return 0;
 750 }
 751
 752 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
 753 {
 754         int rc = 0;
 755
 756         if (!(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_STOPPED))
 757                 rc = -EBUSY;
 758         else {
 759                 vcpu->run->psw_mask = psw.mask;
 760                 vcpu->run->psw_addr = psw.addr;
 761         }
 762         return rc;
 763 }
 764
 765 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 766                                   struct kvm_translation *tr)
 767 {
 768         return -EINVAL; /* not implemented yet */
 769 }
 770
 771 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 772                                         struct kvm_guest_debug *dbg)
 773 {
 774         return -EINVAL; /* not implemented yet */
 775 }
 776
 777 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 778                                     struct kvm_mp_state *mp_state)
 779 {
 780         return -EINVAL; /* not implemented yet */
 781 }
 782
 783 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 784                                     struct kvm_mp_state *mp_state)
 785 {
 786         return -EINVAL; /* not implemented yet */
 787 }
 788
 789 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
 790 {
 791         /*
 792          * We use MMU_RELOAD just to re-arm the ipte notifier for the
 793          * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
 794          * This ensures that the ipte instruction for this request has
 795          * already finished. We might race against a second unmapper that
 796          * wants to set the blocking bit. Lets just retry the request loop.
 797          */
 798         while (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
 799                 int rc;
 800                 rc = gmap_ipte_notify(vcpu->arch.gmap,
 801                                       vcpu->arch.sie_block->prefix,
 802                                       PAGE_SIZE * 2);
 803                 if (rc)
 804                         return rc;
 805                 s390_vcpu_unblock(vcpu);
 806         }
 807         return 0;
 808 }
 809
 810 static long kvm_arch_fault_in_sync(struct kvm_vcpu *vcpu)
 811 {
 812         long rc;
 813         hva_t fault = gmap_fault(current->thread.gmap_addr, vcpu->arch.gmap);
 814         struct mm_struct *mm = current->mm;
 815         down_read(&mm->mmap_sem);
 816         rc = get_user_pages(current, mm, fault, 1, 1, 0, NULL, NULL);
 817         up_read(&mm->mmap_sem);
 818         return rc;
 819 }
 820
 821 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
 822                                       unsigned long token)
 823 {
 824         struct kvm_s390_interrupt inti;
 825         inti.parm64 = token;
 826
 827         if (start_token) {
 828                 inti.type = KVM_S390_INT_PFAULT_INIT;
 829                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &inti));
 830         } else {
 831                 inti.type = KVM_S390_INT_PFAULT_DONE;
 832                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
 833         }
 834 }
 835
 836 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
 837                                      struct kvm_async_pf *work)
 838 {
 839         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
 840         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
 841 }
 842
 843 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
 844                                  struct kvm_async_pf *work)
 845 {
 846         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
 847         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
 848 }
 849
 850 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
 851                                struct kvm_async_pf *work)
 852 {
 853         /* s390 will always inject the page directly */
 854 }
 855
 856 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
 857 {
 858         /*
 859          * s390 will always inject the page directly,
 860          * but we still want check_async_completion to cleanup
 861          */
 862         return true;
 863 }
 864
 865 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
 866 {
 867         hva_t hva;
 868         struct kvm_arch_async_pf arch;
 869         int rc;
 870
 871         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
 872                 return 0;
 873         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
 874             vcpu->arch.pfault_compare)
 875                 return 0;
 876         if (psw_extint_disabled(vcpu))
 877                 return 0;
 878         if (kvm_cpu_has_interrupt(vcpu))
 879                 return 0;
 880         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
 881                 return 0;
 882         if (!vcpu->arch.gmap->pfault_enabled)
 883                 return 0;
 884
 885         hva = gmap_fault(current->thread.gmap_addr, vcpu->arch.gmap);
 886         if (copy_from_guest(vcpu, &arch.pfault_token, vcpu->arch.pfault_token, 8))
 887                 return 0;
 888
 889         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
 890         return rc;
 891 }
 892
 893 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
 894 {
 895         int rc, cpuflags;
 896
 897         /*
 898          * On s390 notifications for arriving pages will be delivered directly
 899          * to the guest but the house keeping for completed pfaults is
 900          * handled outside the worker.
 901          */
 902         kvm_check_async_pf_completion(vcpu);
 903
 904         memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
 905
 906         if (need_resched())
 907                 schedule();
 908
 909         if (test_thread_flag(TIF_MCCK_PENDING))
 910                 s390_handle_mcck();
 911
 912         if (!kvm_is_ucontrol(vcpu->kvm))
 913                 kvm_s390_deliver_pending_interrupts(vcpu);
 914
 915         rc = kvm_s390_handle_requests(vcpu);
 916         if (rc)
 917                 return rc;
 918
 919         vcpu->arch.sie_block->icptcode = 0;
 920         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
 921         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
 922         trace_kvm_s390_sie_enter(vcpu, cpuflags);
 923
 924         return 0;
 925 }
 926
 927 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
 928 {
 929         int rc = -1;
 930
 931         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
 932                    vcpu->arch.sie_block->icptcode);
 933         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
 934
 935         if (exit_reason >= 0) {
 936                 rc = 0;
 937         } else if (kvm_is_ucontrol(vcpu->kvm)) {
 938                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
 939                 vcpu->run->s390_ucontrol.trans_exc_code =
 940                                                 current->thread.gmap_addr;
 941                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
 942                 rc = -EREMOTE;
 943
 944         } else if (current->thread.gmap_pfault) {
 945                 trace_kvm_s390_major_guest_pfault(vcpu);
 946                 current->thread.gmap_pfault = 0;
 947                 if (kvm_arch_setup_async_pf(vcpu) ||
 948                     (kvm_arch_fault_in_sync(vcpu) >= 0))
 949                         rc = 0;
 950         }
 951
 952         if (rc == -1) {
 953                 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
 954                 trace_kvm_s390_sie_fault(vcpu);
 955                 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
 956         }
 957
 958         memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
 959
 960         if (rc == 0) {
 961                 if (kvm_is_ucontrol(vcpu->kvm))
 962                         /* Don't exit for host interrupts. */
 963                         rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
 964                 else
 965                         rc = kvm_handle_sie_intercept(vcpu);
 966         }
 967
 968         return rc;
 969 }
 970
 971 bool kvm_enabled_cmma(void)
 972 {
 973         if (!MACHINE_IS_LPAR)
 974                 return false;
 975         /* only enable for z10 and later */
 976         if (!MACHINE_HAS_EDAT1)
 977                 return false;
 978         return true;
 979 }
 980
 981 static int __vcpu_run(struct kvm_vcpu *vcpu)
 982 {
 983         int rc, exit_reason;
 984
 985         /*
 986          * We try to hold kvm->srcu during most of vcpu_run (except when run-
 987          * ning the guest), so that memslots (and other stuff) are protected
 988          */
 989         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
 990
 991         do {
 992                 rc = vcpu_pre_run(vcpu);
 993                 if (rc)
 994                         break;
 995
 996                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
 997                 /*
 998                  * As PF_VCPU will be used in fault handler, between
 999                  * guest_enter and guest_exit should be no uaccess.
1000                  */
1001                 preempt_disable();
1002                 kvm_guest_enter();
1003                 preempt_enable();
1004                 exit_reason = sie64a(vcpu->arch.sie_block,
1005                                      vcpu->run->s.regs.gprs);
1006                 kvm_guest_exit();
1007                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1008
1009                 rc = vcpu_post_run(vcpu, exit_reason);
1010         } while (!signal_pending(current) && !rc);
1011
1012         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1013         return rc;
1014 }
1015
1016 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1017 {
1018         int rc;
1019         sigset_t sigsaved;
1020
1021         if (vcpu->sigset_active)
1022                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1023
1024         atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1025
1026         switch (kvm_run->exit_reason) {
1027         case KVM_EXIT_S390_SIEIC:
1028         case KVM_EXIT_UNKNOWN:
1029         case KVM_EXIT_INTR:
1030         case KVM_EXIT_S390_RESET:
1031         case KVM_EXIT_S390_UCONTROL:
1032         case KVM_EXIT_S390_TSCH:
1033                 break;
1034         default:
1035                 BUG();
1036         }
1037
1038         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1039         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1040         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) {
1041                 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_PREFIX;
1042                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1043         }
1044         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1045                 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_CRS;
1046                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
1047                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1048         }
1049
1050         might_fault();
1051         rc = __vcpu_run(vcpu);
1052
1053         if (signal_pending(current) && !rc) {
1054                 kvm_run->exit_reason = KVM_EXIT_INTR;
1055                 rc = -EINTR;
1056         }
1057
1058         if (rc == -EOPNOTSUPP) {
1059                 /* intercept cannot be handled in-kernel, prepare kvm-run */
1060                 kvm_run->exit_reason         = KVM_EXIT_S390_SIEIC;
1061                 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
1062                 kvm_run->s390_sieic.ipa      = vcpu->arch.sie_block->ipa;
1063                 kvm_run->s390_sieic.ipb      = vcpu->arch.sie_block->ipb;
1064                 rc = 0;
1065         }
1066
1067         if (rc == -EREMOTE) {
1068                 /* intercept was handled, but userspace support is needed
1069                  * kvm_run has been prepared by the handler */
1070                 rc = 0;
1071         }
1072
1073         kvm_run->psw_mask     = vcpu->arch.sie_block->gpsw.mask;
1074         kvm_run->psw_addr     = vcpu->arch.sie_block->gpsw.addr;
1075         kvm_run->s.regs.prefix = vcpu->arch.sie_block->prefix;
1076         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1077
1078         if (vcpu->sigset_active)
1079                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1080
1081         vcpu->stat.exit_userspace++;
1082         return rc;
1083 }
1084
1085 static int __guestcopy(struct kvm_vcpu *vcpu, u64 guestdest, void *from,
1086                        unsigned long n, int prefix)
1087 {
1088         if (prefix)
1089                 return copy_to_guest(vcpu, guestdest, from, n);
1090         else
1091                 return copy_to_guest_absolute(vcpu, guestdest, from, n);
1092 }
1093
1094 /*
1095  * store status at address
1096  * we use have two special cases:
1097  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1098  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1099  */
1100 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long addr)
1101 {
1102         unsigned char archmode = 1;
1103         int prefix;
1104         u64 clkcomp;
1105
1106         if (addr == KVM_S390_STORE_STATUS_NOADDR) {
1107                 if (copy_to_guest_absolute(vcpu, 163ul, &archmode, 1))
1108                         return -EFAULT;
1109                 addr = SAVE_AREA_BASE;
1110                 prefix = 0;
1111         } else if (addr == KVM_S390_STORE_STATUS_PREFIXED) {
1112                 if (copy_to_guest(vcpu, 163ul, &archmode, 1))
1113                         return -EFAULT;
1114                 addr = SAVE_AREA_BASE;
1115                 prefix = 1;
1116         } else
1117                 prefix = 0;
1118
1119         if (__guestcopy(vcpu, addr + offsetof(struct save_area, fp_regs),
1120                         vcpu->arch.guest_fpregs.fprs, 128, prefix))
1121                 return -EFAULT;
1122
1123         if (__guestcopy(vcpu, addr + offsetof(struct save_area, gp_regs),
1124                         vcpu->run->s.regs.gprs, 128, prefix))
1125                 return -EFAULT;
1126
1127         if (__guestcopy(vcpu, addr + offsetof(struct save_area, psw),
1128                         &vcpu->arch.sie_block->gpsw, 16, prefix))
1129                 return -EFAULT;
1130
1131         if (__guestcopy(vcpu, addr + offsetof(struct save_area, pref_reg),
1132                         &vcpu->arch.sie_block->prefix, 4, prefix))
1133                 return -EFAULT;
1134
1135         if (__guestcopy(vcpu,
1136                         addr + offsetof(struct save_area, fp_ctrl_reg),
1137                         &vcpu->arch.guest_fpregs.fpc, 4, prefix))
1138                 return -EFAULT;
1139
1140         if (__guestcopy(vcpu, addr + offsetof(struct save_area, tod_reg),
1141                         &vcpu->arch.sie_block->todpr, 4, prefix))
1142                 return -EFAULT;
1143
1144         if (__guestcopy(vcpu, addr + offsetof(struct save_area, timer),
1145                         &vcpu->arch.sie_block->cputm, 8, prefix))
1146                 return -EFAULT;
1147
1148         clkcomp = vcpu->arch.sie_block->ckc >> 8;
1149         if (__guestcopy(vcpu, addr + offsetof(struct save_area, clk_cmp),
1150                         &clkcomp, 8, prefix))
1151                 return -EFAULT;
1152
1153         if (__guestcopy(vcpu, addr + offsetof(struct save_area, acc_regs),
1154                         &vcpu->run->s.regs.acrs, 64, prefix))
1155                 return -EFAULT;
1156
1157         if (__guestcopy(vcpu,
1158                         addr + offsetof(struct save_area, ctrl_regs),
1159                         &vcpu->arch.sie_block->gcr, 128, prefix))
1160                 return -EFAULT;
1161         return 0;
1162 }
1163
1164 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1165 {
1166         /*
1167          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1168          * copying in vcpu load/put. Lets update our copies before we save
1169          * it into the save area
1170          */
1171         save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1172         save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1173         save_access_regs(vcpu->run->s.regs.acrs);
1174
1175         return kvm_s390_store_status_unloaded(vcpu, addr);
1176 }
1177
1178 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1179                                      struct kvm_enable_cap *cap)
1180 {
1181         int r;
1182
1183         if (cap->flags)
1184                 return -EINVAL;
1185
1186         switch (cap->cap) {
1187         case KVM_CAP_S390_CSS_SUPPORT:
1188                 if (!vcpu->kvm->arch.css_support) {
1189                         vcpu->kvm->arch.css_support = 1;
1190                         trace_kvm_s390_enable_css(vcpu->kvm);
1191                 }
1192                 r = 0;
1193                 break;
1194         default:
1195                 r = -EINVAL;
1196                 break;
1197         }
1198         return r;
1199 }
1200
1201 long kvm_arch_vcpu_ioctl(struct file *filp,
1202                          unsigned int ioctl, unsigned long arg)
1203 {
1204         struct kvm_vcpu *vcpu = filp->private_data;
1205         void __user *argp = (void __user *)arg;
1206         int idx;
1207         long r;
1208
1209         switch (ioctl) {
1210         case KVM_S390_INTERRUPT: {
1211                 struct kvm_s390_interrupt s390int;
1212
1213                 r = -EFAULT;
1214                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1215                         break;
1216                 r = kvm_s390_inject_vcpu(vcpu, &s390int);
1217                 break;
1218         }
1219         case KVM_S390_STORE_STATUS:
1220                 idx = srcu_read_lock(&vcpu->kvm->srcu);
1221                 r = kvm_s390_vcpu_store_status(vcpu, arg);
1222                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
1223                 break;
1224         case KVM_S390_SET_INITIAL_PSW: {
1225                 psw_t psw;
1226
1227                 r = -EFAULT;
1228                 if (copy_from_user(&psw, argp, sizeof(psw)))
1229                         break;
1230                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1231                 break;
1232         }
1233         case KVM_S390_INITIAL_RESET:
1234                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1235                 break;
1236         case KVM_SET_ONE_REG:
1237         case KVM_GET_ONE_REG: {
1238                 struct kvm_one_reg reg;
1239                 r = -EFAULT;
1240                 if (copy_from_user(&reg, argp, sizeof(reg)))
1241                         break;
1242                 if (ioctl == KVM_SET_ONE_REG)
1243                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
1244                 else
1245                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
1246                 break;
1247         }
1248 #ifdef CONFIG_KVM_S390_UCONTROL
1249         case KVM_S390_UCAS_MAP: {
1250                 struct kvm_s390_ucas_mapping ucasmap;
1251
1252                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1253                         r = -EFAULT;
1254                         break;
1255                 }
1256
1257                 if (!kvm_is_ucontrol(vcpu->kvm)) {
1258                         r = -EINVAL;
1259                         break;
1260                 }
1261
1262                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1263                                      ucasmap.vcpu_addr, ucasmap.length);
1264                 break;
1265         }
1266         case KVM_S390_UCAS_UNMAP: {
1267                 struct kvm_s390_ucas_mapping ucasmap;
1268
1269                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1270                         r = -EFAULT;
1271                         break;
1272                 }
1273
1274                 if (!kvm_is_ucontrol(vcpu->kvm)) {
1275                         r = -EINVAL;
1276                         break;
1277                 }
1278
1279                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1280                         ucasmap.length);
1281                 break;
1282         }
1283 #endif
1284         case KVM_S390_VCPU_FAULT: {
1285                 r = gmap_fault(arg, vcpu->arch.gmap);
1286                 if (!IS_ERR_VALUE(r))
1287                         r = 0;
1288                 break;
1289         }
1290         case KVM_ENABLE_CAP:
1291         {
1292                 struct kvm_enable_cap cap;
1293                 r = -EFAULT;
1294                 if (copy_from_user(&cap, argp, sizeof(cap)))
1295                         break;
1296                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1297                 break;
1298         }
1299         default:
1300                 r = -ENOTTY;
1301         }
1302         return r;
1303 }
1304
1305 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1306 {
1307 #ifdef CONFIG_KVM_S390_UCONTROL
1308         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1309                  && (kvm_is_ucontrol(vcpu->kvm))) {
1310                 vmf->page = virt_to_page(vcpu->arch.sie_block);
1311                 get_page(vmf->page);
1312                 return 0;
1313         }
1314 #endif
1315         return VM_FAULT_SIGBUS;
1316 }
1317
1318 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
1319                            struct kvm_memory_slot *dont)
1320 {
1321 }
1322
1323 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
1324                             unsigned long npages)
1325 {
1326         return 0;
1327 }
1328
1329 void kvm_arch_memslots_updated(struct kvm *kvm)
1330 {
1331 }
1332
1333 /* Section: memory related */
1334 int kvm_arch_prepare_memory_region(struct kvm *kvm,
1335                                    struct kvm_memory_slot *memslot,
1336                                    struct kvm_userspace_memory_region *mem,
1337                                    enum kvm_mr_change change)
1338 {
1339         /* A few sanity checks. We can have memory slots which have to be
1340            located/ended at a segment boundary (1MB). The memory in userland is
1341            ok to be fragmented into various different vmas. It is okay to mmap()
1342            and munmap() stuff in this slot after doing this call at any time */
1343
1344         if (mem->userspace_addr & 0xffffful)
1345                 return -EINVAL;
1346
1347         if (mem->memory_size & 0xffffful)
1348                 return -EINVAL;
1349
1350         return 0;
1351 }
1352
1353 void kvm_arch_commit_memory_region(struct kvm *kvm,
1354                                 struct kvm_userspace_memory_region *mem,
1355                                 const struct kvm_memory_slot *old,
1356                                 enum kvm_mr_change change)
1357 {
1358         int rc;
1359
1360         /* If the basics of the memslot do not change, we do not want
1361          * to update the gmap. Every update causes several unnecessary
1362          * segment translation exceptions. This is usually handled just
1363          * fine by the normal fault handler + gmap, but it will also
1364          * cause faults on the prefix page of running guest CPUs.
1365          */
1366         if (old->userspace_addr == mem->userspace_addr &&
1367             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
1368             old->npages * PAGE_SIZE == mem->memory_size)
1369                 return;
1370
1371         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
1372                 mem->guest_phys_addr, mem->memory_size);
1373         if (rc)
1374                 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
1375         return;
1376 }
1377
1378 void kvm_arch_flush_shadow_all(struct kvm *kvm)
1379 {
1380 }
1381
1382 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
1383                                    struct kvm_memory_slot *slot)
1384 {
1385 }
1386
1387 static int __init kvm_s390_init(void)
1388 {
1389         int ret;
1390         ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1391         if (ret)
1392                 return ret;
1393
1394         /*
1395          * guests can ask for up to 255+1 double words, we need a full page
1396          * to hold the maximum amount of facilities. On the other hand, we
1397          * only set facilities that are known to work in KVM.
1398          */
1399         vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
1400         if (!vfacilities) {
1401                 kvm_exit();
1402                 return -ENOMEM;
1403         }
1404         memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
1405         vfacilities[0] &= 0xff82fff3f4fc2000UL;
1406         vfacilities[1] &= 0x005c000000000000UL;
1407         return 0;
1408 }
1409
1410 static void __exit kvm_s390_exit(void)
1411 {
1412         free_page((unsigned long) vfacilities);
1413         kvm_exit();
1414 }
1415
1416 module_init(kvm_s390_init);
1417 module_exit(kvm_s390_exit);
1418
1419 /*
1420  * Enable autoloading of the kvm module.
1421  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1422  * since x86 takes a different approach.
1423  */
1424 #include <linux/miscdevice.h>
1425 MODULE_ALIAS_MISCDEV(KVM_MINOR);
1426 MODULE_ALIAS("devname:kvm");