arch/s390/kvm/interrupt.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * handling kvm guest interrupts
   4  *
   5  * Copyright IBM Corp. 2008, 2020
   6  *
   7  *    Author(s): Carsten Otte <[email protected]>
   8  */
   9
  10 #define KMSG_COMPONENT "kvm-s390"
  11 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  12
  13 #include <linux/interrupt.h>
  14 #include <linux/kvm_host.h>
  15 #include <linux/hrtimer.h>
  16 #include <linux/mmu_context.h>
  17 #include <linux/nospec.h>
  18 #include <linux/signal.h>
  19 #include <linux/slab.h>
  20 #include <linux/bitmap.h>
  21 #include <linux/vmalloc.h>
  22 #include <asm/access-regs.h>
  23 #include <asm/asm-offsets.h>
  24 #include <asm/dis.h>
  25 #include <linux/uaccess.h>
  26 #include <asm/sclp.h>
  27 #include <asm/isc.h>
  28 #include <asm/gmap.h>
  29 #include <asm/nmi.h>
  30 #include <asm/airq.h>
  31 #include <asm/tpi.h>
  32 #include "kvm-s390.h"
  33 #include "gaccess.h"
  34 #include "trace-s390.h"
  35 #include "pci.h"
  36
  37 #define PFAULT_INIT 0x0600
  38 #define PFAULT_DONE 0x0680
  39 #define VIRTIO_PARAM 0x0d00
  40
  41 static struct kvm_s390_gib *gib;
  42
  43 /* handle external calls via sigp interpretation facility */
  44 static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id)
  45 {
  46         int c, scn;
  47
  48         if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND))
  49                 return 0;
  50
  51         BUG_ON(!kvm_s390_use_sca_entries());
  52         read_lock(&vcpu->kvm->arch.sca_lock);
  53         if (vcpu->kvm->arch.use_esca) {
  54                 struct esca_block *sca = vcpu->kvm->arch.sca;
  55                 union esca_sigp_ctrl sigp_ctrl =
  56                         sca->cpu[vcpu->vcpu_id].sigp_ctrl;
  57
  58                 c = sigp_ctrl.c;
  59                 scn = sigp_ctrl.scn;
  60         } else {
  61                 struct bsca_block *sca = vcpu->kvm->arch.sca;
  62                 union bsca_sigp_ctrl sigp_ctrl =
  63                         sca->cpu[vcpu->vcpu_id].sigp_ctrl;
  64
  65                 c = sigp_ctrl.c;
  66                 scn = sigp_ctrl.scn;
  67         }
  68         read_unlock(&vcpu->kvm->arch.sca_lock);
  69
  70         if (src_id)
  71                 *src_id = scn;
  72
  73         return c;
  74 }
  75
  76 static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
  77 {
  78         int expect, rc;
  79
  80         BUG_ON(!kvm_s390_use_sca_entries());
  81         read_lock(&vcpu->kvm->arch.sca_lock);
  82         if (vcpu->kvm->arch.use_esca) {
  83                 struct esca_block *sca = vcpu->kvm->arch.sca;
  84                 union esca_sigp_ctrl *sigp_ctrl =
  85                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
  86                 union esca_sigp_ctrl new_val = {0}, old_val;
  87
  88                 old_val = READ_ONCE(*sigp_ctrl);
  89                 new_val.scn = src_id;
  90                 new_val.c = 1;
  91                 old_val.c = 0;
  92
  93                 expect = old_val.value;
  94                 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
  95         } else {
  96                 struct bsca_block *sca = vcpu->kvm->arch.sca;
  97                 union bsca_sigp_ctrl *sigp_ctrl =
  98                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
  99                 union bsca_sigp_ctrl new_val = {0}, old_val;
 100
 101                 old_val = READ_ONCE(*sigp_ctrl);
 102                 new_val.scn = src_id;
 103                 new_val.c = 1;
 104                 old_val.c = 0;
 105
 106                 expect = old_val.value;
 107                 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
 108         }
 109         read_unlock(&vcpu->kvm->arch.sca_lock);
 110
 111         if (rc != expect) {
 112                 /* another external call is pending */
 113                 return -EBUSY;
 114         }
 115         kvm_s390_set_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
 116         return 0;
 117 }
 118
 119 static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
 120 {
 121         if (!kvm_s390_use_sca_entries())
 122                 return;
 123         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
 124         read_lock(&vcpu->kvm->arch.sca_lock);
 125         if (vcpu->kvm->arch.use_esca) {
 126                 struct esca_block *sca = vcpu->kvm->arch.sca;
 127                 union esca_sigp_ctrl *sigp_ctrl =
 128                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
 129
 130                 WRITE_ONCE(sigp_ctrl->value, 0);
 131         } else {
 132                 struct bsca_block *sca = vcpu->kvm->arch.sca;
 133                 union bsca_sigp_ctrl *sigp_ctrl =
 134                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
 135
 136                 WRITE_ONCE(sigp_ctrl->value, 0);
 137         }
 138         read_unlock(&vcpu->kvm->arch.sca_lock);
 139 }
 140
 141 int psw_extint_disabled(struct kvm_vcpu *vcpu)
 142 {
 143         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
 144 }
 145
 146 static int psw_ioint_disabled(struct kvm_vcpu *vcpu)
 147 {
 148         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO);
 149 }
 150
 151 static int psw_mchk_disabled(struct kvm_vcpu *vcpu)
 152 {
 153         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK);
 154 }
 155
 156 static int psw_interrupts_disabled(struct kvm_vcpu *vcpu)
 157 {
 158         return psw_extint_disabled(vcpu) &&
 159                psw_ioint_disabled(vcpu) &&
 160                psw_mchk_disabled(vcpu);
 161 }
 162
 163 static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
 164 {
 165         if (psw_extint_disabled(vcpu) ||
 166             !(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
 167                 return 0;
 168         if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu))
 169                 /* No timer interrupts when single stepping */
 170                 return 0;
 171         return 1;
 172 }
 173
 174 static int ckc_irq_pending(struct kvm_vcpu *vcpu)
 175 {
 176         const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
 177         const u64 ckc = vcpu->arch.sie_block->ckc;
 178
 179         if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
 180                 if ((s64)ckc >= (s64)now)
 181                         return 0;
 182         } else if (ckc >= now) {
 183                 return 0;
 184         }
 185         return ckc_interrupts_enabled(vcpu);
 186 }
 187
 188 static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu)
 189 {
 190         return !psw_extint_disabled(vcpu) &&
 191                (vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK);
 192 }
 193
 194 static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
 195 {
 196         if (!cpu_timer_interrupts_enabled(vcpu))
 197                 return 0;
 198         return kvm_s390_get_cpu_timer(vcpu) >> 63;
 199 }
 200
 201 static uint64_t isc_to_isc_bits(int isc)
 202 {
 203         return (0x80 >> isc) << 24;
 204 }
 205
 206 static inline u32 isc_to_int_word(u8 isc)
 207 {
 208         return ((u32)isc << 27) | 0x80000000;
 209 }
 210
 211 static inline u8 int_word_to_isc(u32 int_word)
 212 {
 213         return (int_word & 0x38000000) >> 27;
 214 }
 215
 216 /*
 217  * To use atomic bitmap functions, we have to provide a bitmap address
 218  * that is u64 aligned. However, the ipm might be u32 aligned.
 219  * Therefore, we logically start the bitmap at the very beginning of the
 220  * struct and fixup the bit number.
 221  */
 222 #define IPM_BIT_OFFSET (offsetof(struct kvm_s390_gisa, ipm) * BITS_PER_BYTE)
 223
 224 /**
 225  * gisa_set_iam - change the GISA interruption alert mask
 226  *
 227  * @gisa: gisa to operate on
 228  * @iam: new IAM value to use
 229  *
 230  * Change the IAM atomically with the next alert address and the IPM
 231  * of the GISA if the GISA is not part of the GIB alert list. All three
 232  * fields are located in the first long word of the GISA.
 233  *
 234  * Returns: 0 on success
 235  *          -EBUSY in case the gisa is part of the alert list
 236  */
 237 static inline int gisa_set_iam(struct kvm_s390_gisa *gisa, u8 iam)
 238 {
 239         u64 word, _word;
 240
 241         word = READ_ONCE(gisa->u64.word[0]);
 242         do {
 243                 if ((u64)gisa != word >> 32)
 244                         return -EBUSY;
 245                 _word = (word & ~0xffUL) | iam;
 246         } while (!try_cmpxchg(&gisa->u64.word[0], &word, _word));
 247
 248         return 0;
 249 }
 250
 251 /**
 252  * gisa_clear_ipm - clear the GISA interruption pending mask
 253  *
 254  * @gisa: gisa to operate on
 255  *
 256  * Clear the IPM atomically with the next alert address and the IAM
 257  * of the GISA unconditionally. All three fields are located in the
 258  * first long word of the GISA.
 259  */
 260 static inline void gisa_clear_ipm(struct kvm_s390_gisa *gisa)
 261 {
 262         u64 word, _word;
 263
 264         word = READ_ONCE(gisa->u64.word[0]);
 265         do {
 266                 _word = word & ~(0xffUL << 24);
 267         } while (!try_cmpxchg(&gisa->u64.word[0], &word, _word));
 268 }
 269
 270 /**
 271  * gisa_get_ipm_or_restore_iam - return IPM or restore GISA IAM
 272  *
 273  * @gi: gisa interrupt struct to work on
 274  *
 275  * Atomically restores the interruption alert mask if none of the
 276  * relevant ISCs are pending and return the IPM.
 277  *
 278  * Returns: the relevant pending ISCs
 279  */
 280 static inline u8 gisa_get_ipm_or_restore_iam(struct kvm_s390_gisa_interrupt *gi)
 281 {
 282         u8 pending_mask, alert_mask;
 283         u64 word, _word;
 284
 285         word = READ_ONCE(gi->origin->u64.word[0]);
 286         do {
 287                 alert_mask = READ_ONCE(gi->alert.mask);
 288                 pending_mask = (u8)(word >> 24) & alert_mask;
 289                 if (pending_mask)
 290                         return pending_mask;
 291                 _word = (word & ~0xffUL) | alert_mask;
 292         } while (!try_cmpxchg(&gi->origin->u64.word[0], &word, _word));
 293
 294         return 0;
 295 }
 296
 297 static inline void gisa_set_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
 298 {
 299         set_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
 300 }
 301
 302 static inline u8 gisa_get_ipm(struct kvm_s390_gisa *gisa)
 303 {
 304         return READ_ONCE(gisa->ipm);
 305 }
 306
 307 static inline int gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
 308 {
 309         return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
 310 }
 311
 312 static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu)
 313 {
 314         unsigned long pending = vcpu->kvm->arch.float_int.pending_irqs |
 315                                 vcpu->arch.local_int.pending_irqs;
 316
 317         pending &= ~vcpu->kvm->arch.float_int.masked_irqs;
 318         return pending;
 319 }
 320
 321 static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu)
 322 {
 323         struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
 324         unsigned long pending_mask;
 325
 326         pending_mask = pending_irqs_no_gisa(vcpu);
 327         if (gi->origin)
 328                 pending_mask |= gisa_get_ipm(gi->origin) << IRQ_PEND_IO_ISC_7;
 329         return pending_mask;
 330 }
 331
 332 static inline int isc_to_irq_type(unsigned long isc)
 333 {
 334         return IRQ_PEND_IO_ISC_0 - isc;
 335 }
 336
 337 static inline int irq_type_to_isc(unsigned long irq_type)
 338 {
 339         return IRQ_PEND_IO_ISC_0 - irq_type;
 340 }
 341
 342 static unsigned long disable_iscs(struct kvm_vcpu *vcpu,
 343                                    unsigned long active_mask)
 344 {
 345         int i;
 346
 347         for (i = 0; i <= MAX_ISC; i++)
 348                 if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i)))
 349                         active_mask &= ~(1UL << (isc_to_irq_type(i)));
 350
 351         return active_mask;
 352 }
 353
 354 static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
 355 {
 356         unsigned long active_mask;
 357
 358         active_mask = pending_irqs(vcpu);
 359         if (!active_mask)
 360                 return 0;
 361
 362         if (psw_extint_disabled(vcpu))
 363                 active_mask &= ~IRQ_PEND_EXT_MASK;
 364         if (psw_ioint_disabled(vcpu))
 365                 active_mask &= ~IRQ_PEND_IO_MASK;
 366         else
 367                 active_mask = disable_iscs(vcpu, active_mask);
 368         if (!(vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
 369                 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask);
 370         if (!(vcpu->arch.sie_block->gcr[0] & CR0_EMERGENCY_SIGNAL_SUBMASK))
 371                 __clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask);
 372         if (!(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
 373                 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask);
 374         if (!(vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK))
 375                 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask);
 376         if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK)) {
 377                 __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask);
 378                 __clear_bit(IRQ_PEND_EXT_SERVICE_EV, &active_mask);
 379         }
 380         if (psw_mchk_disabled(vcpu))
 381                 active_mask &= ~IRQ_PEND_MCHK_MASK;
 382         /* PV guest cpus can have a single interruption injected at a time. */
 383         if (kvm_s390_pv_cpu_get_handle(vcpu) &&
 384             vcpu->arch.sie_block->iictl != IICTL_CODE_NONE)
 385                 active_mask &= ~(IRQ_PEND_EXT_II_MASK |
 386                                  IRQ_PEND_IO_MASK |
 387                                  IRQ_PEND_MCHK_MASK);
 388         /*
 389          * Check both floating and local interrupt's cr14 because
 390          * bit IRQ_PEND_MCHK_REP could be set in both cases.
 391          */
 392         if (!(vcpu->arch.sie_block->gcr[14] &
 393            (vcpu->kvm->arch.float_int.mchk.cr14 |
 394            vcpu->arch.local_int.irq.mchk.cr14)))
 395                 __clear_bit(IRQ_PEND_MCHK_REP, &active_mask);
 396
 397         /*
 398          * STOP irqs will never be actively delivered. They are triggered via
 399          * intercept requests and cleared when the stop intercept is performed.
 400          */
 401         __clear_bit(IRQ_PEND_SIGP_STOP, &active_mask);
 402
 403         return active_mask;
 404 }
 405
 406 static void __set_cpu_idle(struct kvm_vcpu *vcpu)
 407 {
 408         kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT);
 409         set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
 410 }
 411
 412 static void __unset_cpu_idle(struct kvm_vcpu *vcpu)
 413 {
 414         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT);
 415         clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
 416 }
 417
 418 static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
 419 {
 420         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IO_INT | CPUSTAT_EXT_INT |
 421                                       CPUSTAT_STOP_INT);
 422         vcpu->arch.sie_block->lctl = 0x0000;
 423         vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT);
 424
 425         if (guestdbg_enabled(vcpu)) {
 426                 vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 |
 427                                                LCTL_CR10 | LCTL_CR11);
 428                 vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT);
 429         }
 430 }
 431
 432 static void set_intercept_indicators_io(struct kvm_vcpu *vcpu)
 433 {
 434         if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_IO_MASK))
 435                 return;
 436         if (psw_ioint_disabled(vcpu))
 437                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT);
 438         else
 439                 vcpu->arch.sie_block->lctl |= LCTL_CR6;
 440 }
 441
 442 static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu)
 443 {
 444         if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_EXT_MASK))
 445                 return;
 446         if (psw_extint_disabled(vcpu))
 447                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
 448         else
 449                 vcpu->arch.sie_block->lctl |= LCTL_CR0;
 450 }
 451
 452 static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu)
 453 {
 454         if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_MCHK_MASK))
 455                 return;
 456         if (psw_mchk_disabled(vcpu))
 457                 vcpu->arch.sie_block->ictl |= ICTL_LPSW;
 458         else
 459                 vcpu->arch.sie_block->lctl |= LCTL_CR14;
 460 }
 461
 462 static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu)
 463 {
 464         if (kvm_s390_is_stop_irq_pending(vcpu))
 465                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
 466 }
 467
 468 /* Set interception request for non-deliverable interrupts */
 469 static void set_intercept_indicators(struct kvm_vcpu *vcpu)
 470 {
 471         set_intercept_indicators_io(vcpu);
 472         set_intercept_indicators_ext(vcpu);
 473         set_intercept_indicators_mchk(vcpu);
 474         set_intercept_indicators_stop(vcpu);
 475 }
 476
 477 static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu)
 478 {
 479         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 480         int rc = 0;
 481
 482         vcpu->stat.deliver_cputm++;
 483         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
 484                                          0, 0);
 485         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 486                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 487                 vcpu->arch.sie_block->eic = EXT_IRQ_CPU_TIMER;
 488         } else {
 489                 rc  = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER,
 490                                    (u16 *)__LC_EXT_INT_CODE);
 491                 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 492                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 493                                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 494                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 495                                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 496         }
 497         clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
 498         return rc ? -EFAULT : 0;
 499 }
 500
 501 static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu)
 502 {
 503         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 504         int rc = 0;
 505
 506         vcpu->stat.deliver_ckc++;
 507         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
 508                                          0, 0);
 509         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 510                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 511                 vcpu->arch.sie_block->eic = EXT_IRQ_CLK_COMP;
 512         } else {
 513                 rc  = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP,
 514                                    (u16 __user *)__LC_EXT_INT_CODE);
 515                 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 516                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 517                                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 518                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 519                                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 520         }
 521         clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
 522         return rc ? -EFAULT : 0;
 523 }
 524
 525 static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu)
 526 {
 527         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 528         struct kvm_s390_ext_info ext;
 529         int rc;
 530
 531         spin_lock(&li->lock);
 532         ext = li->irq.ext;
 533         clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
 534         li->irq.ext.ext_params2 = 0;
 535         spin_unlock(&li->lock);
 536
 537         VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx",
 538                    ext.ext_params2);
 539         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 540                                          KVM_S390_INT_PFAULT_INIT,
 541                                          0, ext.ext_params2);
 542
 543         rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE);
 544         rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR);
 545         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 546                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 547         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 548                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 549         rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2);
 550         return rc ? -EFAULT : 0;
 551 }
 552
 553 static int __write_machine_check(struct kvm_vcpu *vcpu,
 554                                  struct kvm_s390_mchk_info *mchk)
 555 {
 556         unsigned long ext_sa_addr;
 557         unsigned long lc;
 558         freg_t fprs[NUM_FPRS];
 559         union mci mci;
 560         int rc;
 561
 562         /*
 563          * All other possible payload for a machine check (e.g. the register
 564          * contents in the save area) will be handled by the ultravisor, as
 565          * the hypervisor does not not have the needed information for
 566          * protected guests.
 567          */
 568         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 569                 vcpu->arch.sie_block->iictl = IICTL_CODE_MCHK;
 570                 vcpu->arch.sie_block->mcic = mchk->mcic;
 571                 vcpu->arch.sie_block->faddr = mchk->failing_storage_address;
 572                 vcpu->arch.sie_block->edc = mchk->ext_damage_code;
 573                 return 0;
 574         }
 575
 576         mci.val = mchk->mcic;
 577         /* take care of lazy register loading */
 578         kvm_s390_fpu_store(vcpu->run);
 579         save_access_regs(vcpu->run->s.regs.acrs);
 580         if (MACHINE_HAS_GS && vcpu->arch.gs_enabled)
 581                 save_gs_cb(current->thread.gs_cb);
 582
 583         /* Extended save area */
 584         rc = read_guest_lc(vcpu, __LC_MCESAD, &ext_sa_addr,
 585                            sizeof(unsigned long));
 586         /* Only bits 0 through 63-LC are used for address formation */
 587         lc = ext_sa_addr & MCESA_LC_MASK;
 588         if (test_kvm_facility(vcpu->kvm, 133)) {
 589                 switch (lc) {
 590                 case 0:
 591                 case 10:
 592                         ext_sa_addr &= ~0x3ffUL;
 593                         break;
 594                 case 11:
 595                         ext_sa_addr &= ~0x7ffUL;
 596                         break;
 597                 case 12:
 598                         ext_sa_addr &= ~0xfffUL;
 599                         break;
 600                 default:
 601                         ext_sa_addr = 0;
 602                         break;
 603                 }
 604         } else {
 605                 ext_sa_addr &= ~0x3ffUL;
 606         }
 607
 608         if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(vcpu->kvm, 129)) {
 609                 if (write_guest_abs(vcpu, ext_sa_addr, vcpu->run->s.regs.vrs,
 610                                     512))
 611                         mci.vr = 0;
 612         } else {
 613                 mci.vr = 0;
 614         }
 615         if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(vcpu->kvm, 133)
 616             && (lc == 11 || lc == 12)) {
 617                 if (write_guest_abs(vcpu, ext_sa_addr + 1024,
 618                                     &vcpu->run->s.regs.gscb, 32))
 619                         mci.gs = 0;
 620         } else {
 621                 mci.gs = 0;
 622         }
 623
 624         /* General interruption information */
 625         rc |= put_guest_lc(vcpu, 1, (u8 __user *) __LC_AR_MODE_ID);
 626         rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
 627                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 628         rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
 629                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 630         rc |= put_guest_lc(vcpu, mci.val, (u64 __user *) __LC_MCCK_CODE);
 631
 632         /* Register-save areas */
 633         if (cpu_has_vx()) {
 634                 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
 635                 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, fprs, 128);
 636         } else {
 637                 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA,
 638                                      vcpu->run->s.regs.fprs, 128);
 639         }
 640         rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA,
 641                              vcpu->run->s.regs.gprs, 128);
 642         rc |= put_guest_lc(vcpu, vcpu->run->s.regs.fpc,
 643                            (u32 __user *) __LC_FP_CREG_SAVE_AREA);
 644         rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr,
 645                            (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA);
 646         rc |= put_guest_lc(vcpu, kvm_s390_get_cpu_timer(vcpu),
 647                            (u64 __user *) __LC_CPU_TIMER_SAVE_AREA);
 648         rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->ckc >> 8,
 649                            (u64 __user *) __LC_CLOCK_COMP_SAVE_AREA);
 650         rc |= write_guest_lc(vcpu, __LC_AREGS_SAVE_AREA,
 651                              &vcpu->run->s.regs.acrs, 64);
 652         rc |= write_guest_lc(vcpu, __LC_CREGS_SAVE_AREA,
 653                              &vcpu->arch.sie_block->gcr, 128);
 654
 655         /* Extended interruption information */
 656         rc |= put_guest_lc(vcpu, mchk->ext_damage_code,
 657                            (u32 __user *) __LC_EXT_DAMAGE_CODE);
 658         rc |= put_guest_lc(vcpu, mchk->failing_storage_address,
 659                            (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
 660         rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, &mchk->fixed_logout,
 661                              sizeof(mchk->fixed_logout));
 662         return rc ? -EFAULT : 0;
 663 }
 664
 665 static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu)
 666 {
 667         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
 668         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 669         struct kvm_s390_mchk_info mchk = {};
 670         int deliver = 0;
 671         int rc = 0;
 672
 673         spin_lock(&fi->lock);
 674         spin_lock(&li->lock);
 675         if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) ||
 676             test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) {
 677                 /*
 678                  * If there was an exigent machine check pending, then any
 679                  * repressible machine checks that might have been pending
 680                  * are indicated along with it, so always clear bits for
 681                  * repressible and exigent interrupts
 682                  */
 683                 mchk = li->irq.mchk;
 684                 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
 685                 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
 686                 memset(&li->irq.mchk, 0, sizeof(mchk));
 687                 deliver = 1;
 688         }
 689         /*
 690          * We indicate floating repressible conditions along with
 691          * other pending conditions. Channel Report Pending and Channel
 692          * Subsystem damage are the only two and are indicated by
 693          * bits in mcic and masked in cr14.
 694          */
 695         if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
 696                 mchk.mcic |= fi->mchk.mcic;
 697                 mchk.cr14 |= fi->mchk.cr14;
 698                 memset(&fi->mchk, 0, sizeof(mchk));
 699                 deliver = 1;
 700         }
 701         spin_unlock(&li->lock);
 702         spin_unlock(&fi->lock);
 703
 704         if (deliver) {
 705                 VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx",
 706                            mchk.mcic);
 707                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 708                                                  KVM_S390_MCHK,
 709                                                  mchk.cr14, mchk.mcic);
 710                 vcpu->stat.deliver_machine_check++;
 711                 rc = __write_machine_check(vcpu, &mchk);
 712         }
 713         return rc;
 714 }
 715
 716 static int __must_check __deliver_restart(struct kvm_vcpu *vcpu)
 717 {
 718         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 719         int rc = 0;
 720
 721         VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart");
 722         vcpu->stat.deliver_restart_signal++;
 723         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
 724
 725         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 726                 vcpu->arch.sie_block->iictl = IICTL_CODE_RESTART;
 727         } else {
 728                 rc  = write_guest_lc(vcpu,
 729                                      offsetof(struct lowcore, restart_old_psw),
 730                                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 731                 rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw),
 732                                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 733         }
 734         clear_bit(IRQ_PEND_RESTART, &li->pending_irqs);
 735         return rc ? -EFAULT : 0;
 736 }
 737
 738 static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu)
 739 {
 740         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 741         struct kvm_s390_prefix_info prefix;
 742
 743         spin_lock(&li->lock);
 744         prefix = li->irq.prefix;
 745         li->irq.prefix.address = 0;
 746         clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
 747         spin_unlock(&li->lock);
 748
 749         vcpu->stat.deliver_prefix_signal++;
 750         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 751                                          KVM_S390_SIGP_SET_PREFIX,
 752                                          prefix.address, 0);
 753
 754         kvm_s390_set_prefix(vcpu, prefix.address);
 755         return 0;
 756 }
 757
 758 static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu)
 759 {
 760         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 761         int rc;
 762         int cpu_addr;
 763
 764         spin_lock(&li->lock);
 765         cpu_addr = find_first_bit(li->sigp_emerg_pending, KVM_MAX_VCPUS);
 766         clear_bit(cpu_addr, li->sigp_emerg_pending);
 767         if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS))
 768                 clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
 769         spin_unlock(&li->lock);
 770
 771         VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg");
 772         vcpu->stat.deliver_emergency_signal++;
 773         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
 774                                          cpu_addr, 0);
 775         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 776                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 777                 vcpu->arch.sie_block->eic = EXT_IRQ_EMERGENCY_SIG;
 778                 vcpu->arch.sie_block->extcpuaddr = cpu_addr;
 779                 return 0;
 780         }
 781
 782         rc  = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG,
 783                            (u16 *)__LC_EXT_INT_CODE);
 784         rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR);
 785         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 786                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 787         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 788                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 789         return rc ? -EFAULT : 0;
 790 }
 791
 792 static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu)
 793 {
 794         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 795         struct kvm_s390_extcall_info extcall;
 796         int rc;
 797
 798         spin_lock(&li->lock);
 799         extcall = li->irq.extcall;
 800         li->irq.extcall.code = 0;
 801         clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
 802         spin_unlock(&li->lock);
 803
 804         VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call");
 805         vcpu->stat.deliver_external_call++;
 806         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 807                                          KVM_S390_INT_EXTERNAL_CALL,
 808                                          extcall.code, 0);
 809         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 810                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 811                 vcpu->arch.sie_block->eic = EXT_IRQ_EXTERNAL_CALL;
 812                 vcpu->arch.sie_block->extcpuaddr = extcall.code;
 813                 return 0;
 814         }
 815
 816         rc  = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL,
 817                            (u16 *)__LC_EXT_INT_CODE);
 818         rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR);
 819         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 820                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 821         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw,
 822                             sizeof(psw_t));
 823         return rc ? -EFAULT : 0;
 824 }
 825
 826 static int __deliver_prog_pv(struct kvm_vcpu *vcpu, u16 code)
 827 {
 828         switch (code) {
 829         case PGM_SPECIFICATION:
 830                 vcpu->arch.sie_block->iictl = IICTL_CODE_SPECIFICATION;
 831                 break;
 832         case PGM_OPERAND:
 833                 vcpu->arch.sie_block->iictl = IICTL_CODE_OPERAND;
 834                 break;
 835         default:
 836                 return -EINVAL;
 837         }
 838         return 0;
 839 }
 840
 841 static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
 842 {
 843         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 844         struct kvm_s390_pgm_info pgm_info;
 845         int rc = 0, nullifying = false;
 846         u16 ilen;
 847
 848         spin_lock(&li->lock);
 849         pgm_info = li->irq.pgm;
 850         clear_bit(IRQ_PEND_PROG, &li->pending_irqs);
 851         memset(&li->irq.pgm, 0, sizeof(pgm_info));
 852         spin_unlock(&li->lock);
 853
 854         ilen = pgm_info.flags & KVM_S390_PGM_FLAGS_ILC_MASK;
 855         VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilen:%d",
 856                    pgm_info.code, ilen);
 857         vcpu->stat.deliver_program++;
 858         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
 859                                          pgm_info.code, 0);
 860
 861         /* PER is handled by the ultravisor */
 862         if (kvm_s390_pv_cpu_is_protected(vcpu))
 863                 return __deliver_prog_pv(vcpu, pgm_info.code & ~PGM_PER);
 864
 865         switch (pgm_info.code & ~PGM_PER) {
 866         case PGM_AFX_TRANSLATION:
 867         case PGM_ASX_TRANSLATION:
 868         case PGM_EX_TRANSLATION:
 869         case PGM_LFX_TRANSLATION:
 870         case PGM_LSTE_SEQUENCE:
 871         case PGM_LSX_TRANSLATION:
 872         case PGM_LX_TRANSLATION:
 873         case PGM_PRIMARY_AUTHORITY:
 874         case PGM_SECONDARY_AUTHORITY:
 875                 nullifying = true;
 876                 fallthrough;
 877         case PGM_SPACE_SWITCH:
 878                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 879                                   (u64 *)__LC_TRANS_EXC_CODE);
 880                 break;
 881         case PGM_ALEN_TRANSLATION:
 882         case PGM_ALE_SEQUENCE:
 883         case PGM_ASTE_INSTANCE:
 884         case PGM_ASTE_SEQUENCE:
 885         case PGM_ASTE_VALIDITY:
 886         case PGM_EXTENDED_AUTHORITY:
 887                 rc = put_guest_lc(vcpu, pgm_info.exc_access_id,
 888                                   (u8 *)__LC_EXC_ACCESS_ID);
 889                 nullifying = true;
 890                 break;
 891         case PGM_ASCE_TYPE:
 892         case PGM_PAGE_TRANSLATION:
 893         case PGM_REGION_FIRST_TRANS:
 894         case PGM_REGION_SECOND_TRANS:
 895         case PGM_REGION_THIRD_TRANS:
 896         case PGM_SEGMENT_TRANSLATION:
 897                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 898                                   (u64 *)__LC_TRANS_EXC_CODE);
 899                 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
 900                                    (u8 *)__LC_EXC_ACCESS_ID);
 901                 rc |= put_guest_lc(vcpu, pgm_info.op_access_id,
 902                                    (u8 *)__LC_OP_ACCESS_ID);
 903                 nullifying = true;
 904                 break;
 905         case PGM_MONITOR:
 906                 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr,
 907                                   (u16 *)__LC_MON_CLASS_NR);
 908                 rc |= put_guest_lc(vcpu, pgm_info.mon_code,
 909                                    (u64 *)__LC_MON_CODE);
 910                 break;
 911         case PGM_VECTOR_PROCESSING:
 912         case PGM_DATA:
 913                 rc = put_guest_lc(vcpu, pgm_info.data_exc_code,
 914                                   (u32 *)__LC_DATA_EXC_CODE);
 915                 break;
 916         case PGM_PROTECTION:
 917                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 918                                   (u64 *)__LC_TRANS_EXC_CODE);
 919                 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
 920                                    (u8 *)__LC_EXC_ACCESS_ID);
 921                 break;
 922         case PGM_STACK_FULL:
 923         case PGM_STACK_EMPTY:
 924         case PGM_STACK_SPECIFICATION:
 925         case PGM_STACK_TYPE:
 926         case PGM_STACK_OPERATION:
 927         case PGM_TRACE_TABEL:
 928         case PGM_CRYPTO_OPERATION:
 929                 nullifying = true;
 930                 break;
 931         }
 932
 933         if (pgm_info.code & PGM_PER) {
 934                 rc |= put_guest_lc(vcpu, pgm_info.per_code,
 935                                    (u8 *) __LC_PER_CODE);
 936                 rc |= put_guest_lc(vcpu, pgm_info.per_atmid,
 937                                    (u8 *)__LC_PER_ATMID);
 938                 rc |= put_guest_lc(vcpu, pgm_info.per_address,
 939                                    (u64 *) __LC_PER_ADDRESS);
 940                 rc |= put_guest_lc(vcpu, pgm_info.per_access_id,
 941                                    (u8 *) __LC_PER_ACCESS_ID);
 942         }
 943
 944         if (nullifying && !(pgm_info.flags & KVM_S390_PGM_FLAGS_NO_REWIND))
 945                 kvm_s390_rewind_psw(vcpu, ilen);
 946
 947         /* bit 1+2 of the target are the ilc, so we can directly use ilen */
 948         rc |= put_guest_lc(vcpu, ilen, (u16 *) __LC_PGM_ILC);
 949         rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea,
 950                                  (u64 *) __LC_PGM_LAST_BREAK);
 951         rc |= put_guest_lc(vcpu, pgm_info.code,
 952                            (u16 *)__LC_PGM_INT_CODE);
 953         rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW,
 954                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 955         rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW,
 956                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 957         return rc ? -EFAULT : 0;
 958 }
 959
 960 #define SCCB_MASK 0xFFFFFFF8
 961 #define SCCB_EVENT_PENDING 0x3
 962
 963 static int write_sclp(struct kvm_vcpu *vcpu, u32 parm)
 964 {
 965         int rc;
 966
 967         if (kvm_s390_pv_cpu_get_handle(vcpu)) {
 968                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 969                 vcpu->arch.sie_block->eic = EXT_IRQ_SERVICE_SIG;
 970                 vcpu->arch.sie_block->eiparams = parm;
 971                 return 0;
 972         }
 973
 974         rc  = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE);
 975         rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 976         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 977                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 978         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 979                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 980         rc |= put_guest_lc(vcpu, parm,
 981                            (u32 *)__LC_EXT_PARAMS);
 982
 983         return rc ? -EFAULT : 0;
 984 }
 985
 986 static int __must_check __deliver_service(struct kvm_vcpu *vcpu)
 987 {
 988         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
 989         struct kvm_s390_ext_info ext;
 990
 991         spin_lock(&fi->lock);
 992         if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs) ||
 993             !(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) {
 994                 spin_unlock(&fi->lock);
 995                 return 0;
 996         }
 997         ext = fi->srv_signal;
 998         memset(&fi->srv_signal, 0, sizeof(ext));
 999         clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1000         clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1001         if (kvm_s390_pv_cpu_is_protected(vcpu))
1002                 set_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs);
1003         spin_unlock(&fi->lock);
1004
1005         VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x",
1006                    ext.ext_params);
1007         vcpu->stat.deliver_service_signal++;
1008         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
1009                                          ext.ext_params, 0);
1010
1011         return write_sclp(vcpu, ext.ext_params);
1012 }
1013
1014 static int __must_check __deliver_service_ev(struct kvm_vcpu *vcpu)
1015 {
1016         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1017         struct kvm_s390_ext_info ext;
1018
1019         spin_lock(&fi->lock);
1020         if (!(test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs))) {
1021                 spin_unlock(&fi->lock);
1022                 return 0;
1023         }
1024         ext = fi->srv_signal;
1025         /* only clear the event bits */
1026         fi->srv_signal.ext_params &= ~SCCB_EVENT_PENDING;
1027         clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1028         spin_unlock(&fi->lock);
1029
1030         VCPU_EVENT(vcpu, 4, "%s", "deliver: sclp parameter event");
1031         vcpu->stat.deliver_service_signal++;
1032         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
1033                                          ext.ext_params, 0);
1034
1035         return write_sclp(vcpu, ext.ext_params & SCCB_EVENT_PENDING);
1036 }
1037
1038 static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu)
1039 {
1040         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1041         struct kvm_s390_interrupt_info *inti;
1042         int rc = 0;
1043
1044         spin_lock(&fi->lock);
1045         inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT],
1046                                         struct kvm_s390_interrupt_info,
1047                                         list);
1048         if (inti) {
1049                 list_del(&inti->list);
1050                 fi->counters[FIRQ_CNTR_PFAULT] -= 1;
1051         }
1052         if (list_empty(&fi->lists[FIRQ_LIST_PFAULT]))
1053                 clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1054         spin_unlock(&fi->lock);
1055
1056         if (inti) {
1057                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1058                                                  KVM_S390_INT_PFAULT_DONE, 0,
1059                                                  inti->ext.ext_params2);
1060                 VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx",
1061                            inti->ext.ext_params2);
1062
1063                 rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
1064                                 (u16 *)__LC_EXT_INT_CODE);
1065                 rc |= put_guest_lc(vcpu, PFAULT_DONE,
1066                                 (u16 *)__LC_EXT_CPU_ADDR);
1067                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
1068                                 &vcpu->arch.sie_block->gpsw,
1069                                 sizeof(psw_t));
1070                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
1071                                 &vcpu->arch.sie_block->gpsw,
1072                                 sizeof(psw_t));
1073                 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
1074                                 (u64 *)__LC_EXT_PARAMS2);
1075                 kfree(inti);
1076         }
1077         return rc ? -EFAULT : 0;
1078 }
1079
1080 static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu)
1081 {
1082         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1083         struct kvm_s390_interrupt_info *inti;
1084         int rc = 0;
1085
1086         spin_lock(&fi->lock);
1087         inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO],
1088                                         struct kvm_s390_interrupt_info,
1089                                         list);
1090         if (inti) {
1091                 VCPU_EVENT(vcpu, 4,
1092                            "deliver: virtio parm: 0x%x,parm64: 0x%llx",
1093                            inti->ext.ext_params, inti->ext.ext_params2);
1094                 vcpu->stat.deliver_virtio++;
1095                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1096                                 inti->type,
1097                                 inti->ext.ext_params,
1098                                 inti->ext.ext_params2);
1099                 list_del(&inti->list);
1100                 fi->counters[FIRQ_CNTR_VIRTIO] -= 1;
1101         }
1102         if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO]))
1103                 clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1104         spin_unlock(&fi->lock);
1105
1106         if (inti) {
1107                 rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
1108                                 (u16 *)__LC_EXT_INT_CODE);
1109                 rc |= put_guest_lc(vcpu, VIRTIO_PARAM,
1110                                 (u16 *)__LC_EXT_CPU_ADDR);
1111                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
1112                                 &vcpu->arch.sie_block->gpsw,
1113                                 sizeof(psw_t));
1114                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
1115                                 &vcpu->arch.sie_block->gpsw,
1116                                 sizeof(psw_t));
1117                 rc |= put_guest_lc(vcpu, inti->ext.ext_params,
1118                                 (u32 *)__LC_EXT_PARAMS);
1119                 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
1120                                 (u64 *)__LC_EXT_PARAMS2);
1121                 kfree(inti);
1122         }
1123         return rc ? -EFAULT : 0;
1124 }
1125
1126 static int __do_deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info *io)
1127 {
1128         int rc;
1129
1130         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
1131                 vcpu->arch.sie_block->iictl = IICTL_CODE_IO;
1132                 vcpu->arch.sie_block->subchannel_id = io->subchannel_id;
1133                 vcpu->arch.sie_block->subchannel_nr = io->subchannel_nr;
1134                 vcpu->arch.sie_block->io_int_parm = io->io_int_parm;
1135                 vcpu->arch.sie_block->io_int_word = io->io_int_word;
1136                 return 0;
1137         }
1138
1139         rc  = put_guest_lc(vcpu, io->subchannel_id, (u16 *)__LC_SUBCHANNEL_ID);
1140         rc |= put_guest_lc(vcpu, io->subchannel_nr, (u16 *)__LC_SUBCHANNEL_NR);
1141         rc |= put_guest_lc(vcpu, io->io_int_parm, (u32 *)__LC_IO_INT_PARM);
1142         rc |= put_guest_lc(vcpu, io->io_int_word, (u32 *)__LC_IO_INT_WORD);
1143         rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
1144                              &vcpu->arch.sie_block->gpsw,
1145                              sizeof(psw_t));
1146         rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
1147                             &vcpu->arch.sie_block->gpsw,
1148                             sizeof(psw_t));
1149         return rc ? -EFAULT : 0;
1150 }
1151
1152 static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
1153                                      unsigned long irq_type)
1154 {
1155         struct list_head *isc_list;
1156         struct kvm_s390_float_interrupt *fi;
1157         struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
1158         struct kvm_s390_interrupt_info *inti = NULL;
1159         struct kvm_s390_io_info io;
1160         u32 isc;
1161         int rc = 0;
1162
1163         fi = &vcpu->kvm->arch.float_int;
1164
1165         spin_lock(&fi->lock);
1166         isc = irq_type_to_isc(irq_type);
1167         isc_list = &fi->lists[isc];
1168         inti = list_first_entry_or_null(isc_list,
1169                                         struct kvm_s390_interrupt_info,
1170                                         list);
1171         if (inti) {
1172                 if (inti->type & KVM_S390_INT_IO_AI_MASK)
1173                         VCPU_EVENT(vcpu, 4, "%s", "deliver: I/O (AI)");
1174                 else
1175                         VCPU_EVENT(vcpu, 4, "deliver: I/O %x ss %x schid %04x",
1176                         inti->io.subchannel_id >> 8,
1177                         inti->io.subchannel_id >> 1 & 0x3,
1178                         inti->io.subchannel_nr);
1179
1180                 vcpu->stat.deliver_io++;
1181                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1182                                 inti->type,
1183                                 ((__u32)inti->io.subchannel_id << 16) |
1184                                 inti->io.subchannel_nr,
1185                                 ((__u64)inti->io.io_int_parm << 32) |
1186                                 inti->io.io_int_word);
1187                 list_del(&inti->list);
1188                 fi->counters[FIRQ_CNTR_IO] -= 1;
1189         }
1190         if (list_empty(isc_list))
1191                 clear_bit(irq_type, &fi->pending_irqs);
1192         spin_unlock(&fi->lock);
1193
1194         if (inti) {
1195                 rc = __do_deliver_io(vcpu, &(inti->io));
1196                 kfree(inti);
1197                 goto out;
1198         }
1199
1200         if (gi->origin && gisa_tac_ipm_gisc(gi->origin, isc)) {
1201                 /*
1202                  * in case an adapter interrupt was not delivered
1203                  * in SIE context KVM will handle the delivery
1204                  */
1205                 VCPU_EVENT(vcpu, 4, "%s isc %u", "deliver: I/O (AI/gisa)", isc);
1206                 memset(&io, 0, sizeof(io));
1207                 io.io_int_word = isc_to_int_word(isc);
1208                 vcpu->stat.deliver_io++;
1209                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1210                         KVM_S390_INT_IO(1, 0, 0, 0),
1211                         ((__u32)io.subchannel_id << 16) |
1212                         io.subchannel_nr,
1213                         ((__u64)io.io_int_parm << 32) |
1214                         io.io_int_word);
1215                 rc = __do_deliver_io(vcpu, &io);
1216         }
1217 out:
1218         return rc;
1219 }
1220
1221 /* Check whether an external call is pending (deliverable or not) */
1222 int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
1223 {
1224         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1225
1226         if (!sclp.has_sigpif)
1227                 return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
1228
1229         return sca_ext_call_pending(vcpu, NULL);
1230 }
1231
1232 int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
1233 {
1234         if (deliverable_irqs(vcpu))
1235                 return 1;
1236
1237         if (kvm_cpu_has_pending_timer(vcpu))
1238                 return 1;
1239
1240         /* external call pending and deliverable */
1241         if (kvm_s390_ext_call_pending(vcpu) &&
1242             !psw_extint_disabled(vcpu) &&
1243             (vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
1244                 return 1;
1245
1246         if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu))
1247                 return 1;
1248         return 0;
1249 }
1250
1251 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
1252 {
1253         return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu);
1254 }
1255
1256 static u64 __calculate_sltime(struct kvm_vcpu *vcpu)
1257 {
1258         const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
1259         const u64 ckc = vcpu->arch.sie_block->ckc;
1260         u64 cputm, sltime = 0;
1261
1262         if (ckc_interrupts_enabled(vcpu)) {
1263                 if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
1264                         if ((s64)now < (s64)ckc)
1265                                 sltime = tod_to_ns((s64)ckc - (s64)now);
1266                 } else if (now < ckc) {
1267                         sltime = tod_to_ns(ckc - now);
1268                 }
1269                 /* already expired */
1270                 if (!sltime)
1271                         return 0;
1272                 if (cpu_timer_interrupts_enabled(vcpu)) {
1273                         cputm = kvm_s390_get_cpu_timer(vcpu);
1274                         /* already expired? */
1275                         if (cputm >> 63)
1276                                 return 0;
1277                         return min_t(u64, sltime, tod_to_ns(cputm));
1278                 }
1279         } else if (cpu_timer_interrupts_enabled(vcpu)) {
1280                 sltime = kvm_s390_get_cpu_timer(vcpu);
1281                 /* already expired? */
1282                 if (sltime >> 63)
1283                         return 0;
1284         }
1285         return sltime;
1286 }
1287
1288 int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
1289 {
1290         struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
1291         u64 sltime;
1292
1293         vcpu->stat.exit_wait_state++;
1294
1295         /* fast path */
1296         if (kvm_arch_vcpu_runnable(vcpu))
1297                 return 0;
1298
1299         if (psw_interrupts_disabled(vcpu)) {
1300                 VCPU_EVENT(vcpu, 3, "%s", "disabled wait");
1301                 return -EOPNOTSUPP; /* disabled wait */
1302         }
1303
1304         if (gi->origin &&
1305             (gisa_get_ipm_or_restore_iam(gi) &
1306              vcpu->arch.sie_block->gcr[6] >> 24))
1307                 return 0;
1308
1309         if (!ckc_interrupts_enabled(vcpu) &&
1310             !cpu_timer_interrupts_enabled(vcpu)) {
1311                 VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer");
1312                 __set_cpu_idle(vcpu);
1313                 goto no_timer;
1314         }
1315
1316         sltime = __calculate_sltime(vcpu);
1317         if (!sltime)
1318                 return 0;
1319
1320         __set_cpu_idle(vcpu);
1321         hrtimer_start(&vcpu->arch.ckc_timer, sltime, HRTIMER_MODE_REL);
1322         VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime);
1323 no_timer:
1324         kvm_vcpu_srcu_read_unlock(vcpu);
1325         kvm_vcpu_halt(vcpu);
1326         vcpu->valid_wakeup = false;
1327         __unset_cpu_idle(vcpu);
1328         kvm_vcpu_srcu_read_lock(vcpu);
1329
1330         hrtimer_cancel(&vcpu->arch.ckc_timer);
1331         return 0;
1332 }
1333
1334 void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu)
1335 {
1336         vcpu->valid_wakeup = true;
1337         kvm_vcpu_wake_up(vcpu);
1338
1339         /*
1340          * The VCPU might not be sleeping but rather executing VSIE. Let's
1341          * kick it, so it leaves the SIE to process the request.
1342          */
1343         kvm_s390_vsie_kick(vcpu);
1344 }
1345
1346 enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer)
1347 {
1348         struct kvm_vcpu *vcpu;
1349         u64 sltime;
1350
1351         vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer);
1352         sltime = __calculate_sltime(vcpu);
1353
1354         /*
1355          * If the monotonic clock runs faster than the tod clock we might be
1356          * woken up too early and have to go back to sleep to avoid deadlocks.
1357          */
1358         if (sltime && hrtimer_forward_now(timer, ns_to_ktime(sltime)))
1359                 return HRTIMER_RESTART;
1360         kvm_s390_vcpu_wakeup(vcpu);
1361         return HRTIMER_NORESTART;
1362 }
1363
1364 void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu)
1365 {
1366         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1367
1368         spin_lock(&li->lock);
1369         li->pending_irqs = 0;
1370         bitmap_zero(li->sigp_emerg_pending, KVM_MAX_VCPUS);
1371         memset(&li->irq, 0, sizeof(li->irq));
1372         spin_unlock(&li->lock);
1373
1374         sca_clear_ext_call(vcpu);
1375 }
1376
1377 int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
1378 {
1379         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1380         int rc = 0;
1381         bool delivered = false;
1382         unsigned long irq_type;
1383         unsigned long irqs;
1384
1385         __reset_intercept_indicators(vcpu);
1386
1387         /* pending ckc conditions might have been invalidated */
1388         clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1389         if (ckc_irq_pending(vcpu))
1390                 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1391
1392         /* pending cpu timer conditions might have been invalidated */
1393         clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1394         if (cpu_timer_irq_pending(vcpu))
1395                 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1396
1397         while ((irqs = deliverable_irqs(vcpu)) && !rc) {
1398                 /* bits are in the reverse order of interrupt priority */
1399                 irq_type = find_last_bit(&irqs, IRQ_PEND_COUNT);
1400                 switch (irq_type) {
1401                 case IRQ_PEND_IO_ISC_0:
1402                 case IRQ_PEND_IO_ISC_1:
1403                 case IRQ_PEND_IO_ISC_2:
1404                 case IRQ_PEND_IO_ISC_3:
1405                 case IRQ_PEND_IO_ISC_4:
1406                 case IRQ_PEND_IO_ISC_5:
1407                 case IRQ_PEND_IO_ISC_6:
1408                 case IRQ_PEND_IO_ISC_7:
1409                         rc = __deliver_io(vcpu, irq_type);
1410                         break;
1411                 case IRQ_PEND_MCHK_EX:
1412                 case IRQ_PEND_MCHK_REP:
1413                         rc = __deliver_machine_check(vcpu);
1414                         break;
1415                 case IRQ_PEND_PROG:
1416                         rc = __deliver_prog(vcpu);
1417                         break;
1418                 case IRQ_PEND_EXT_EMERGENCY:
1419                         rc = __deliver_emergency_signal(vcpu);
1420                         break;
1421                 case IRQ_PEND_EXT_EXTERNAL:
1422                         rc = __deliver_external_call(vcpu);
1423                         break;
1424                 case IRQ_PEND_EXT_CLOCK_COMP:
1425                         rc = __deliver_ckc(vcpu);
1426                         break;
1427                 case IRQ_PEND_EXT_CPU_TIMER:
1428                         rc = __deliver_cpu_timer(vcpu);
1429                         break;
1430                 case IRQ_PEND_RESTART:
1431                         rc = __deliver_restart(vcpu);
1432                         break;
1433                 case IRQ_PEND_SET_PREFIX:
1434                         rc = __deliver_set_prefix(vcpu);
1435                         break;
1436                 case IRQ_PEND_PFAULT_INIT:
1437                         rc = __deliver_pfault_init(vcpu);
1438                         break;
1439                 case IRQ_PEND_EXT_SERVICE:
1440                         rc = __deliver_service(vcpu);
1441                         break;
1442                 case IRQ_PEND_EXT_SERVICE_EV:
1443                         rc = __deliver_service_ev(vcpu);
1444                         break;
1445                 case IRQ_PEND_PFAULT_DONE:
1446                         rc = __deliver_pfault_done(vcpu);
1447                         break;
1448                 case IRQ_PEND_VIRTIO:
1449                         rc = __deliver_virtio(vcpu);
1450                         break;
1451                 default:
1452                         WARN_ONCE(1, "Unknown pending irq type %ld", irq_type);
1453                         clear_bit(irq_type, &li->pending_irqs);
1454                 }
1455                 delivered |= !rc;
1456         }
1457
1458         /*
1459          * We delivered at least one interrupt and modified the PC. Force a
1460          * singlestep event now.
1461          */
1462         if (delivered && guestdbg_sstep_enabled(vcpu)) {
1463                 struct kvm_debug_exit_arch *debug_exit = &vcpu->run->debug.arch;
1464
1465                 debug_exit->addr = vcpu->arch.sie_block->gpsw.addr;
1466                 debug_exit->type = KVM_SINGLESTEP;
1467                 vcpu->guest_debug |= KVM_GUESTDBG_EXIT_PENDING;
1468         }
1469
1470         set_intercept_indicators(vcpu);
1471
1472         return rc;
1473 }
1474
1475 static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1476 {
1477         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1478
1479         vcpu->stat.inject_program++;
1480         VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code);
1481         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
1482                                    irq->u.pgm.code, 0);
1483
1484         if (!(irq->u.pgm.flags & KVM_S390_PGM_FLAGS_ILC_VALID)) {
1485                 /* auto detection if no valid ILC was given */
1486                 irq->u.pgm.flags &= ~KVM_S390_PGM_FLAGS_ILC_MASK;
1487                 irq->u.pgm.flags |= kvm_s390_get_ilen(vcpu);
1488                 irq->u.pgm.flags |= KVM_S390_PGM_FLAGS_ILC_VALID;
1489         }
1490
1491         if (irq->u.pgm.code == PGM_PER) {
1492                 li->irq.pgm.code |= PGM_PER;
1493                 li->irq.pgm.flags = irq->u.pgm.flags;
1494                 /* only modify PER related information */
1495                 li->irq.pgm.per_address = irq->u.pgm.per_address;
1496                 li->irq.pgm.per_code = irq->u.pgm.per_code;
1497                 li->irq.pgm.per_atmid = irq->u.pgm.per_atmid;
1498                 li->irq.pgm.per_access_id = irq->u.pgm.per_access_id;
1499         } else if (!(irq->u.pgm.code & PGM_PER)) {
1500                 li->irq.pgm.code = (li->irq.pgm.code & PGM_PER) |
1501                                    irq->u.pgm.code;
1502                 li->irq.pgm.flags = irq->u.pgm.flags;
1503                 /* only modify non-PER information */
1504                 li->irq.pgm.trans_exc_code = irq->u.pgm.trans_exc_code;
1505                 li->irq.pgm.mon_code = irq->u.pgm.mon_code;
1506                 li->irq.pgm.data_exc_code = irq->u.pgm.data_exc_code;
1507                 li->irq.pgm.mon_class_nr = irq->u.pgm.mon_class_nr;
1508                 li->irq.pgm.exc_access_id = irq->u.pgm.exc_access_id;
1509                 li->irq.pgm.op_access_id = irq->u.pgm.op_access_id;
1510         } else {
1511                 li->irq.pgm = irq->u.pgm;
1512         }
1513         set_bit(IRQ_PEND_PROG, &li->pending_irqs);
1514         return 0;
1515 }
1516
1517 static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1518 {
1519         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1520
1521         vcpu->stat.inject_pfault_init++;
1522         VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx",
1523                    irq->u.ext.ext_params2);
1524         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT,
1525                                    irq->u.ext.ext_params,
1526                                    irq->u.ext.ext_params2);
1527
1528         li->irq.ext = irq->u.ext;
1529         set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
1530         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1531         return 0;
1532 }
1533
1534 static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1535 {
1536         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1537         struct kvm_s390_extcall_info *extcall = &li->irq.extcall;
1538         uint16_t src_id = irq->u.extcall.code;
1539
1540         vcpu->stat.inject_external_call++;
1541         VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u",
1542                    src_id);
1543         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL,
1544                                    src_id, 0);
1545
1546         /* sending vcpu invalid */
1547         if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL)
1548                 return -EINVAL;
1549
1550         if (sclp.has_sigpif && !kvm_s390_pv_cpu_get_handle(vcpu))
1551                 return sca_inject_ext_call(vcpu, src_id);
1552
1553         if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs))
1554                 return -EBUSY;
1555         *extcall = irq->u.extcall;
1556         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1557         return 0;
1558 }
1559
1560 static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1561 {
1562         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1563         struct kvm_s390_prefix_info *prefix = &li->irq.prefix;
1564
1565         vcpu->stat.inject_set_prefix++;
1566         VCPU_EVENT(vcpu, 3, "inject: set prefix to %x",
1567                    irq->u.prefix.address);
1568         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX,
1569                                    irq->u.prefix.address, 0);
1570
1571         if (!is_vcpu_stopped(vcpu))
1572                 return -EBUSY;
1573
1574         *prefix = irq->u.prefix;
1575         set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
1576         return 0;
1577 }
1578
1579 #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS)
1580 static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1581 {
1582         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1583         struct kvm_s390_stop_info *stop = &li->irq.stop;
1584         int rc = 0;
1585
1586         vcpu->stat.inject_stop_signal++;
1587         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0);
1588
1589         if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS)
1590                 return -EINVAL;
1591
1592         if (is_vcpu_stopped(vcpu)) {
1593                 if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS)
1594                         rc = kvm_s390_store_status_unloaded(vcpu,
1595                                                 KVM_S390_STORE_STATUS_NOADDR);
1596                 return rc;
1597         }
1598
1599         if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs))
1600                 return -EBUSY;
1601         stop->flags = irq->u.stop.flags;
1602         kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
1603         return 0;
1604 }
1605
1606 static int __inject_sigp_restart(struct kvm_vcpu *vcpu)
1607 {
1608         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1609
1610         vcpu->stat.inject_restart++;
1611         VCPU_EVENT(vcpu, 3, "%s", "inject: restart int");
1612         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
1613
1614         set_bit(IRQ_PEND_RESTART, &li->pending_irqs);
1615         return 0;
1616 }
1617
1618 static int __inject_sigp_emergency(struct kvm_vcpu *vcpu,
1619                                    struct kvm_s390_irq *irq)
1620 {
1621         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1622
1623         vcpu->stat.inject_emergency_signal++;
1624         VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u",
1625                    irq->u.emerg.code);
1626         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
1627                                    irq->u.emerg.code, 0);
1628
1629         /* sending vcpu invalid */
1630         if (kvm_get_vcpu_by_id(vcpu->kvm, irq->u.emerg.code) == NULL)
1631                 return -EINVAL;
1632
1633         set_bit(irq->u.emerg.code, li->sigp_emerg_pending);
1634         set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
1635         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1636         return 0;
1637 }
1638
1639 static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1640 {
1641         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1642         struct kvm_s390_mchk_info *mchk = &li->irq.mchk;
1643
1644         vcpu->stat.inject_mchk++;
1645         VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx",
1646                    irq->u.mchk.mcic);
1647         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0,
1648                                    irq->u.mchk.mcic);
1649
1650         /*
1651          * Because repressible machine checks can be indicated along with
1652          * exigent machine checks (PoP, Chapter 11, Interruption action)
1653          * we need to combine cr14, mcic and external damage code.
1654          * Failing storage address and the logout area should not be or'ed
1655          * together, we just indicate the last occurrence of the corresponding
1656          * machine check
1657          */
1658         mchk->cr14 |= irq->u.mchk.cr14;
1659         mchk->mcic |= irq->u.mchk.mcic;
1660         mchk->ext_damage_code |= irq->u.mchk.ext_damage_code;
1661         mchk->failing_storage_address = irq->u.mchk.failing_storage_address;
1662         memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout,
1663                sizeof(mchk->fixed_logout));
1664         if (mchk->mcic & MCHK_EX_MASK)
1665                 set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
1666         else if (mchk->mcic & MCHK_REP_MASK)
1667                 set_bit(IRQ_PEND_MCHK_REP,  &li->pending_irqs);
1668         return 0;
1669 }
1670
1671 static int __inject_ckc(struct kvm_vcpu *vcpu)
1672 {
1673         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1674
1675         vcpu->stat.inject_ckc++;
1676         VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external");
1677         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
1678                                    0, 0);
1679
1680         set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1681         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1682         return 0;
1683 }
1684
1685 static int __inject_cpu_timer(struct kvm_vcpu *vcpu)
1686 {
1687         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1688
1689         vcpu->stat.inject_cputm++;
1690         VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external");
1691         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
1692                                    0, 0);
1693
1694         set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1695         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1696         return 0;
1697 }
1698
1699 static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
1700                                                   int isc, u32 schid)
1701 {
1702         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1703         struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1704         struct kvm_s390_interrupt_info *iter;
1705         u16 id = (schid & 0xffff0000U) >> 16;
1706         u16 nr = schid & 0x0000ffffU;
1707
1708         spin_lock(&fi->lock);
1709         list_for_each_entry(iter, isc_list, list) {
1710                 if (schid && (id != iter->io.subchannel_id ||
1711                               nr != iter->io.subchannel_nr))
1712                         continue;
1713                 /* found an appropriate entry */
1714                 list_del_init(&iter->list);
1715                 fi->counters[FIRQ_CNTR_IO] -= 1;
1716                 if (list_empty(isc_list))
1717                         clear_bit(isc_to_irq_type(isc), &fi->pending_irqs);
1718                 spin_unlock(&fi->lock);
1719                 return iter;
1720         }
1721         spin_unlock(&fi->lock);
1722         return NULL;
1723 }
1724
1725 static struct kvm_s390_interrupt_info *get_top_io_int(struct kvm *kvm,
1726                                                       u64 isc_mask, u32 schid)
1727 {
1728         struct kvm_s390_interrupt_info *inti = NULL;
1729         int isc;
1730
1731         for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
1732                 if (isc_mask & isc_to_isc_bits(isc))
1733                         inti = get_io_int(kvm, isc, schid);
1734         }
1735         return inti;
1736 }
1737
1738 static int get_top_gisa_isc(struct kvm *kvm, u64 isc_mask, u32 schid)
1739 {
1740         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1741         unsigned long active_mask;
1742         int isc;
1743
1744         if (schid)
1745                 goto out;
1746         if (!gi->origin)
1747                 goto out;
1748
1749         active_mask = (isc_mask & gisa_get_ipm(gi->origin) << 24) << 32;
1750         while (active_mask) {
1751                 isc = __fls(active_mask) ^ (BITS_PER_LONG - 1);
1752                 if (gisa_tac_ipm_gisc(gi->origin, isc))
1753                         return isc;
1754                 clear_bit_inv(isc, &active_mask);
1755         }
1756 out:
1757         return -EINVAL;
1758 }
1759
1760 /*
1761  * Dequeue and return an I/O interrupt matching any of the interruption
1762  * subclasses as designated by the isc mask in cr6 and the schid (if != 0).
1763  * Take into account the interrupts pending in the interrupt list and in GISA.
1764  *
1765  * Note that for a guest that does not enable I/O interrupts
1766  * but relies on TPI, a flood of classic interrupts may starve
1767  * out adapter interrupts on the same isc. Linux does not do
1768  * that, and it is possible to work around the issue by configuring
1769  * different iscs for classic and adapter interrupts in the guest,
1770  * but we may want to revisit this in the future.
1771  */
1772 struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
1773                                                     u64 isc_mask, u32 schid)
1774 {
1775         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1776         struct kvm_s390_interrupt_info *inti, *tmp_inti;
1777         int isc;
1778
1779         inti = get_top_io_int(kvm, isc_mask, schid);
1780
1781         isc = get_top_gisa_isc(kvm, isc_mask, schid);
1782         if (isc < 0)
1783                 /* no AI in GISA */
1784                 goto out;
1785
1786         if (!inti)
1787                 /* AI in GISA but no classical IO int */
1788                 goto gisa_out;
1789
1790         /* both types of interrupts present */
1791         if (int_word_to_isc(inti->io.io_int_word) <= isc) {
1792                 /* classical IO int with higher priority */
1793                 gisa_set_ipm_gisc(gi->origin, isc);
1794                 goto out;
1795         }
1796 gisa_out:
1797         tmp_inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
1798         if (tmp_inti) {
1799                 tmp_inti->type = KVM_S390_INT_IO(1, 0, 0, 0);
1800                 tmp_inti->io.io_int_word = isc_to_int_word(isc);
1801                 if (inti)
1802                         kvm_s390_reinject_io_int(kvm, inti);
1803                 inti = tmp_inti;
1804         } else
1805                 gisa_set_ipm_gisc(gi->origin, isc);
1806 out:
1807         return inti;
1808 }
1809
1810 static int __inject_service(struct kvm *kvm,
1811                              struct kvm_s390_interrupt_info *inti)
1812 {
1813         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1814
1815         kvm->stat.inject_service_signal++;
1816         spin_lock(&fi->lock);
1817         fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING;
1818
1819         /* We always allow events, track them separately from the sccb ints */
1820         if (fi->srv_signal.ext_params & SCCB_EVENT_PENDING)
1821                 set_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1822
1823         /*
1824          * Early versions of the QEMU s390 bios will inject several
1825          * service interrupts after another without handling a
1826          * condition code indicating busy.
1827          * We will silently ignore those superfluous sccb values.
1828          * A future version of QEMU will take care of serialization
1829          * of servc requests
1830          */
1831         if (fi->srv_signal.ext_params & SCCB_MASK)
1832                 goto out;
1833         fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK;
1834         set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1835 out:
1836         spin_unlock(&fi->lock);
1837         kfree(inti);
1838         return 0;
1839 }
1840
1841 static int __inject_virtio(struct kvm *kvm,
1842                             struct kvm_s390_interrupt_info *inti)
1843 {
1844         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1845
1846         kvm->stat.inject_virtio++;
1847         spin_lock(&fi->lock);
1848         if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) {
1849                 spin_unlock(&fi->lock);
1850                 return -EBUSY;
1851         }
1852         fi->counters[FIRQ_CNTR_VIRTIO] += 1;
1853         list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]);
1854         set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1855         spin_unlock(&fi->lock);
1856         return 0;
1857 }
1858
1859 static int __inject_pfault_done(struct kvm *kvm,
1860                                  struct kvm_s390_interrupt_info *inti)
1861 {
1862         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1863
1864         kvm->stat.inject_pfault_done++;
1865         spin_lock(&fi->lock);
1866         if (fi->counters[FIRQ_CNTR_PFAULT] >=
1867                 (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) {
1868                 spin_unlock(&fi->lock);
1869                 return -EBUSY;
1870         }
1871         fi->counters[FIRQ_CNTR_PFAULT] += 1;
1872         list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]);
1873         set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1874         spin_unlock(&fi->lock);
1875         return 0;
1876 }
1877
1878 #define CR_PENDING_SUBCLASS 28
1879 static int __inject_float_mchk(struct kvm *kvm,
1880                                 struct kvm_s390_interrupt_info *inti)
1881 {
1882         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1883
1884         kvm->stat.inject_float_mchk++;
1885         spin_lock(&fi->lock);
1886         fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS);
1887         fi->mchk.mcic |= inti->mchk.mcic;
1888         set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs);
1889         spin_unlock(&fi->lock);
1890         kfree(inti);
1891         return 0;
1892 }
1893
1894 static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1895 {
1896         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1897         struct kvm_s390_float_interrupt *fi;
1898         struct list_head *list;
1899         int isc;
1900
1901         kvm->stat.inject_io++;
1902         isc = int_word_to_isc(inti->io.io_int_word);
1903
1904         /*
1905          * We do not use the lock checking variant as this is just a
1906          * performance optimization and we do not hold the lock here.
1907          * This is ok as the code will pick interrupts from both "lists"
1908          * for delivery.
1909          */
1910         if (gi->origin && inti->type & KVM_S390_INT_IO_AI_MASK) {
1911                 VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc);
1912                 gisa_set_ipm_gisc(gi->origin, isc);
1913                 kfree(inti);
1914                 return 0;
1915         }
1916
1917         fi = &kvm->arch.float_int;
1918         spin_lock(&fi->lock);
1919         if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) {
1920                 spin_unlock(&fi->lock);
1921                 return -EBUSY;
1922         }
1923         fi->counters[FIRQ_CNTR_IO] += 1;
1924
1925         if (inti->type & KVM_S390_INT_IO_AI_MASK)
1926                 VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)");
1927         else
1928                 VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x",
1929                         inti->io.subchannel_id >> 8,
1930                         inti->io.subchannel_id >> 1 & 0x3,
1931                         inti->io.subchannel_nr);
1932         list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1933         list_add_tail(&inti->list, list);
1934         set_bit(isc_to_irq_type(isc), &fi->pending_irqs);
1935         spin_unlock(&fi->lock);
1936         return 0;
1937 }
1938
1939 /*
1940  * Find a destination VCPU for a floating irq and kick it.
1941  */
1942 static void __floating_irq_kick(struct kvm *kvm, u64 type)
1943 {
1944         struct kvm_vcpu *dst_vcpu;
1945         int sigcpu, online_vcpus, nr_tries = 0;
1946
1947         online_vcpus = atomic_read(&kvm->online_vcpus);
1948         if (!online_vcpus)
1949                 return;
1950
1951         /* find idle VCPUs first, then round robin */
1952         sigcpu = find_first_bit(kvm->arch.idle_mask, online_vcpus);
1953         if (sigcpu == online_vcpus) {
1954                 do {
1955                         sigcpu = kvm->arch.float_int.next_rr_cpu++;
1956                         kvm->arch.float_int.next_rr_cpu %= online_vcpus;
1957                         /* avoid endless loops if all vcpus are stopped */
1958                         if (nr_tries++ >= online_vcpus)
1959                                 return;
1960                 } while (is_vcpu_stopped(kvm_get_vcpu(kvm, sigcpu)));
1961         }
1962         dst_vcpu = kvm_get_vcpu(kvm, sigcpu);
1963
1964         /* make the VCPU drop out of the SIE, or wake it up if sleeping */
1965         switch (type) {
1966         case KVM_S390_MCHK:
1967                 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT);
1968                 break;
1969         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1970                 if (!(type & KVM_S390_INT_IO_AI_MASK &&
1971                       kvm->arch.gisa_int.origin) ||
1972                       kvm_s390_pv_cpu_get_handle(dst_vcpu))
1973                         kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT);
1974                 break;
1975         default:
1976                 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT);
1977                 break;
1978         }
1979         kvm_s390_vcpu_wakeup(dst_vcpu);
1980 }
1981
1982 static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1983 {
1984         u64 type = READ_ONCE(inti->type);
1985         int rc;
1986
1987         switch (type) {
1988         case KVM_S390_MCHK:
1989                 rc = __inject_float_mchk(kvm, inti);
1990                 break;
1991         case KVM_S390_INT_VIRTIO:
1992                 rc = __inject_virtio(kvm, inti);
1993                 break;
1994         case KVM_S390_INT_SERVICE:
1995                 rc = __inject_service(kvm, inti);
1996                 break;
1997         case KVM_S390_INT_PFAULT_DONE:
1998                 rc = __inject_pfault_done(kvm, inti);
1999                 break;
2000         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2001                 rc = __inject_io(kvm, inti);
2002                 break;
2003         default:
2004                 rc = -EINVAL;
2005         }
2006         if (rc)
2007                 return rc;
2008
2009         __floating_irq_kick(kvm, type);
2010         return 0;
2011 }
2012
2013 int kvm_s390_inject_vm(struct kvm *kvm,
2014                        struct kvm_s390_interrupt *s390int)
2015 {
2016         struct kvm_s390_interrupt_info *inti;
2017         int rc;
2018
2019         inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
2020         if (!inti)
2021                 return -ENOMEM;
2022
2023         inti->type = s390int->type;
2024         switch (inti->type) {
2025         case KVM_S390_INT_VIRTIO:
2026                 VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx",
2027                          s390int->parm, s390int->parm64);
2028                 inti->ext.ext_params = s390int->parm;
2029                 inti->ext.ext_params2 = s390int->parm64;
2030                 break;
2031         case KVM_S390_INT_SERVICE:
2032                 VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm);
2033                 inti->ext.ext_params = s390int->parm;
2034                 break;
2035         case KVM_S390_INT_PFAULT_DONE:
2036                 inti->ext.ext_params2 = s390int->parm64;
2037                 break;
2038         case KVM_S390_MCHK:
2039                 VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx",
2040                          s390int->parm64);
2041                 inti->mchk.cr14 = s390int->parm; /* upper bits are not used */
2042                 inti->mchk.mcic = s390int->parm64;
2043                 break;
2044         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2045                 inti->io.subchannel_id = s390int->parm >> 16;
2046                 inti->io.subchannel_nr = s390int->parm & 0x0000ffffu;
2047                 inti->io.io_int_parm = s390int->parm64 >> 32;
2048                 inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull;
2049                 break;
2050         default:
2051                 kfree(inti);
2052                 return -EINVAL;
2053         }
2054         trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64,
2055                                  2);
2056
2057         rc = __inject_vm(kvm, inti);
2058         if (rc)
2059                 kfree(inti);
2060         return rc;
2061 }
2062
2063 int kvm_s390_reinject_io_int(struct kvm *kvm,
2064                               struct kvm_s390_interrupt_info *inti)
2065 {
2066         return __inject_vm(kvm, inti);
2067 }
2068
2069 int s390int_to_s390irq(struct kvm_s390_interrupt *s390int,
2070                        struct kvm_s390_irq *irq)
2071 {
2072         irq->type = s390int->type;
2073         switch (irq->type) {
2074         case KVM_S390_PROGRAM_INT:
2075                 if (s390int->parm & 0xffff0000)
2076                         return -EINVAL;
2077                 irq->u.pgm.code = s390int->parm;
2078                 break;
2079         case KVM_S390_SIGP_SET_PREFIX:
2080                 irq->u.prefix.address = s390int->parm;
2081                 break;
2082         case KVM_S390_SIGP_STOP:
2083                 irq->u.stop.flags = s390int->parm;
2084                 break;
2085         case KVM_S390_INT_EXTERNAL_CALL:
2086                 if (s390int->parm & 0xffff0000)
2087                         return -EINVAL;
2088                 irq->u.extcall.code = s390int->parm;
2089                 break;
2090         case KVM_S390_INT_EMERGENCY:
2091                 if (s390int->parm & 0xffff0000)
2092                         return -EINVAL;
2093                 irq->u.emerg.code = s390int->parm;
2094                 break;
2095         case KVM_S390_MCHK:
2096                 irq->u.mchk.mcic = s390int->parm64;
2097                 break;
2098         case KVM_S390_INT_PFAULT_INIT:
2099                 irq->u.ext.ext_params = s390int->parm;
2100                 irq->u.ext.ext_params2 = s390int->parm64;
2101                 break;
2102         case KVM_S390_RESTART:
2103         case KVM_S390_INT_CLOCK_COMP:
2104         case KVM_S390_INT_CPU_TIMER:
2105                 break;
2106         default:
2107                 return -EINVAL;
2108         }
2109         return 0;
2110 }
2111
2112 int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu)
2113 {
2114         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2115
2116         return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
2117 }
2118
2119 int kvm_s390_is_restart_irq_pending(struct kvm_vcpu *vcpu)
2120 {
2121         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2122
2123         return test_bit(IRQ_PEND_RESTART, &li->pending_irqs);
2124 }
2125
2126 void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu)
2127 {
2128         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2129
2130         spin_lock(&li->lock);
2131         li->irq.stop.flags = 0;
2132         clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
2133         spin_unlock(&li->lock);
2134 }
2135
2136 static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
2137 {
2138         int rc;
2139
2140         switch (irq->type) {
2141         case KVM_S390_PROGRAM_INT:
2142                 rc = __inject_prog(vcpu, irq);
2143                 break;
2144         case KVM_S390_SIGP_SET_PREFIX:
2145                 rc = __inject_set_prefix(vcpu, irq);
2146                 break;
2147         case KVM_S390_SIGP_STOP:
2148                 rc = __inject_sigp_stop(vcpu, irq);
2149                 break;
2150         case KVM_S390_RESTART:
2151                 rc = __inject_sigp_restart(vcpu);
2152                 break;
2153         case KVM_S390_INT_CLOCK_COMP:
2154                 rc = __inject_ckc(vcpu);
2155                 break;
2156         case KVM_S390_INT_CPU_TIMER:
2157                 rc = __inject_cpu_timer(vcpu);
2158                 break;
2159         case KVM_S390_INT_EXTERNAL_CALL:
2160                 rc = __inject_extcall(vcpu, irq);
2161                 break;
2162         case KVM_S390_INT_EMERGENCY:
2163                 rc = __inject_sigp_emergency(vcpu, irq);
2164                 break;
2165         case KVM_S390_MCHK:
2166                 rc = __inject_mchk(vcpu, irq);
2167                 break;
2168         case KVM_S390_INT_PFAULT_INIT:
2169                 rc = __inject_pfault_init(vcpu, irq);
2170                 break;
2171         case KVM_S390_INT_VIRTIO:
2172         case KVM_S390_INT_SERVICE:
2173         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2174         default:
2175                 rc = -EINVAL;
2176         }
2177
2178         return rc;
2179 }
2180
2181 int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
2182 {
2183         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2184         int rc;
2185
2186         spin_lock(&li->lock);
2187         rc = do_inject_vcpu(vcpu, irq);
2188         spin_unlock(&li->lock);
2189         if (!rc)
2190                 kvm_s390_vcpu_wakeup(vcpu);
2191         return rc;
2192 }
2193
2194 static inline void clear_irq_list(struct list_head *_list)
2195 {
2196         struct kvm_s390_interrupt_info *inti, *n;
2197
2198         list_for_each_entry_safe(inti, n, _list, list) {
2199                 list_del(&inti->list);
2200                 kfree(inti);
2201         }
2202 }
2203
2204 static void inti_to_irq(struct kvm_s390_interrupt_info *inti,
2205                        struct kvm_s390_irq *irq)
2206 {
2207         irq->type = inti->type;
2208         switch (inti->type) {
2209         case KVM_S390_INT_PFAULT_INIT:
2210         case KVM_S390_INT_PFAULT_DONE:
2211         case KVM_S390_INT_VIRTIO:
2212                 irq->u.ext = inti->ext;
2213                 break;
2214         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2215                 irq->u.io = inti->io;
2216                 break;
2217         }
2218 }
2219
2220 void kvm_s390_clear_float_irqs(struct kvm *kvm)
2221 {
2222         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2223         int i;
2224
2225         mutex_lock(&kvm->lock);
2226         if (!kvm_s390_pv_is_protected(kvm))
2227                 fi->masked_irqs = 0;
2228         mutex_unlock(&kvm->lock);
2229         spin_lock(&fi->lock);
2230         fi->pending_irqs = 0;
2231         memset(&fi->srv_signal, 0, sizeof(fi->srv_signal));
2232         memset(&fi->mchk, 0, sizeof(fi->mchk));
2233         for (i = 0; i < FIRQ_LIST_COUNT; i++)
2234                 clear_irq_list(&fi->lists[i]);
2235         for (i = 0; i < FIRQ_MAX_COUNT; i++)
2236                 fi->counters[i] = 0;
2237         spin_unlock(&fi->lock);
2238         kvm_s390_gisa_clear(kvm);
2239 };
2240
2241 static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
2242 {
2243         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
2244         struct kvm_s390_interrupt_info *inti;
2245         struct kvm_s390_float_interrupt *fi;
2246         struct kvm_s390_irq *buf;
2247         struct kvm_s390_irq *irq;
2248         int max_irqs;
2249         int ret = 0;
2250         int n = 0;
2251         int i;
2252
2253         if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0)
2254                 return -EINVAL;
2255
2256         /*
2257          * We are already using -ENOMEM to signal
2258          * userspace it may retry with a bigger buffer,
2259          * so we need to use something else for this case
2260          */
2261         buf = vzalloc(len);
2262         if (!buf)
2263                 return -ENOBUFS;
2264
2265         max_irqs = len / sizeof(struct kvm_s390_irq);
2266
2267         if (gi->origin && gisa_get_ipm(gi->origin)) {
2268                 for (i = 0; i <= MAX_ISC; i++) {
2269                         if (n == max_irqs) {
2270                                 /* signal userspace to try again */
2271                                 ret = -ENOMEM;
2272                                 goto out_nolock;
2273                         }
2274                         if (gisa_tac_ipm_gisc(gi->origin, i)) {
2275                                 irq = (struct kvm_s390_irq *) &buf[n];
2276                                 irq->type = KVM_S390_INT_IO(1, 0, 0, 0);
2277                                 irq->u.io.io_int_word = isc_to_int_word(i);
2278                                 n++;
2279                         }
2280                 }
2281         }
2282         fi = &kvm->arch.float_int;
2283         spin_lock(&fi->lock);
2284         for (i = 0; i < FIRQ_LIST_COUNT; i++) {
2285                 list_for_each_entry(inti, &fi->lists[i], list) {
2286                         if (n == max_irqs) {
2287                                 /* signal userspace to try again */
2288                                 ret = -ENOMEM;
2289                                 goto out;
2290                         }
2291                         inti_to_irq(inti, &buf[n]);
2292                         n++;
2293                 }
2294         }
2295         if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs) ||
2296             test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs)) {
2297                 if (n == max_irqs) {
2298                         /* signal userspace to try again */
2299                         ret = -ENOMEM;
2300                         goto out;
2301                 }
2302                 irq = (struct kvm_s390_irq *) &buf[n];
2303                 irq->type = KVM_S390_INT_SERVICE;
2304                 irq->u.ext = fi->srv_signal;
2305                 n++;
2306         }
2307         if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
2308                 if (n == max_irqs) {
2309                                 /* signal userspace to try again */
2310                                 ret = -ENOMEM;
2311                                 goto out;
2312                 }
2313                 irq = (struct kvm_s390_irq *) &buf[n];
2314                 irq->type = KVM_S390_MCHK;
2315                 irq->u.mchk = fi->mchk;
2316                 n++;
2317 }
2318
2319 out:
2320         spin_unlock(&fi->lock);
2321 out_nolock:
2322         if (!ret && n > 0) {
2323                 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n))
2324                         ret = -EFAULT;
2325         }
2326         vfree(buf);
2327
2328         return ret < 0 ? ret : n;
2329 }
2330
2331 static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr)
2332 {
2333         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2334         struct kvm_s390_ais_all ais;
2335
2336         if (attr->attr < sizeof(ais))
2337                 return -EINVAL;
2338
2339         if (!test_kvm_facility(kvm, 72))
2340                 return -EOPNOTSUPP;
2341
2342         mutex_lock(&fi->ais_lock);
2343         ais.simm = fi->simm;
2344         ais.nimm = fi->nimm;
2345         mutex_unlock(&fi->ais_lock);
2346
2347         if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais)))
2348                 return -EFAULT;
2349
2350         return 0;
2351 }
2352
2353 static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2354 {
2355         int r;
2356
2357         switch (attr->group) {
2358         case KVM_DEV_FLIC_GET_ALL_IRQS:
2359                 r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr,
2360                                           attr->attr);
2361                 break;
2362         case KVM_DEV_FLIC_AISM_ALL:
2363                 r = flic_ais_mode_get_all(dev->kvm, attr);
2364                 break;
2365         default:
2366                 r = -EINVAL;
2367         }
2368
2369         return r;
2370 }
2371
2372 static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti,
2373                                      u64 addr)
2374 {
2375         struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr;
2376         void *target = NULL;
2377         void __user *source;
2378         u64 size;
2379
2380         if (get_user(inti->type, (u64 __user *)addr))
2381                 return -EFAULT;
2382
2383         switch (inti->type) {
2384         case KVM_S390_INT_PFAULT_INIT:
2385         case KVM_S390_INT_PFAULT_DONE:
2386         case KVM_S390_INT_VIRTIO:
2387         case KVM_S390_INT_SERVICE:
2388                 target = (void *) &inti->ext;
2389                 source = &uptr->u.ext;
2390                 size = sizeof(inti->ext);
2391                 break;
2392         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2393                 target = (void *) &inti->io;
2394                 source = &uptr->u.io;
2395                 size = sizeof(inti->io);
2396                 break;
2397         case KVM_S390_MCHK:
2398                 target = (void *) &inti->mchk;
2399                 source = &uptr->u.mchk;
2400                 size = sizeof(inti->mchk);
2401                 break;
2402         default:
2403                 return -EINVAL;
2404         }
2405
2406         if (copy_from_user(target, source, size))
2407                 return -EFAULT;
2408
2409         return 0;
2410 }
2411
2412 static int enqueue_floating_irq(struct kvm_device *dev,
2413                                 struct kvm_device_attr *attr)
2414 {
2415         struct kvm_s390_interrupt_info *inti = NULL;
2416         int r = 0;
2417         int len = attr->attr;
2418
2419         if (len % sizeof(struct kvm_s390_irq) != 0)
2420                 return -EINVAL;
2421         else if (len > KVM_S390_FLIC_MAX_BUFFER)
2422                 return -EINVAL;
2423
2424         while (len >= sizeof(struct kvm_s390_irq)) {
2425                 inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
2426                 if (!inti)
2427                         return -ENOMEM;
2428
2429                 r = copy_irq_from_user(inti, attr->addr);
2430                 if (r) {
2431                         kfree(inti);
2432                         return r;
2433                 }
2434                 r = __inject_vm(dev->kvm, inti);
2435                 if (r) {
2436                         kfree(inti);
2437                         return r;
2438                 }
2439                 len -= sizeof(struct kvm_s390_irq);
2440                 attr->addr += sizeof(struct kvm_s390_irq);
2441         }
2442
2443         return r;
2444 }
2445
2446 static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id)
2447 {
2448         if (id >= MAX_S390_IO_ADAPTERS)
2449                 return NULL;
2450         id = array_index_nospec(id, MAX_S390_IO_ADAPTERS);
2451         return kvm->arch.adapters[id];
2452 }
2453
2454 static int register_io_adapter(struct kvm_device *dev,
2455                                struct kvm_device_attr *attr)
2456 {
2457         struct s390_io_adapter *adapter;
2458         struct kvm_s390_io_adapter adapter_info;
2459
2460         if (copy_from_user(&adapter_info,
2461                            (void __user *)attr->addr, sizeof(adapter_info)))
2462                 return -EFAULT;
2463
2464         if (adapter_info.id >= MAX_S390_IO_ADAPTERS)
2465                 return -EINVAL;
2466
2467         adapter_info.id = array_index_nospec(adapter_info.id,
2468                                              MAX_S390_IO_ADAPTERS);
2469
2470         if (dev->kvm->arch.adapters[adapter_info.id] != NULL)
2471                 return -EINVAL;
2472
2473         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL_ACCOUNT);
2474         if (!adapter)
2475                 return -ENOMEM;
2476
2477         adapter->id = adapter_info.id;
2478         adapter->isc = adapter_info.isc;
2479         adapter->maskable = adapter_info.maskable;
2480         adapter->masked = false;
2481         adapter->swap = adapter_info.swap;
2482         adapter->suppressible = (adapter_info.flags) &
2483                                 KVM_S390_ADAPTER_SUPPRESSIBLE;
2484         dev->kvm->arch.adapters[adapter->id] = adapter;
2485
2486         return 0;
2487 }
2488
2489 int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked)
2490 {
2491         int ret;
2492         struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2493
2494         if (!adapter || !adapter->maskable)
2495                 return -EINVAL;
2496         ret = adapter->masked;
2497         adapter->masked = masked;
2498         return ret;
2499 }
2500
2501 void kvm_s390_destroy_adapters(struct kvm *kvm)
2502 {
2503         int i;
2504
2505         for (i = 0; i < MAX_S390_IO_ADAPTERS; i++)
2506                 kfree(kvm->arch.adapters[i]);
2507 }
2508
2509 static int modify_io_adapter(struct kvm_device *dev,
2510                              struct kvm_device_attr *attr)
2511 {
2512         struct kvm_s390_io_adapter_req req;
2513         struct s390_io_adapter *adapter;
2514         int ret;
2515
2516         if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
2517                 return -EFAULT;
2518
2519         adapter = get_io_adapter(dev->kvm, req.id);
2520         if (!adapter)
2521                 return -EINVAL;
2522         switch (req.type) {
2523         case KVM_S390_IO_ADAPTER_MASK:
2524                 ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask);
2525                 if (ret > 0)
2526                         ret = 0;
2527                 break;
2528         /*
2529          * The following operations are no longer needed and therefore no-ops.
2530          * The gpa to hva translation is done when an IRQ route is set up. The
2531          * set_irq code uses get_user_pages_remote() to do the actual write.
2532          */
2533         case KVM_S390_IO_ADAPTER_MAP:
2534         case KVM_S390_IO_ADAPTER_UNMAP:
2535                 ret = 0;
2536                 break;
2537         default:
2538                 ret = -EINVAL;
2539         }
2540
2541         return ret;
2542 }
2543
2544 static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr)
2545
2546 {
2547         const u64 isc_mask = 0xffUL << 24; /* all iscs set */
2548         u32 schid;
2549
2550         if (attr->flags)
2551                 return -EINVAL;
2552         if (attr->attr != sizeof(schid))
2553                 return -EINVAL;
2554         if (copy_from_user(&schid, (void __user *) attr->addr, sizeof(schid)))
2555                 return -EFAULT;
2556         if (!schid)
2557                 return -EINVAL;
2558         kfree(kvm_s390_get_io_int(kvm, isc_mask, schid));
2559         /*
2560          * If userspace is conforming to the architecture, we can have at most
2561          * one pending I/O interrupt per subchannel, so this is effectively a
2562          * clear all.
2563          */
2564         return 0;
2565 }
2566
2567 static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr)
2568 {
2569         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2570         struct kvm_s390_ais_req req;
2571         int ret = 0;
2572
2573         if (!test_kvm_facility(kvm, 72))
2574                 return -EOPNOTSUPP;
2575
2576         if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
2577                 return -EFAULT;
2578
2579         if (req.isc > MAX_ISC)
2580                 return -EINVAL;
2581
2582         trace_kvm_s390_modify_ais_mode(req.isc,
2583                                        (fi->simm & AIS_MODE_MASK(req.isc)) ?
2584                                        (fi->nimm & AIS_MODE_MASK(req.isc)) ?
2585                                        2 : KVM_S390_AIS_MODE_SINGLE :
2586                                        KVM_S390_AIS_MODE_ALL, req.mode);
2587
2588         mutex_lock(&fi->ais_lock);
2589         switch (req.mode) {
2590         case KVM_S390_AIS_MODE_ALL:
2591                 fi->simm &= ~AIS_MODE_MASK(req.isc);
2592                 fi->nimm &= ~AIS_MODE_MASK(req.isc);
2593                 break;
2594         case KVM_S390_AIS_MODE_SINGLE:
2595                 fi->simm |= AIS_MODE_MASK(req.isc);
2596                 fi->nimm &= ~AIS_MODE_MASK(req.isc);
2597                 break;
2598         default:
2599                 ret = -EINVAL;
2600         }
2601         mutex_unlock(&fi->ais_lock);
2602
2603         return ret;
2604 }
2605
2606 static int kvm_s390_inject_airq(struct kvm *kvm,
2607                                 struct s390_io_adapter *adapter)
2608 {
2609         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2610         struct kvm_s390_interrupt s390int = {
2611                 .type = KVM_S390_INT_IO(1, 0, 0, 0),
2612                 .parm = 0,
2613                 .parm64 = isc_to_int_word(adapter->isc),
2614         };
2615         int ret = 0;
2616
2617         if (!test_kvm_facility(kvm, 72) || !adapter->suppressible)
2618                 return kvm_s390_inject_vm(kvm, &s390int);
2619
2620         mutex_lock(&fi->ais_lock);
2621         if (fi->nimm & AIS_MODE_MASK(adapter->isc)) {
2622                 trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc);
2623                 goto out;
2624         }
2625
2626         ret = kvm_s390_inject_vm(kvm, &s390int);
2627         if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) {
2628                 fi->nimm |= AIS_MODE_MASK(adapter->isc);
2629                 trace_kvm_s390_modify_ais_mode(adapter->isc,
2630                                                KVM_S390_AIS_MODE_SINGLE, 2);
2631         }
2632 out:
2633         mutex_unlock(&fi->ais_lock);
2634         return ret;
2635 }
2636
2637 static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr)
2638 {
2639         unsigned int id = attr->attr;
2640         struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2641
2642         if (!adapter)
2643                 return -EINVAL;
2644
2645         return kvm_s390_inject_airq(kvm, adapter);
2646 }
2647
2648 static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr)
2649 {
2650         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2651         struct kvm_s390_ais_all ais;
2652
2653         if (!test_kvm_facility(kvm, 72))
2654                 return -EOPNOTSUPP;
2655
2656         if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais)))
2657                 return -EFAULT;
2658
2659         mutex_lock(&fi->ais_lock);
2660         fi->simm = ais.simm;
2661         fi->nimm = ais.nimm;
2662         mutex_unlock(&fi->ais_lock);
2663
2664         return 0;
2665 }
2666
2667 static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2668 {
2669         int r = 0;
2670         unsigned long i;
2671         struct kvm_vcpu *vcpu;
2672
2673         switch (attr->group) {
2674         case KVM_DEV_FLIC_ENQUEUE:
2675                 r = enqueue_floating_irq(dev, attr);
2676                 break;
2677         case KVM_DEV_FLIC_CLEAR_IRQS:
2678                 kvm_s390_clear_float_irqs(dev->kvm);
2679                 break;
2680         case KVM_DEV_FLIC_APF_ENABLE:
2681                 dev->kvm->arch.gmap->pfault_enabled = 1;
2682                 break;
2683         case KVM_DEV_FLIC_APF_DISABLE_WAIT:
2684                 dev->kvm->arch.gmap->pfault_enabled = 0;
2685                 /*
2686                  * Make sure no async faults are in transition when
2687                  * clearing the queues. So we don't need to worry
2688                  * about late coming workers.
2689                  */
2690                 synchronize_srcu(&dev->kvm->srcu);
2691                 kvm_for_each_vcpu(i, vcpu, dev->kvm)
2692                         kvm_clear_async_pf_completion_queue(vcpu);
2693                 break;
2694         case KVM_DEV_FLIC_ADAPTER_REGISTER:
2695                 r = register_io_adapter(dev, attr);
2696                 break;
2697         case KVM_DEV_FLIC_ADAPTER_MODIFY:
2698                 r = modify_io_adapter(dev, attr);
2699                 break;
2700         case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2701                 r = clear_io_irq(dev->kvm, attr);
2702                 break;
2703         case KVM_DEV_FLIC_AISM:
2704                 r = modify_ais_mode(dev->kvm, attr);
2705                 break;
2706         case KVM_DEV_FLIC_AIRQ_INJECT:
2707                 r = flic_inject_airq(dev->kvm, attr);
2708                 break;
2709         case KVM_DEV_FLIC_AISM_ALL:
2710                 r = flic_ais_mode_set_all(dev->kvm, attr);
2711                 break;
2712         default:
2713                 r = -EINVAL;
2714         }
2715
2716         return r;
2717 }
2718
2719 static int flic_has_attr(struct kvm_device *dev,
2720                              struct kvm_device_attr *attr)
2721 {
2722         switch (attr->group) {
2723         case KVM_DEV_FLIC_GET_ALL_IRQS:
2724         case KVM_DEV_FLIC_ENQUEUE:
2725         case KVM_DEV_FLIC_CLEAR_IRQS:
2726         case KVM_DEV_FLIC_APF_ENABLE:
2727         case KVM_DEV_FLIC_APF_DISABLE_WAIT:
2728         case KVM_DEV_FLIC_ADAPTER_REGISTER:
2729         case KVM_DEV_FLIC_ADAPTER_MODIFY:
2730         case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2731         case KVM_DEV_FLIC_AISM:
2732         case KVM_DEV_FLIC_AIRQ_INJECT:
2733         case KVM_DEV_FLIC_AISM_ALL:
2734                 return 0;
2735         }
2736         return -ENXIO;
2737 }
2738
2739 static int flic_create(struct kvm_device *dev, u32 type)
2740 {
2741         if (!dev)
2742                 return -EINVAL;
2743         if (dev->kvm->arch.flic)
2744                 return -EINVAL;
2745         dev->kvm->arch.flic = dev;
2746         return 0;
2747 }
2748
2749 static void flic_destroy(struct kvm_device *dev)
2750 {
2751         dev->kvm->arch.flic = NULL;
2752         kfree(dev);
2753 }
2754
2755 /* s390 floating irq controller (flic) */
2756 struct kvm_device_ops kvm_flic_ops = {
2757         .name = "kvm-flic",
2758         .get_attr = flic_get_attr,
2759         .set_attr = flic_set_attr,
2760         .has_attr = flic_has_attr,
2761         .create = flic_create,
2762         .destroy = flic_destroy,
2763 };
2764
2765 static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap)
2766 {
2767         unsigned long bit;
2768
2769         bit = bit_nr + (addr % PAGE_SIZE) * 8;
2770
2771         return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit;
2772 }
2773
2774 static struct page *get_map_page(struct kvm *kvm, u64 uaddr)
2775 {
2776         struct page *page = NULL;
2777
2778         mmap_read_lock(kvm->mm);
2779         get_user_pages_remote(kvm->mm, uaddr, 1, FOLL_WRITE,
2780                               &page, NULL);
2781         mmap_read_unlock(kvm->mm);
2782         return page;
2783 }
2784
2785 static int adapter_indicators_set(struct kvm *kvm,
2786                                   struct s390_io_adapter *adapter,
2787                                   struct kvm_s390_adapter_int *adapter_int)
2788 {
2789         unsigned long bit;
2790         int summary_set, idx;
2791         struct page *ind_page, *summary_page;
2792         void *map;
2793
2794         ind_page = get_map_page(kvm, adapter_int->ind_addr);
2795         if (!ind_page)
2796                 return -1;
2797         summary_page = get_map_page(kvm, adapter_int->summary_addr);
2798         if (!summary_page) {
2799                 put_page(ind_page);
2800                 return -1;
2801         }
2802
2803         idx = srcu_read_lock(&kvm->srcu);
2804         map = page_address(ind_page);
2805         bit = get_ind_bit(adapter_int->ind_addr,
2806                           adapter_int->ind_offset, adapter->swap);
2807         set_bit(bit, map);
2808         mark_page_dirty(kvm, adapter_int->ind_addr >> PAGE_SHIFT);
2809         set_page_dirty_lock(ind_page);
2810         map = page_address(summary_page);
2811         bit = get_ind_bit(adapter_int->summary_addr,
2812                           adapter_int->summary_offset, adapter->swap);
2813         summary_set = test_and_set_bit(bit, map);
2814         mark_page_dirty(kvm, adapter_int->summary_addr >> PAGE_SHIFT);
2815         set_page_dirty_lock(summary_page);
2816         srcu_read_unlock(&kvm->srcu, idx);
2817
2818         put_page(ind_page);
2819         put_page(summary_page);
2820         return summary_set ? 0 : 1;
2821 }
2822
2823 /*
2824  * < 0 - not injected due to error
2825  * = 0 - coalesced, summary indicator already active
2826  * > 0 - injected interrupt
2827  */
2828 static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e,
2829                            struct kvm *kvm, int irq_source_id, int level,
2830                            bool line_status)
2831 {
2832         int ret;
2833         struct s390_io_adapter *adapter;
2834
2835         /* We're only interested in the 0->1 transition. */
2836         if (!level)
2837                 return 0;
2838         adapter = get_io_adapter(kvm, e->adapter.adapter_id);
2839         if (!adapter)
2840                 return -1;
2841         ret = adapter_indicators_set(kvm, adapter, &e->adapter);
2842         if ((ret > 0) && !adapter->masked) {
2843                 ret = kvm_s390_inject_airq(kvm, adapter);
2844                 if (ret == 0)
2845                         ret = 1;
2846         }
2847         return ret;
2848 }
2849
2850 /*
2851  * Inject the machine check to the guest.
2852  */
2853 void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu,
2854                                      struct mcck_volatile_info *mcck_info)
2855 {
2856         struct kvm_s390_interrupt_info inti;
2857         struct kvm_s390_irq irq;
2858         struct kvm_s390_mchk_info *mchk;
2859         union mci mci;
2860         __u64 cr14 = 0;         /* upper bits are not used */
2861         int rc;
2862
2863         mci.val = mcck_info->mcic;
2864         if (mci.sr)
2865                 cr14 |= CR14_RECOVERY_SUBMASK;
2866         if (mci.dg)
2867                 cr14 |= CR14_DEGRADATION_SUBMASK;
2868         if (mci.w)
2869                 cr14 |= CR14_WARNING_SUBMASK;
2870
2871         mchk = mci.ck ? &inti.mchk : &irq.u.mchk;
2872         mchk->cr14 = cr14;
2873         mchk->mcic = mcck_info->mcic;
2874         mchk->ext_damage_code = mcck_info->ext_damage_code;
2875         mchk->failing_storage_address = mcck_info->failing_storage_address;
2876         if (mci.ck) {
2877                 /* Inject the floating machine check */
2878                 inti.type = KVM_S390_MCHK;
2879                 rc = __inject_vm(vcpu->kvm, &inti);
2880         } else {
2881                 /* Inject the machine check to specified vcpu */
2882                 irq.type = KVM_S390_MCHK;
2883                 rc = kvm_s390_inject_vcpu(vcpu, &irq);
2884         }
2885         WARN_ON_ONCE(rc);
2886 }
2887
2888 int kvm_set_routing_entry(struct kvm *kvm,
2889                           struct kvm_kernel_irq_routing_entry *e,
2890                           const struct kvm_irq_routing_entry *ue)
2891 {
2892         u64 uaddr;
2893
2894         switch (ue->type) {
2895         /* we store the userspace addresses instead of the guest addresses */
2896         case KVM_IRQ_ROUTING_S390_ADAPTER:
2897                 e->set = set_adapter_int;
2898                 uaddr =  gmap_translate(kvm->arch.gmap, ue->u.adapter.summary_addr);
2899                 if (uaddr == -EFAULT)
2900                         return -EFAULT;
2901                 e->adapter.summary_addr = uaddr;
2902                 uaddr =  gmap_translate(kvm->arch.gmap, ue->u.adapter.ind_addr);
2903                 if (uaddr == -EFAULT)
2904                         return -EFAULT;
2905                 e->adapter.ind_addr = uaddr;
2906                 e->adapter.summary_offset = ue->u.adapter.summary_offset;
2907                 e->adapter.ind_offset = ue->u.adapter.ind_offset;
2908                 e->adapter.adapter_id = ue->u.adapter.adapter_id;
2909                 return 0;
2910         default:
2911                 return -EINVAL;
2912         }
2913 }
2914
2915 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
2916                 int irq_source_id, int level, bool line_status)
2917 {
2918         return -EINVAL;
2919 }
2920
2921 int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len)
2922 {
2923         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2924         struct kvm_s390_irq *buf;
2925         int r = 0;
2926         int n;
2927
2928         buf = vmalloc(len);
2929         if (!buf)
2930                 return -ENOMEM;
2931
2932         if (copy_from_user((void *) buf, irqstate, len)) {
2933                 r = -EFAULT;
2934                 goto out_free;
2935         }
2936
2937         /*
2938          * Don't allow setting the interrupt state
2939          * when there are already interrupts pending
2940          */
2941         spin_lock(&li->lock);
2942         if (li->pending_irqs) {
2943                 r = -EBUSY;
2944                 goto out_unlock;
2945         }
2946
2947         for (n = 0; n < len / sizeof(*buf); n++) {
2948                 r = do_inject_vcpu(vcpu, &buf[n]);
2949                 if (r)
2950                         break;
2951         }
2952
2953 out_unlock:
2954         spin_unlock(&li->lock);
2955 out_free:
2956         vfree(buf);
2957
2958         return r;
2959 }
2960
2961 static void store_local_irq(struct kvm_s390_local_interrupt *li,
2962                             struct kvm_s390_irq *irq,
2963                             unsigned long irq_type)
2964 {
2965         switch (irq_type) {
2966         case IRQ_PEND_MCHK_EX:
2967         case IRQ_PEND_MCHK_REP:
2968                 irq->type = KVM_S390_MCHK;
2969                 irq->u.mchk = li->irq.mchk;
2970                 break;
2971         case IRQ_PEND_PROG:
2972                 irq->type = KVM_S390_PROGRAM_INT;
2973                 irq->u.pgm = li->irq.pgm;
2974                 break;
2975         case IRQ_PEND_PFAULT_INIT:
2976                 irq->type = KVM_S390_INT_PFAULT_INIT;
2977                 irq->u.ext = li->irq.ext;
2978                 break;
2979         case IRQ_PEND_EXT_EXTERNAL:
2980                 irq->type = KVM_S390_INT_EXTERNAL_CALL;
2981                 irq->u.extcall = li->irq.extcall;
2982                 break;
2983         case IRQ_PEND_EXT_CLOCK_COMP:
2984                 irq->type = KVM_S390_INT_CLOCK_COMP;
2985                 break;
2986         case IRQ_PEND_EXT_CPU_TIMER:
2987                 irq->type = KVM_S390_INT_CPU_TIMER;
2988                 break;
2989         case IRQ_PEND_SIGP_STOP:
2990                 irq->type = KVM_S390_SIGP_STOP;
2991                 irq->u.stop = li->irq.stop;
2992                 break;
2993         case IRQ_PEND_RESTART:
2994                 irq->type = KVM_S390_RESTART;
2995                 break;
2996         case IRQ_PEND_SET_PREFIX:
2997                 irq->type = KVM_S390_SIGP_SET_PREFIX;
2998                 irq->u.prefix = li->irq.prefix;
2999                 break;
3000         }
3001 }
3002
3003 int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len)
3004 {
3005         int scn;
3006         DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS);
3007         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
3008         unsigned long pending_irqs;
3009         struct kvm_s390_irq irq;
3010         unsigned long irq_type;
3011         int cpuaddr;
3012         int n = 0;
3013
3014         spin_lock(&li->lock);
3015         pending_irqs = li->pending_irqs;
3016         memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending,
3017                sizeof(sigp_emerg_pending));
3018         spin_unlock(&li->lock);
3019
3020         for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) {
3021                 memset(&irq, 0, sizeof(irq));
3022                 if (irq_type == IRQ_PEND_EXT_EMERGENCY)
3023                         continue;
3024                 if (n + sizeof(irq) > len)
3025                         return -ENOBUFS;
3026                 store_local_irq(&vcpu->arch.local_int, &irq, irq_type);
3027                 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3028                         return -EFAULT;
3029                 n += sizeof(irq);
3030         }
3031
3032         if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) {
3033                 for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) {
3034                         memset(&irq, 0, sizeof(irq));
3035                         if (n + sizeof(irq) > len)
3036                                 return -ENOBUFS;
3037                         irq.type = KVM_S390_INT_EMERGENCY;
3038                         irq.u.emerg.code = cpuaddr;
3039                         if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3040                                 return -EFAULT;
3041                         n += sizeof(irq);
3042                 }
3043         }
3044
3045         if (sca_ext_call_pending(vcpu, &scn)) {
3046                 if (n + sizeof(irq) > len)
3047                         return -ENOBUFS;
3048                 memset(&irq, 0, sizeof(irq));
3049                 irq.type = KVM_S390_INT_EXTERNAL_CALL;
3050                 irq.u.extcall.code = scn;
3051                 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3052                         return -EFAULT;
3053                 n += sizeof(irq);
3054         }
3055
3056         return n;
3057 }
3058
3059 static void __airqs_kick_single_vcpu(struct kvm *kvm, u8 deliverable_mask)
3060 {
3061         int vcpu_idx, online_vcpus = atomic_read(&kvm->online_vcpus);
3062         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3063         struct kvm_vcpu *vcpu;
3064         u8 vcpu_isc_mask;
3065
3066         for_each_set_bit(vcpu_idx, kvm->arch.idle_mask, online_vcpus) {
3067                 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
3068                 if (psw_ioint_disabled(vcpu))
3069                         continue;
3070                 vcpu_isc_mask = (u8)(vcpu->arch.sie_block->gcr[6] >> 24);
3071                 if (deliverable_mask & vcpu_isc_mask) {
3072                         /* lately kicked but not yet running */
3073                         if (test_and_set_bit(vcpu_idx, gi->kicked_mask))
3074                                 return;
3075                         kvm_s390_vcpu_wakeup(vcpu);
3076                         return;
3077                 }
3078         }
3079 }
3080
3081 static enum hrtimer_restart gisa_vcpu_kicker(struct hrtimer *timer)
3082 {
3083         struct kvm_s390_gisa_interrupt *gi =
3084                 container_of(timer, struct kvm_s390_gisa_interrupt, timer);
3085         struct kvm *kvm =
3086                 container_of(gi->origin, struct sie_page2, gisa)->kvm;
3087         u8 pending_mask;
3088
3089         pending_mask = gisa_get_ipm_or_restore_iam(gi);
3090         if (pending_mask) {
3091                 __airqs_kick_single_vcpu(kvm, pending_mask);
3092                 hrtimer_forward_now(timer, ns_to_ktime(gi->expires));
3093                 return HRTIMER_RESTART;
3094         }
3095
3096         return HRTIMER_NORESTART;
3097 }
3098
3099 #define NULL_GISA_ADDR 0x00000000UL
3100 #define NONE_GISA_ADDR 0x00000001UL
3101 #define GISA_ADDR_MASK 0xfffff000UL
3102
3103 static void process_gib_alert_list(void)
3104 {
3105         struct kvm_s390_gisa_interrupt *gi;
3106         u32 final, gisa_phys, origin = 0UL;
3107         struct kvm_s390_gisa *gisa;
3108         struct kvm *kvm;
3109
3110         do {
3111                 /*
3112                  * If the NONE_GISA_ADDR is still stored in the alert list
3113                  * origin, we will leave the outer loop. No further GISA has
3114                  * been added to the alert list by millicode while processing
3115                  * the current alert list.
3116                  */
3117                 final = (origin & NONE_GISA_ADDR);
3118                 /*
3119                  * Cut off the alert list and store the NONE_GISA_ADDR in the
3120                  * alert list origin to avoid further GAL interruptions.
3121                  * A new alert list can be build up by millicode in parallel
3122                  * for guests not in the yet cut-off alert list. When in the
3123                  * final loop, store the NULL_GISA_ADDR instead. This will re-
3124                  * enable GAL interruptions on the host again.
3125                  */
3126                 origin = xchg(&gib->alert_list_origin,
3127                               (!final) ? NONE_GISA_ADDR : NULL_GISA_ADDR);
3128                 /*
3129                  * Loop through the just cut-off alert list and start the
3130                  * gisa timers to kick idle vcpus to consume the pending
3131                  * interruptions asap.
3132                  */
3133                 while (origin & GISA_ADDR_MASK) {
3134                         gisa_phys = origin;
3135                         gisa = phys_to_virt(gisa_phys);
3136                         origin = gisa->next_alert;
3137                         gisa->next_alert = gisa_phys;
3138                         kvm = container_of(gisa, struct sie_page2, gisa)->kvm;
3139                         gi = &kvm->arch.gisa_int;
3140                         if (hrtimer_active(&gi->timer))
3141                                 hrtimer_cancel(&gi->timer);
3142                         hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL);
3143                 }
3144         } while (!final);
3145
3146 }
3147
3148 void kvm_s390_gisa_clear(struct kvm *kvm)
3149 {
3150         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3151
3152         if (!gi->origin)
3153                 return;
3154         gisa_clear_ipm(gi->origin);
3155         VM_EVENT(kvm, 3, "gisa 0x%pK cleared", gi->origin);
3156 }
3157
3158 void kvm_s390_gisa_init(struct kvm *kvm)
3159 {
3160         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3161
3162         if (!css_general_characteristics.aiv)
3163                 return;
3164         gi->origin = &kvm->arch.sie_page2->gisa;
3165         gi->alert.mask = 0;
3166         spin_lock_init(&gi->alert.ref_lock);
3167         gi->expires = 50 * 1000; /* 50 usec */
3168         hrtimer_init(&gi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3169         gi->timer.function = gisa_vcpu_kicker;
3170         memset(gi->origin, 0, sizeof(struct kvm_s390_gisa));
3171         gi->origin->next_alert = (u32)virt_to_phys(gi->origin);
3172         VM_EVENT(kvm, 3, "gisa 0x%pK initialized", gi->origin);
3173 }
3174
3175 void kvm_s390_gisa_enable(struct kvm *kvm)
3176 {
3177         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3178         struct kvm_vcpu *vcpu;
3179         unsigned long i;
3180         u32 gisa_desc;
3181
3182         if (gi->origin)
3183                 return;
3184         kvm_s390_gisa_init(kvm);
3185         gisa_desc = kvm_s390_get_gisa_desc(kvm);
3186         if (!gisa_desc)
3187                 return;
3188         kvm_for_each_vcpu(i, vcpu, kvm) {
3189                 mutex_lock(&vcpu->mutex);
3190                 vcpu->arch.sie_block->gd = gisa_desc;
3191                 vcpu->arch.sie_block->eca |= ECA_AIV;
3192                 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3193                            vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3194                 mutex_unlock(&vcpu->mutex);
3195         }
3196 }
3197
3198 void kvm_s390_gisa_destroy(struct kvm *kvm)
3199 {
3200         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3201         struct kvm_s390_gisa *gisa = gi->origin;
3202
3203         if (!gi->origin)
3204                 return;
3205         WARN(gi->alert.mask != 0x00,
3206              "unexpected non zero alert.mask 0x%02x",
3207              gi->alert.mask);
3208         gi->alert.mask = 0x00;
3209         if (gisa_set_iam(gi->origin, gi->alert.mask))
3210                 process_gib_alert_list();
3211         hrtimer_cancel(&gi->timer);
3212         gi->origin = NULL;
3213         VM_EVENT(kvm, 3, "gisa 0x%pK destroyed", gisa);
3214 }
3215
3216 void kvm_s390_gisa_disable(struct kvm *kvm)
3217 {
3218         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3219         struct kvm_vcpu *vcpu;
3220         unsigned long i;
3221
3222         if (!gi->origin)
3223                 return;
3224         kvm_for_each_vcpu(i, vcpu, kvm) {
3225                 mutex_lock(&vcpu->mutex);
3226                 vcpu->arch.sie_block->eca &= ~ECA_AIV;
3227                 vcpu->arch.sie_block->gd = 0U;
3228                 mutex_unlock(&vcpu->mutex);
3229                 VCPU_EVENT(vcpu, 3, "AIV disabled for cpu %03u", vcpu->vcpu_id);
3230         }
3231         kvm_s390_gisa_destroy(kvm);
3232 }
3233
3234 /**
3235  * kvm_s390_gisc_register - register a guest ISC
3236  *
3237  * @kvm:  the kernel vm to work with
3238  * @gisc: the guest interruption sub class to register
3239  *
3240  * The function extends the vm specific alert mask to use.
3241  * The effective IAM mask in the GISA is updated as well
3242  * in case the GISA is not part of the GIB alert list.
3243  * It will be updated latest when the IAM gets restored
3244  * by gisa_get_ipm_or_restore_iam().
3245  *
3246  * Returns: the nonspecific ISC (NISC) the gib alert mechanism
3247  *          has registered with the channel subsystem.
3248  *          -ENODEV in case the vm uses no GISA
3249  *          -ERANGE in case the guest ISC is invalid
3250  */
3251 int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc)
3252 {
3253         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3254
3255         if (!gi->origin)
3256                 return -ENODEV;
3257         if (gisc > MAX_ISC)
3258                 return -ERANGE;
3259
3260         spin_lock(&gi->alert.ref_lock);
3261         gi->alert.ref_count[gisc]++;
3262         if (gi->alert.ref_count[gisc] == 1) {
3263                 gi->alert.mask |= 0x80 >> gisc;
3264                 gisa_set_iam(gi->origin, gi->alert.mask);
3265         }
3266         spin_unlock(&gi->alert.ref_lock);
3267
3268         return gib->nisc;
3269 }
3270 EXPORT_SYMBOL_GPL(kvm_s390_gisc_register);
3271
3272 /**
3273  * kvm_s390_gisc_unregister - unregister a guest ISC
3274  *
3275  * @kvm:  the kernel vm to work with
3276  * @gisc: the guest interruption sub class to register
3277  *
3278  * The function reduces the vm specific alert mask to use.
3279  * The effective IAM mask in the GISA is updated as well
3280  * in case the GISA is not part of the GIB alert list.
3281  * It will be updated latest when the IAM gets restored
3282  * by gisa_get_ipm_or_restore_iam().
3283  *
3284  * Returns: the nonspecific ISC (NISC) the gib alert mechanism
3285  *          has registered with the channel subsystem.
3286  *          -ENODEV in case the vm uses no GISA
3287  *          -ERANGE in case the guest ISC is invalid
3288  *          -EINVAL in case the guest ISC is not registered
3289  */
3290 int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc)
3291 {
3292         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3293         int rc = 0;
3294
3295         if (!gi->origin)
3296                 return -ENODEV;
3297         if (gisc > MAX_ISC)
3298                 return -ERANGE;
3299
3300         spin_lock(&gi->alert.ref_lock);
3301         if (gi->alert.ref_count[gisc] == 0) {
3302                 rc = -EINVAL;
3303                 goto out;
3304         }
3305         gi->alert.ref_count[gisc]--;
3306         if (gi->alert.ref_count[gisc] == 0) {
3307                 gi->alert.mask &= ~(0x80 >> gisc);
3308                 gisa_set_iam(gi->origin, gi->alert.mask);
3309         }
3310 out:
3311         spin_unlock(&gi->alert.ref_lock);
3312
3313         return rc;
3314 }
3315 EXPORT_SYMBOL_GPL(kvm_s390_gisc_unregister);
3316
3317 static void aen_host_forward(unsigned long si)
3318 {
3319         struct kvm_s390_gisa_interrupt *gi;
3320         struct zpci_gaite *gaite;
3321         struct kvm *kvm;
3322
3323         gaite = (struct zpci_gaite *)aift->gait +
3324                 (si * sizeof(struct zpci_gaite));
3325         if (gaite->count == 0)
3326                 return;
3327         if (gaite->aisb != 0)
3328                 set_bit_inv(gaite->aisbo, phys_to_virt(gaite->aisb));
3329
3330         kvm = kvm_s390_pci_si_to_kvm(aift, si);
3331         if (!kvm)
3332                 return;
3333         gi = &kvm->arch.gisa_int;
3334
3335         if (!(gi->origin->g1.simm & AIS_MODE_MASK(gaite->gisc)) ||
3336             !(gi->origin->g1.nimm & AIS_MODE_MASK(gaite->gisc))) {
3337                 gisa_set_ipm_gisc(gi->origin, gaite->gisc);
3338                 if (hrtimer_active(&gi->timer))
3339                         hrtimer_cancel(&gi->timer);
3340                 hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL);
3341                 kvm->stat.aen_forward++;
3342         }
3343 }
3344
3345 static void aen_process_gait(u8 isc)
3346 {
3347         bool found = false, first = true;
3348         union zpci_sic_iib iib = {{0}};
3349         unsigned long si, flags;
3350
3351         spin_lock_irqsave(&aift->gait_lock, flags);
3352
3353         if (!aift->gait) {
3354                 spin_unlock_irqrestore(&aift->gait_lock, flags);
3355                 return;
3356         }
3357
3358         for (si = 0;;) {
3359                 /* Scan adapter summary indicator bit vector */
3360                 si = airq_iv_scan(aift->sbv, si, airq_iv_end(aift->sbv));
3361                 if (si == -1UL) {
3362                         if (first || found) {
3363                                 /* Re-enable interrupts. */
3364                                 zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, isc,
3365                                                   &iib);
3366                                 first = found = false;
3367                         } else {
3368                                 /* Interrupts on and all bits processed */
3369                                 break;
3370                         }
3371                         found = false;
3372                         si = 0;
3373                         /* Scan again after re-enabling interrupts */
3374                         continue;
3375                 }
3376                 found = true;
3377                 aen_host_forward(si);
3378         }
3379
3380         spin_unlock_irqrestore(&aift->gait_lock, flags);
3381 }
3382
3383 static void gib_alert_irq_handler(struct airq_struct *airq,
3384                                   struct tpi_info *tpi_info)
3385 {
3386         struct tpi_adapter_info *info = (struct tpi_adapter_info *)tpi_info;
3387
3388         inc_irq_stat(IRQIO_GAL);
3389
3390         if ((info->forward || info->error) &&
3391             IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
3392                 aen_process_gait(info->isc);
3393                 if (info->aism != 0)
3394                         process_gib_alert_list();
3395         } else {
3396                 process_gib_alert_list();
3397         }
3398 }
3399
3400 static struct airq_struct gib_alert_irq = {
3401         .handler = gib_alert_irq_handler,
3402 };
3403
3404 void kvm_s390_gib_destroy(void)
3405 {
3406         if (!gib)
3407                 return;
3408         if (kvm_s390_pci_interp_allowed() && aift) {
3409                 mutex_lock(&aift->aift_lock);
3410                 kvm_s390_pci_aen_exit();
3411                 mutex_unlock(&aift->aift_lock);
3412         }
3413         chsc_sgib(0);
3414         unregister_adapter_interrupt(&gib_alert_irq);
3415         free_page((unsigned long)gib);
3416         gib = NULL;
3417 }
3418
3419 int __init kvm_s390_gib_init(u8 nisc)
3420 {
3421         u32 gib_origin;
3422         int rc = 0;
3423
3424         if (!css_general_characteristics.aiv) {
3425                 KVM_EVENT(3, "%s", "gib not initialized, no AIV facility");
3426                 goto out;
3427         }
3428
3429         gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
3430         if (!gib) {
3431                 rc = -ENOMEM;
3432                 goto out;
3433         }
3434
3435         gib_alert_irq.isc = nisc;
3436         if (register_adapter_interrupt(&gib_alert_irq)) {
3437                 pr_err("Registering the GIB alert interruption handler failed\n");
3438                 rc = -EIO;
3439                 goto out_free_gib;
3440         }
3441         /* adapter interrupts used for AP (applicable here) don't use the LSI */
3442         *gib_alert_irq.lsi_ptr = 0xff;
3443
3444         gib->nisc = nisc;
3445         gib_origin = virt_to_phys(gib);
3446         if (chsc_sgib(gib_origin)) {
3447                 pr_err("Associating the GIB with the AIV facility failed\n");
3448                 free_page((unsigned long)gib);
3449                 gib = NULL;
3450                 rc = -EIO;
3451                 goto out_unreg_gal;
3452         }
3453
3454         if (kvm_s390_pci_interp_allowed()) {
3455                 if (kvm_s390_pci_aen_init(nisc)) {
3456                         pr_err("Initializing AEN for PCI failed\n");
3457                         rc = -EIO;
3458                         goto out_unreg_gal;
3459                 }
3460         }
3461
3462         KVM_EVENT(3, "gib 0x%pK (nisc=%d) initialized", gib, gib->nisc);
3463         goto out;
3464
3465 out_unreg_gal:
3466         unregister_adapter_interrupt(&gib_alert_irq);
3467 out_free_gib:
3468         free_page((unsigned long)gib);
3469         gib = NULL;
3470 out:
3471         return rc;
3472 }