arch/s390/kvm/interrupt.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * handling kvm guest interrupts
   4  *
   5  * Copyright IBM Corp. 2008, 2020
   6  *
   7  *    Author(s): Carsten Otte <[email protected]>
   8  */
   9
  10 #define KMSG_COMPONENT "kvm-s390"
  11 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  12
  13 #include <linux/interrupt.h>
  14 #include <linux/kvm_host.h>
  15 #include <linux/hrtimer.h>
  16 #include <linux/mmu_context.h>
  17 #include <linux/nospec.h>
  18 #include <linux/signal.h>
  19 #include <linux/slab.h>
  20 #include <linux/bitmap.h>
  21 #include <linux/vmalloc.h>
  22 #include <asm/asm-offsets.h>
  23 #include <asm/dis.h>
  24 #include <linux/uaccess.h>
  25 #include <asm/sclp.h>
  26 #include <asm/isc.h>
  27 #include <asm/gmap.h>
  28 #include <asm/switch_to.h>
  29 #include <asm/nmi.h>
  30 #include <asm/airq.h>
  31 #include <asm/tpi.h>
  32 #include "kvm-s390.h"
  33 #include "gaccess.h"
  34 #include "trace-s390.h"
  35 #include "pci.h"
  36
  37 #define PFAULT_INIT 0x0600
  38 #define PFAULT_DONE 0x0680
  39 #define VIRTIO_PARAM 0x0d00
  40
  41 static struct kvm_s390_gib *gib;
  42
  43 /* handle external calls via sigp interpretation facility */
  44 static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id)
  45 {
  46         int c, scn;
  47
  48         if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND))
  49                 return 0;
  50
  51         BUG_ON(!kvm_s390_use_sca_entries());
  52         read_lock(&vcpu->kvm->arch.sca_lock);
  53         if (vcpu->kvm->arch.use_esca) {
  54                 struct esca_block *sca = vcpu->kvm->arch.sca;
  55                 union esca_sigp_ctrl sigp_ctrl =
  56                         sca->cpu[vcpu->vcpu_id].sigp_ctrl;
  57
  58                 c = sigp_ctrl.c;
  59                 scn = sigp_ctrl.scn;
  60         } else {
  61                 struct bsca_block *sca = vcpu->kvm->arch.sca;
  62                 union bsca_sigp_ctrl sigp_ctrl =
  63                         sca->cpu[vcpu->vcpu_id].sigp_ctrl;
  64
  65                 c = sigp_ctrl.c;
  66                 scn = sigp_ctrl.scn;
  67         }
  68         read_unlock(&vcpu->kvm->arch.sca_lock);
  69
  70         if (src_id)
  71                 *src_id = scn;
  72
  73         return c;
  74 }
  75
  76 static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
  77 {
  78         int expect, rc;
  79
  80         BUG_ON(!kvm_s390_use_sca_entries());
  81         read_lock(&vcpu->kvm->arch.sca_lock);
  82         if (vcpu->kvm->arch.use_esca) {
  83                 struct esca_block *sca = vcpu->kvm->arch.sca;
  84                 union esca_sigp_ctrl *sigp_ctrl =
  85                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
  86                 union esca_sigp_ctrl new_val = {0}, old_val;
  87
  88                 old_val = READ_ONCE(*sigp_ctrl);
  89                 new_val.scn = src_id;
  90                 new_val.c = 1;
  91                 old_val.c = 0;
  92
  93                 expect = old_val.value;
  94                 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
  95         } else {
  96                 struct bsca_block *sca = vcpu->kvm->arch.sca;
  97                 union bsca_sigp_ctrl *sigp_ctrl =
  98                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
  99                 union bsca_sigp_ctrl new_val = {0}, old_val;
 100
 101                 old_val = READ_ONCE(*sigp_ctrl);
 102                 new_val.scn = src_id;
 103                 new_val.c = 1;
 104                 old_val.c = 0;
 105
 106                 expect = old_val.value;
 107                 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
 108         }
 109         read_unlock(&vcpu->kvm->arch.sca_lock);
 110
 111         if (rc != expect) {
 112                 /* another external call is pending */
 113                 return -EBUSY;
 114         }
 115         kvm_s390_set_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
 116         return 0;
 117 }
 118
 119 static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
 120 {
 121         int rc, expect;
 122
 123         if (!kvm_s390_use_sca_entries())
 124                 return;
 125         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
 126         read_lock(&vcpu->kvm->arch.sca_lock);
 127         if (vcpu->kvm->arch.use_esca) {
 128                 struct esca_block *sca = vcpu->kvm->arch.sca;
 129                 union esca_sigp_ctrl *sigp_ctrl =
 130                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
 131                 union esca_sigp_ctrl old;
 132
 133                 old = READ_ONCE(*sigp_ctrl);
 134                 expect = old.value;
 135                 rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
 136         } else {
 137                 struct bsca_block *sca = vcpu->kvm->arch.sca;
 138                 union bsca_sigp_ctrl *sigp_ctrl =
 139                         &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
 140                 union bsca_sigp_ctrl old;
 141
 142                 old = READ_ONCE(*sigp_ctrl);
 143                 expect = old.value;
 144                 rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
 145         }
 146         read_unlock(&vcpu->kvm->arch.sca_lock);
 147         WARN_ON(rc != expect); /* cannot clear? */
 148 }
 149
 150 int psw_extint_disabled(struct kvm_vcpu *vcpu)
 151 {
 152         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
 153 }
 154
 155 static int psw_ioint_disabled(struct kvm_vcpu *vcpu)
 156 {
 157         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO);
 158 }
 159
 160 static int psw_mchk_disabled(struct kvm_vcpu *vcpu)
 161 {
 162         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK);
 163 }
 164
 165 static int psw_interrupts_disabled(struct kvm_vcpu *vcpu)
 166 {
 167         return psw_extint_disabled(vcpu) &&
 168                psw_ioint_disabled(vcpu) &&
 169                psw_mchk_disabled(vcpu);
 170 }
 171
 172 static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
 173 {
 174         if (psw_extint_disabled(vcpu) ||
 175             !(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
 176                 return 0;
 177         if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu))
 178                 /* No timer interrupts when single stepping */
 179                 return 0;
 180         return 1;
 181 }
 182
 183 static int ckc_irq_pending(struct kvm_vcpu *vcpu)
 184 {
 185         const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
 186         const u64 ckc = vcpu->arch.sie_block->ckc;
 187
 188         if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
 189                 if ((s64)ckc >= (s64)now)
 190                         return 0;
 191         } else if (ckc >= now) {
 192                 return 0;
 193         }
 194         return ckc_interrupts_enabled(vcpu);
 195 }
 196
 197 static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu)
 198 {
 199         return !psw_extint_disabled(vcpu) &&
 200                (vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK);
 201 }
 202
 203 static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
 204 {
 205         if (!cpu_timer_interrupts_enabled(vcpu))
 206                 return 0;
 207         return kvm_s390_get_cpu_timer(vcpu) >> 63;
 208 }
 209
 210 static uint64_t isc_to_isc_bits(int isc)
 211 {
 212         return (0x80 >> isc) << 24;
 213 }
 214
 215 static inline u32 isc_to_int_word(u8 isc)
 216 {
 217         return ((u32)isc << 27) | 0x80000000;
 218 }
 219
 220 static inline u8 int_word_to_isc(u32 int_word)
 221 {
 222         return (int_word & 0x38000000) >> 27;
 223 }
 224
 225 /*
 226  * To use atomic bitmap functions, we have to provide a bitmap address
 227  * that is u64 aligned. However, the ipm might be u32 aligned.
 228  * Therefore, we logically start the bitmap at the very beginning of the
 229  * struct and fixup the bit number.
 230  */
 231 #define IPM_BIT_OFFSET (offsetof(struct kvm_s390_gisa, ipm) * BITS_PER_BYTE)
 232
 233 /**
 234  * gisa_set_iam - change the GISA interruption alert mask
 235  *
 236  * @gisa: gisa to operate on
 237  * @iam: new IAM value to use
 238  *
 239  * Change the IAM atomically with the next alert address and the IPM
 240  * of the GISA if the GISA is not part of the GIB alert list. All three
 241  * fields are located in the first long word of the GISA.
 242  *
 243  * Returns: 0 on success
 244  *          -EBUSY in case the gisa is part of the alert list
 245  */
 246 static inline int gisa_set_iam(struct kvm_s390_gisa *gisa, u8 iam)
 247 {
 248         u64 word, _word;
 249
 250         do {
 251                 word = READ_ONCE(gisa->u64.word[0]);
 252                 if ((u64)gisa != word >> 32)
 253                         return -EBUSY;
 254                 _word = (word & ~0xffUL) | iam;
 255         } while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
 256
 257         return 0;
 258 }
 259
 260 /**
 261  * gisa_clear_ipm - clear the GISA interruption pending mask
 262  *
 263  * @gisa: gisa to operate on
 264  *
 265  * Clear the IPM atomically with the next alert address and the IAM
 266  * of the GISA unconditionally. All three fields are located in the
 267  * first long word of the GISA.
 268  */
 269 static inline void gisa_clear_ipm(struct kvm_s390_gisa *gisa)
 270 {
 271         u64 word, _word;
 272
 273         do {
 274                 word = READ_ONCE(gisa->u64.word[0]);
 275                 _word = word & ~(0xffUL << 24);
 276         } while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
 277 }
 278
 279 /**
 280  * gisa_get_ipm_or_restore_iam - return IPM or restore GISA IAM
 281  *
 282  * @gi: gisa interrupt struct to work on
 283  *
 284  * Atomically restores the interruption alert mask if none of the
 285  * relevant ISCs are pending and return the IPM.
 286  *
 287  * Returns: the relevant pending ISCs
 288  */
 289 static inline u8 gisa_get_ipm_or_restore_iam(struct kvm_s390_gisa_interrupt *gi)
 290 {
 291         u8 pending_mask, alert_mask;
 292         u64 word, _word;
 293
 294         do {
 295                 word = READ_ONCE(gi->origin->u64.word[0]);
 296                 alert_mask = READ_ONCE(gi->alert.mask);
 297                 pending_mask = (u8)(word >> 24) & alert_mask;
 298                 if (pending_mask)
 299                         return pending_mask;
 300                 _word = (word & ~0xffUL) | alert_mask;
 301         } while (cmpxchg(&gi->origin->u64.word[0], word, _word) != word);
 302
 303         return 0;
 304 }
 305
 306 static inline int gisa_in_alert_list(struct kvm_s390_gisa *gisa)
 307 {
 308         return READ_ONCE(gisa->next_alert) != (u32)virt_to_phys(gisa);
 309 }
 310
 311 static inline void gisa_set_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
 312 {
 313         set_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
 314 }
 315
 316 static inline u8 gisa_get_ipm(struct kvm_s390_gisa *gisa)
 317 {
 318         return READ_ONCE(gisa->ipm);
 319 }
 320
 321 static inline int gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
 322 {
 323         return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
 324 }
 325
 326 static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu)
 327 {
 328         unsigned long pending = vcpu->kvm->arch.float_int.pending_irqs |
 329                                 vcpu->arch.local_int.pending_irqs;
 330
 331         pending &= ~vcpu->kvm->arch.float_int.masked_irqs;
 332         return pending;
 333 }
 334
 335 static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu)
 336 {
 337         struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
 338         unsigned long pending_mask;
 339
 340         pending_mask = pending_irqs_no_gisa(vcpu);
 341         if (gi->origin)
 342                 pending_mask |= gisa_get_ipm(gi->origin) << IRQ_PEND_IO_ISC_7;
 343         return pending_mask;
 344 }
 345
 346 static inline int isc_to_irq_type(unsigned long isc)
 347 {
 348         return IRQ_PEND_IO_ISC_0 - isc;
 349 }
 350
 351 static inline int irq_type_to_isc(unsigned long irq_type)
 352 {
 353         return IRQ_PEND_IO_ISC_0 - irq_type;
 354 }
 355
 356 static unsigned long disable_iscs(struct kvm_vcpu *vcpu,
 357                                    unsigned long active_mask)
 358 {
 359         int i;
 360
 361         for (i = 0; i <= MAX_ISC; i++)
 362                 if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i)))
 363                         active_mask &= ~(1UL << (isc_to_irq_type(i)));
 364
 365         return active_mask;
 366 }
 367
 368 static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
 369 {
 370         unsigned long active_mask;
 371
 372         active_mask = pending_irqs(vcpu);
 373         if (!active_mask)
 374                 return 0;
 375
 376         if (psw_extint_disabled(vcpu))
 377                 active_mask &= ~IRQ_PEND_EXT_MASK;
 378         if (psw_ioint_disabled(vcpu))
 379                 active_mask &= ~IRQ_PEND_IO_MASK;
 380         else
 381                 active_mask = disable_iscs(vcpu, active_mask);
 382         if (!(vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
 383                 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask);
 384         if (!(vcpu->arch.sie_block->gcr[0] & CR0_EMERGENCY_SIGNAL_SUBMASK))
 385                 __clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask);
 386         if (!(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
 387                 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask);
 388         if (!(vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK))
 389                 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask);
 390         if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK)) {
 391                 __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask);
 392                 __clear_bit(IRQ_PEND_EXT_SERVICE_EV, &active_mask);
 393         }
 394         if (psw_mchk_disabled(vcpu))
 395                 active_mask &= ~IRQ_PEND_MCHK_MASK;
 396         /* PV guest cpus can have a single interruption injected at a time. */
 397         if (kvm_s390_pv_cpu_get_handle(vcpu) &&
 398             vcpu->arch.sie_block->iictl != IICTL_CODE_NONE)
 399                 active_mask &= ~(IRQ_PEND_EXT_II_MASK |
 400                                  IRQ_PEND_IO_MASK |
 401                                  IRQ_PEND_MCHK_MASK);
 402         /*
 403          * Check both floating and local interrupt's cr14 because
 404          * bit IRQ_PEND_MCHK_REP could be set in both cases.
 405          */
 406         if (!(vcpu->arch.sie_block->gcr[14] &
 407            (vcpu->kvm->arch.float_int.mchk.cr14 |
 408            vcpu->arch.local_int.irq.mchk.cr14)))
 409                 __clear_bit(IRQ_PEND_MCHK_REP, &active_mask);
 410
 411         /*
 412          * STOP irqs will never be actively delivered. They are triggered via
 413          * intercept requests and cleared when the stop intercept is performed.
 414          */
 415         __clear_bit(IRQ_PEND_SIGP_STOP, &active_mask);
 416
 417         return active_mask;
 418 }
 419
 420 static void __set_cpu_idle(struct kvm_vcpu *vcpu)
 421 {
 422         kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT);
 423         set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
 424 }
 425
 426 static void __unset_cpu_idle(struct kvm_vcpu *vcpu)
 427 {
 428         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT);
 429         clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
 430 }
 431
 432 static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
 433 {
 434         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IO_INT | CPUSTAT_EXT_INT |
 435                                       CPUSTAT_STOP_INT);
 436         vcpu->arch.sie_block->lctl = 0x0000;
 437         vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT);
 438
 439         if (guestdbg_enabled(vcpu)) {
 440                 vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 |
 441                                                LCTL_CR10 | LCTL_CR11);
 442                 vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT);
 443         }
 444 }
 445
 446 static void set_intercept_indicators_io(struct kvm_vcpu *vcpu)
 447 {
 448         if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_IO_MASK))
 449                 return;
 450         if (psw_ioint_disabled(vcpu))
 451                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT);
 452         else
 453                 vcpu->arch.sie_block->lctl |= LCTL_CR6;
 454 }
 455
 456 static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu)
 457 {
 458         if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_EXT_MASK))
 459                 return;
 460         if (psw_extint_disabled(vcpu))
 461                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
 462         else
 463                 vcpu->arch.sie_block->lctl |= LCTL_CR0;
 464 }
 465
 466 static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu)
 467 {
 468         if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_MCHK_MASK))
 469                 return;
 470         if (psw_mchk_disabled(vcpu))
 471                 vcpu->arch.sie_block->ictl |= ICTL_LPSW;
 472         else
 473                 vcpu->arch.sie_block->lctl |= LCTL_CR14;
 474 }
 475
 476 static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu)
 477 {
 478         if (kvm_s390_is_stop_irq_pending(vcpu))
 479                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
 480 }
 481
 482 /* Set interception request for non-deliverable interrupts */
 483 static void set_intercept_indicators(struct kvm_vcpu *vcpu)
 484 {
 485         set_intercept_indicators_io(vcpu);
 486         set_intercept_indicators_ext(vcpu);
 487         set_intercept_indicators_mchk(vcpu);
 488         set_intercept_indicators_stop(vcpu);
 489 }
 490
 491 static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu)
 492 {
 493         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 494         int rc = 0;
 495
 496         vcpu->stat.deliver_cputm++;
 497         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
 498                                          0, 0);
 499         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 500                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 501                 vcpu->arch.sie_block->eic = EXT_IRQ_CPU_TIMER;
 502         } else {
 503                 rc  = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER,
 504                                    (u16 *)__LC_EXT_INT_CODE);
 505                 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 506                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 507                                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 508                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 509                                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 510         }
 511         clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
 512         return rc ? -EFAULT : 0;
 513 }
 514
 515 static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu)
 516 {
 517         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 518         int rc = 0;
 519
 520         vcpu->stat.deliver_ckc++;
 521         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
 522                                          0, 0);
 523         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 524                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 525                 vcpu->arch.sie_block->eic = EXT_IRQ_CLK_COMP;
 526         } else {
 527                 rc  = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP,
 528                                    (u16 __user *)__LC_EXT_INT_CODE);
 529                 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 530                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 531                                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 532                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 533                                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 534         }
 535         clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
 536         return rc ? -EFAULT : 0;
 537 }
 538
 539 static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu)
 540 {
 541         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 542         struct kvm_s390_ext_info ext;
 543         int rc;
 544
 545         spin_lock(&li->lock);
 546         ext = li->irq.ext;
 547         clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
 548         li->irq.ext.ext_params2 = 0;
 549         spin_unlock(&li->lock);
 550
 551         VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx",
 552                    ext.ext_params2);
 553         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 554                                          KVM_S390_INT_PFAULT_INIT,
 555                                          0, ext.ext_params2);
 556
 557         rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE);
 558         rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR);
 559         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 560                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 561         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 562                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 563         rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2);
 564         return rc ? -EFAULT : 0;
 565 }
 566
 567 static int __write_machine_check(struct kvm_vcpu *vcpu,
 568                                  struct kvm_s390_mchk_info *mchk)
 569 {
 570         unsigned long ext_sa_addr;
 571         unsigned long lc;
 572         freg_t fprs[NUM_FPRS];
 573         union mci mci;
 574         int rc;
 575
 576         /*
 577          * All other possible payload for a machine check (e.g. the register
 578          * contents in the save area) will be handled by the ultravisor, as
 579          * the hypervisor does not not have the needed information for
 580          * protected guests.
 581          */
 582         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 583                 vcpu->arch.sie_block->iictl = IICTL_CODE_MCHK;
 584                 vcpu->arch.sie_block->mcic = mchk->mcic;
 585                 vcpu->arch.sie_block->faddr = mchk->failing_storage_address;
 586                 vcpu->arch.sie_block->edc = mchk->ext_damage_code;
 587                 return 0;
 588         }
 589
 590         mci.val = mchk->mcic;
 591         /* take care of lazy register loading */
 592         save_fpu_regs();
 593         save_access_regs(vcpu->run->s.regs.acrs);
 594         if (MACHINE_HAS_GS && vcpu->arch.gs_enabled)
 595                 save_gs_cb(current->thread.gs_cb);
 596
 597         /* Extended save area */
 598         rc = read_guest_lc(vcpu, __LC_MCESAD, &ext_sa_addr,
 599                            sizeof(unsigned long));
 600         /* Only bits 0 through 63-LC are used for address formation */
 601         lc = ext_sa_addr & MCESA_LC_MASK;
 602         if (test_kvm_facility(vcpu->kvm, 133)) {
 603                 switch (lc) {
 604                 case 0:
 605                 case 10:
 606                         ext_sa_addr &= ~0x3ffUL;
 607                         break;
 608                 case 11:
 609                         ext_sa_addr &= ~0x7ffUL;
 610                         break;
 611                 case 12:
 612                         ext_sa_addr &= ~0xfffUL;
 613                         break;
 614                 default:
 615                         ext_sa_addr = 0;
 616                         break;
 617                 }
 618         } else {
 619                 ext_sa_addr &= ~0x3ffUL;
 620         }
 621
 622         if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(vcpu->kvm, 129)) {
 623                 if (write_guest_abs(vcpu, ext_sa_addr, vcpu->run->s.regs.vrs,
 624                                     512))
 625                         mci.vr = 0;
 626         } else {
 627                 mci.vr = 0;
 628         }
 629         if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(vcpu->kvm, 133)
 630             && (lc == 11 || lc == 12)) {
 631                 if (write_guest_abs(vcpu, ext_sa_addr + 1024,
 632                                     &vcpu->run->s.regs.gscb, 32))
 633                         mci.gs = 0;
 634         } else {
 635                 mci.gs = 0;
 636         }
 637
 638         /* General interruption information */
 639         rc |= put_guest_lc(vcpu, 1, (u8 __user *) __LC_AR_MODE_ID);
 640         rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
 641                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 642         rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
 643                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 644         rc |= put_guest_lc(vcpu, mci.val, (u64 __user *) __LC_MCCK_CODE);
 645
 646         /* Register-save areas */
 647         if (MACHINE_HAS_VX) {
 648                 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
 649                 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, fprs, 128);
 650         } else {
 651                 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA,
 652                                      vcpu->run->s.regs.fprs, 128);
 653         }
 654         rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA,
 655                              vcpu->run->s.regs.gprs, 128);
 656         rc |= put_guest_lc(vcpu, current->thread.fpu.fpc,
 657                            (u32 __user *) __LC_FP_CREG_SAVE_AREA);
 658         rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr,
 659                            (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA);
 660         rc |= put_guest_lc(vcpu, kvm_s390_get_cpu_timer(vcpu),
 661                            (u64 __user *) __LC_CPU_TIMER_SAVE_AREA);
 662         rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->ckc >> 8,
 663                            (u64 __user *) __LC_CLOCK_COMP_SAVE_AREA);
 664         rc |= write_guest_lc(vcpu, __LC_AREGS_SAVE_AREA,
 665                              &vcpu->run->s.regs.acrs, 64);
 666         rc |= write_guest_lc(vcpu, __LC_CREGS_SAVE_AREA,
 667                              &vcpu->arch.sie_block->gcr, 128);
 668
 669         /* Extended interruption information */
 670         rc |= put_guest_lc(vcpu, mchk->ext_damage_code,
 671                            (u32 __user *) __LC_EXT_DAMAGE_CODE);
 672         rc |= put_guest_lc(vcpu, mchk->failing_storage_address,
 673                            (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
 674         rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, &mchk->fixed_logout,
 675                              sizeof(mchk->fixed_logout));
 676         return rc ? -EFAULT : 0;
 677 }
 678
 679 static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu)
 680 {
 681         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
 682         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 683         struct kvm_s390_mchk_info mchk = {};
 684         int deliver = 0;
 685         int rc = 0;
 686
 687         spin_lock(&fi->lock);
 688         spin_lock(&li->lock);
 689         if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) ||
 690             test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) {
 691                 /*
 692                  * If there was an exigent machine check pending, then any
 693                  * repressible machine checks that might have been pending
 694                  * are indicated along with it, so always clear bits for
 695                  * repressible and exigent interrupts
 696                  */
 697                 mchk = li->irq.mchk;
 698                 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
 699                 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
 700                 memset(&li->irq.mchk, 0, sizeof(mchk));
 701                 deliver = 1;
 702         }
 703         /*
 704          * We indicate floating repressible conditions along with
 705          * other pending conditions. Channel Report Pending and Channel
 706          * Subsystem damage are the only two and are indicated by
 707          * bits in mcic and masked in cr14.
 708          */
 709         if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
 710                 mchk.mcic |= fi->mchk.mcic;
 711                 mchk.cr14 |= fi->mchk.cr14;
 712                 memset(&fi->mchk, 0, sizeof(mchk));
 713                 deliver = 1;
 714         }
 715         spin_unlock(&li->lock);
 716         spin_unlock(&fi->lock);
 717
 718         if (deliver) {
 719                 VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx",
 720                            mchk.mcic);
 721                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 722                                                  KVM_S390_MCHK,
 723                                                  mchk.cr14, mchk.mcic);
 724                 vcpu->stat.deliver_machine_check++;
 725                 rc = __write_machine_check(vcpu, &mchk);
 726         }
 727         return rc;
 728 }
 729
 730 static int __must_check __deliver_restart(struct kvm_vcpu *vcpu)
 731 {
 732         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 733         int rc = 0;
 734
 735         VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart");
 736         vcpu->stat.deliver_restart_signal++;
 737         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
 738
 739         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 740                 vcpu->arch.sie_block->iictl = IICTL_CODE_RESTART;
 741         } else {
 742                 rc  = write_guest_lc(vcpu,
 743                                      offsetof(struct lowcore, restart_old_psw),
 744                                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 745                 rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw),
 746                                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 747         }
 748         clear_bit(IRQ_PEND_RESTART, &li->pending_irqs);
 749         return rc ? -EFAULT : 0;
 750 }
 751
 752 static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu)
 753 {
 754         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 755         struct kvm_s390_prefix_info prefix;
 756
 757         spin_lock(&li->lock);
 758         prefix = li->irq.prefix;
 759         li->irq.prefix.address = 0;
 760         clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
 761         spin_unlock(&li->lock);
 762
 763         vcpu->stat.deliver_prefix_signal++;
 764         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 765                                          KVM_S390_SIGP_SET_PREFIX,
 766                                          prefix.address, 0);
 767
 768         kvm_s390_set_prefix(vcpu, prefix.address);
 769         return 0;
 770 }
 771
 772 static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu)
 773 {
 774         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 775         int rc;
 776         int cpu_addr;
 777
 778         spin_lock(&li->lock);
 779         cpu_addr = find_first_bit(li->sigp_emerg_pending, KVM_MAX_VCPUS);
 780         clear_bit(cpu_addr, li->sigp_emerg_pending);
 781         if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS))
 782                 clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
 783         spin_unlock(&li->lock);
 784
 785         VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg");
 786         vcpu->stat.deliver_emergency_signal++;
 787         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
 788                                          cpu_addr, 0);
 789         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 790                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 791                 vcpu->arch.sie_block->eic = EXT_IRQ_EMERGENCY_SIG;
 792                 vcpu->arch.sie_block->extcpuaddr = cpu_addr;
 793                 return 0;
 794         }
 795
 796         rc  = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG,
 797                            (u16 *)__LC_EXT_INT_CODE);
 798         rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR);
 799         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 800                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 801         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 802                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 803         return rc ? -EFAULT : 0;
 804 }
 805
 806 static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu)
 807 {
 808         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 809         struct kvm_s390_extcall_info extcall;
 810         int rc;
 811
 812         spin_lock(&li->lock);
 813         extcall = li->irq.extcall;
 814         li->irq.extcall.code = 0;
 815         clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
 816         spin_unlock(&li->lock);
 817
 818         VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call");
 819         vcpu->stat.deliver_external_call++;
 820         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
 821                                          KVM_S390_INT_EXTERNAL_CALL,
 822                                          extcall.code, 0);
 823         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 824                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 825                 vcpu->arch.sie_block->eic = EXT_IRQ_EXTERNAL_CALL;
 826                 vcpu->arch.sie_block->extcpuaddr = extcall.code;
 827                 return 0;
 828         }
 829
 830         rc  = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL,
 831                            (u16 *)__LC_EXT_INT_CODE);
 832         rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR);
 833         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 834                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 835         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw,
 836                             sizeof(psw_t));
 837         return rc ? -EFAULT : 0;
 838 }
 839
 840 static int __deliver_prog_pv(struct kvm_vcpu *vcpu, u16 code)
 841 {
 842         switch (code) {
 843         case PGM_SPECIFICATION:
 844                 vcpu->arch.sie_block->iictl = IICTL_CODE_SPECIFICATION;
 845                 break;
 846         case PGM_OPERAND:
 847                 vcpu->arch.sie_block->iictl = IICTL_CODE_OPERAND;
 848                 break;
 849         default:
 850                 return -EINVAL;
 851         }
 852         return 0;
 853 }
 854
 855 static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
 856 {
 857         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 858         struct kvm_s390_pgm_info pgm_info;
 859         int rc = 0, nullifying = false;
 860         u16 ilen;
 861
 862         spin_lock(&li->lock);
 863         pgm_info = li->irq.pgm;
 864         clear_bit(IRQ_PEND_PROG, &li->pending_irqs);
 865         memset(&li->irq.pgm, 0, sizeof(pgm_info));
 866         spin_unlock(&li->lock);
 867
 868         ilen = pgm_info.flags & KVM_S390_PGM_FLAGS_ILC_MASK;
 869         VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilen:%d",
 870                    pgm_info.code, ilen);
 871         vcpu->stat.deliver_program++;
 872         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
 873                                          pgm_info.code, 0);
 874
 875         /* PER is handled by the ultravisor */
 876         if (kvm_s390_pv_cpu_is_protected(vcpu))
 877                 return __deliver_prog_pv(vcpu, pgm_info.code & ~PGM_PER);
 878
 879         switch (pgm_info.code & ~PGM_PER) {
 880         case PGM_AFX_TRANSLATION:
 881         case PGM_ASX_TRANSLATION:
 882         case PGM_EX_TRANSLATION:
 883         case PGM_LFX_TRANSLATION:
 884         case PGM_LSTE_SEQUENCE:
 885         case PGM_LSX_TRANSLATION:
 886         case PGM_LX_TRANSLATION:
 887         case PGM_PRIMARY_AUTHORITY:
 888         case PGM_SECONDARY_AUTHORITY:
 889                 nullifying = true;
 890                 fallthrough;
 891         case PGM_SPACE_SWITCH:
 892                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 893                                   (u64 *)__LC_TRANS_EXC_CODE);
 894                 break;
 895         case PGM_ALEN_TRANSLATION:
 896         case PGM_ALE_SEQUENCE:
 897         case PGM_ASTE_INSTANCE:
 898         case PGM_ASTE_SEQUENCE:
 899         case PGM_ASTE_VALIDITY:
 900         case PGM_EXTENDED_AUTHORITY:
 901                 rc = put_guest_lc(vcpu, pgm_info.exc_access_id,
 902                                   (u8 *)__LC_EXC_ACCESS_ID);
 903                 nullifying = true;
 904                 break;
 905         case PGM_ASCE_TYPE:
 906         case PGM_PAGE_TRANSLATION:
 907         case PGM_REGION_FIRST_TRANS:
 908         case PGM_REGION_SECOND_TRANS:
 909         case PGM_REGION_THIRD_TRANS:
 910         case PGM_SEGMENT_TRANSLATION:
 911                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 912                                   (u64 *)__LC_TRANS_EXC_CODE);
 913                 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
 914                                    (u8 *)__LC_EXC_ACCESS_ID);
 915                 rc |= put_guest_lc(vcpu, pgm_info.op_access_id,
 916                                    (u8 *)__LC_OP_ACCESS_ID);
 917                 nullifying = true;
 918                 break;
 919         case PGM_MONITOR:
 920                 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr,
 921                                   (u16 *)__LC_MON_CLASS_NR);
 922                 rc |= put_guest_lc(vcpu, pgm_info.mon_code,
 923                                    (u64 *)__LC_MON_CODE);
 924                 break;
 925         case PGM_VECTOR_PROCESSING:
 926         case PGM_DATA:
 927                 rc = put_guest_lc(vcpu, pgm_info.data_exc_code,
 928                                   (u32 *)__LC_DATA_EXC_CODE);
 929                 break;
 930         case PGM_PROTECTION:
 931                 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
 932                                   (u64 *)__LC_TRANS_EXC_CODE);
 933                 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
 934                                    (u8 *)__LC_EXC_ACCESS_ID);
 935                 break;
 936         case PGM_STACK_FULL:
 937         case PGM_STACK_EMPTY:
 938         case PGM_STACK_SPECIFICATION:
 939         case PGM_STACK_TYPE:
 940         case PGM_STACK_OPERATION:
 941         case PGM_TRACE_TABEL:
 942         case PGM_CRYPTO_OPERATION:
 943                 nullifying = true;
 944                 break;
 945         }
 946
 947         if (pgm_info.code & PGM_PER) {
 948                 rc |= put_guest_lc(vcpu, pgm_info.per_code,
 949                                    (u8 *) __LC_PER_CODE);
 950                 rc |= put_guest_lc(vcpu, pgm_info.per_atmid,
 951                                    (u8 *)__LC_PER_ATMID);
 952                 rc |= put_guest_lc(vcpu, pgm_info.per_address,
 953                                    (u64 *) __LC_PER_ADDRESS);
 954                 rc |= put_guest_lc(vcpu, pgm_info.per_access_id,
 955                                    (u8 *) __LC_PER_ACCESS_ID);
 956         }
 957
 958         if (nullifying && !(pgm_info.flags & KVM_S390_PGM_FLAGS_NO_REWIND))
 959                 kvm_s390_rewind_psw(vcpu, ilen);
 960
 961         /* bit 1+2 of the target are the ilc, so we can directly use ilen */
 962         rc |= put_guest_lc(vcpu, ilen, (u16 *) __LC_PGM_ILC);
 963         rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea,
 964                                  (u64 *) __LC_PGM_LAST_BREAK);
 965         rc |= put_guest_lc(vcpu, pgm_info.code,
 966                            (u16 *)__LC_PGM_INT_CODE);
 967         rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW,
 968                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 969         rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW,
 970                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 971         return rc ? -EFAULT : 0;
 972 }
 973
 974 #define SCCB_MASK 0xFFFFFFF8
 975 #define SCCB_EVENT_PENDING 0x3
 976
 977 static int write_sclp(struct kvm_vcpu *vcpu, u32 parm)
 978 {
 979         int rc;
 980
 981         if (kvm_s390_pv_cpu_get_handle(vcpu)) {
 982                 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
 983                 vcpu->arch.sie_block->eic = EXT_IRQ_SERVICE_SIG;
 984                 vcpu->arch.sie_block->eiparams = parm;
 985                 return 0;
 986         }
 987
 988         rc  = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE);
 989         rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
 990         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
 991                              &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 992         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
 993                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 994         rc |= put_guest_lc(vcpu, parm,
 995                            (u32 *)__LC_EXT_PARAMS);
 996
 997         return rc ? -EFAULT : 0;
 998 }
 999
1000 static int __must_check __deliver_service(struct kvm_vcpu *vcpu)
1001 {
1002         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1003         struct kvm_s390_ext_info ext;
1004
1005         spin_lock(&fi->lock);
1006         if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs) ||
1007             !(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) {
1008                 spin_unlock(&fi->lock);
1009                 return 0;
1010         }
1011         ext = fi->srv_signal;
1012         memset(&fi->srv_signal, 0, sizeof(ext));
1013         clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1014         clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1015         if (kvm_s390_pv_cpu_is_protected(vcpu))
1016                 set_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs);
1017         spin_unlock(&fi->lock);
1018
1019         VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x",
1020                    ext.ext_params);
1021         vcpu->stat.deliver_service_signal++;
1022         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
1023                                          ext.ext_params, 0);
1024
1025         return write_sclp(vcpu, ext.ext_params);
1026 }
1027
1028 static int __must_check __deliver_service_ev(struct kvm_vcpu *vcpu)
1029 {
1030         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1031         struct kvm_s390_ext_info ext;
1032
1033         spin_lock(&fi->lock);
1034         if (!(test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs))) {
1035                 spin_unlock(&fi->lock);
1036                 return 0;
1037         }
1038         ext = fi->srv_signal;
1039         /* only clear the event bit */
1040         fi->srv_signal.ext_params &= ~SCCB_EVENT_PENDING;
1041         clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1042         spin_unlock(&fi->lock);
1043
1044         VCPU_EVENT(vcpu, 4, "%s", "deliver: sclp parameter event");
1045         vcpu->stat.deliver_service_signal++;
1046         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
1047                                          ext.ext_params, 0);
1048
1049         return write_sclp(vcpu, SCCB_EVENT_PENDING);
1050 }
1051
1052 static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu)
1053 {
1054         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1055         struct kvm_s390_interrupt_info *inti;
1056         int rc = 0;
1057
1058         spin_lock(&fi->lock);
1059         inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT],
1060                                         struct kvm_s390_interrupt_info,
1061                                         list);
1062         if (inti) {
1063                 list_del(&inti->list);
1064                 fi->counters[FIRQ_CNTR_PFAULT] -= 1;
1065         }
1066         if (list_empty(&fi->lists[FIRQ_LIST_PFAULT]))
1067                 clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1068         spin_unlock(&fi->lock);
1069
1070         if (inti) {
1071                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1072                                                  KVM_S390_INT_PFAULT_DONE, 0,
1073                                                  inti->ext.ext_params2);
1074                 VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx",
1075                            inti->ext.ext_params2);
1076
1077                 rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
1078                                 (u16 *)__LC_EXT_INT_CODE);
1079                 rc |= put_guest_lc(vcpu, PFAULT_DONE,
1080                                 (u16 *)__LC_EXT_CPU_ADDR);
1081                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
1082                                 &vcpu->arch.sie_block->gpsw,
1083                                 sizeof(psw_t));
1084                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
1085                                 &vcpu->arch.sie_block->gpsw,
1086                                 sizeof(psw_t));
1087                 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
1088                                 (u64 *)__LC_EXT_PARAMS2);
1089                 kfree(inti);
1090         }
1091         return rc ? -EFAULT : 0;
1092 }
1093
1094 static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu)
1095 {
1096         struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1097         struct kvm_s390_interrupt_info *inti;
1098         int rc = 0;
1099
1100         spin_lock(&fi->lock);
1101         inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO],
1102                                         struct kvm_s390_interrupt_info,
1103                                         list);
1104         if (inti) {
1105                 VCPU_EVENT(vcpu, 4,
1106                            "deliver: virtio parm: 0x%x,parm64: 0x%llx",
1107                            inti->ext.ext_params, inti->ext.ext_params2);
1108                 vcpu->stat.deliver_virtio++;
1109                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1110                                 inti->type,
1111                                 inti->ext.ext_params,
1112                                 inti->ext.ext_params2);
1113                 list_del(&inti->list);
1114                 fi->counters[FIRQ_CNTR_VIRTIO] -= 1;
1115         }
1116         if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO]))
1117                 clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1118         spin_unlock(&fi->lock);
1119
1120         if (inti) {
1121                 rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
1122                                 (u16 *)__LC_EXT_INT_CODE);
1123                 rc |= put_guest_lc(vcpu, VIRTIO_PARAM,
1124                                 (u16 *)__LC_EXT_CPU_ADDR);
1125                 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
1126                                 &vcpu->arch.sie_block->gpsw,
1127                                 sizeof(psw_t));
1128                 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
1129                                 &vcpu->arch.sie_block->gpsw,
1130                                 sizeof(psw_t));
1131                 rc |= put_guest_lc(vcpu, inti->ext.ext_params,
1132                                 (u32 *)__LC_EXT_PARAMS);
1133                 rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
1134                                 (u64 *)__LC_EXT_PARAMS2);
1135                 kfree(inti);
1136         }
1137         return rc ? -EFAULT : 0;
1138 }
1139
1140 static int __do_deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info *io)
1141 {
1142         int rc;
1143
1144         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
1145                 vcpu->arch.sie_block->iictl = IICTL_CODE_IO;
1146                 vcpu->arch.sie_block->subchannel_id = io->subchannel_id;
1147                 vcpu->arch.sie_block->subchannel_nr = io->subchannel_nr;
1148                 vcpu->arch.sie_block->io_int_parm = io->io_int_parm;
1149                 vcpu->arch.sie_block->io_int_word = io->io_int_word;
1150                 return 0;
1151         }
1152
1153         rc  = put_guest_lc(vcpu, io->subchannel_id, (u16 *)__LC_SUBCHANNEL_ID);
1154         rc |= put_guest_lc(vcpu, io->subchannel_nr, (u16 *)__LC_SUBCHANNEL_NR);
1155         rc |= put_guest_lc(vcpu, io->io_int_parm, (u32 *)__LC_IO_INT_PARM);
1156         rc |= put_guest_lc(vcpu, io->io_int_word, (u32 *)__LC_IO_INT_WORD);
1157         rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
1158                              &vcpu->arch.sie_block->gpsw,
1159                              sizeof(psw_t));
1160         rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
1161                             &vcpu->arch.sie_block->gpsw,
1162                             sizeof(psw_t));
1163         return rc ? -EFAULT : 0;
1164 }
1165
1166 static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
1167                                      unsigned long irq_type)
1168 {
1169         struct list_head *isc_list;
1170         struct kvm_s390_float_interrupt *fi;
1171         struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
1172         struct kvm_s390_interrupt_info *inti = NULL;
1173         struct kvm_s390_io_info io;
1174         u32 isc;
1175         int rc = 0;
1176
1177         fi = &vcpu->kvm->arch.float_int;
1178
1179         spin_lock(&fi->lock);
1180         isc = irq_type_to_isc(irq_type);
1181         isc_list = &fi->lists[isc];
1182         inti = list_first_entry_or_null(isc_list,
1183                                         struct kvm_s390_interrupt_info,
1184                                         list);
1185         if (inti) {
1186                 if (inti->type & KVM_S390_INT_IO_AI_MASK)
1187                         VCPU_EVENT(vcpu, 4, "%s", "deliver: I/O (AI)");
1188                 else
1189                         VCPU_EVENT(vcpu, 4, "deliver: I/O %x ss %x schid %04x",
1190                         inti->io.subchannel_id >> 8,
1191                         inti->io.subchannel_id >> 1 & 0x3,
1192                         inti->io.subchannel_nr);
1193
1194                 vcpu->stat.deliver_io++;
1195                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1196                                 inti->type,
1197                                 ((__u32)inti->io.subchannel_id << 16) |
1198                                 inti->io.subchannel_nr,
1199                                 ((__u64)inti->io.io_int_parm << 32) |
1200                                 inti->io.io_int_word);
1201                 list_del(&inti->list);
1202                 fi->counters[FIRQ_CNTR_IO] -= 1;
1203         }
1204         if (list_empty(isc_list))
1205                 clear_bit(irq_type, &fi->pending_irqs);
1206         spin_unlock(&fi->lock);
1207
1208         if (inti) {
1209                 rc = __do_deliver_io(vcpu, &(inti->io));
1210                 kfree(inti);
1211                 goto out;
1212         }
1213
1214         if (gi->origin && gisa_tac_ipm_gisc(gi->origin, isc)) {
1215                 /*
1216                  * in case an adapter interrupt was not delivered
1217                  * in SIE context KVM will handle the delivery
1218                  */
1219                 VCPU_EVENT(vcpu, 4, "%s isc %u", "deliver: I/O (AI/gisa)", isc);
1220                 memset(&io, 0, sizeof(io));
1221                 io.io_int_word = isc_to_int_word(isc);
1222                 vcpu->stat.deliver_io++;
1223                 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1224                         KVM_S390_INT_IO(1, 0, 0, 0),
1225                         ((__u32)io.subchannel_id << 16) |
1226                         io.subchannel_nr,
1227                         ((__u64)io.io_int_parm << 32) |
1228                         io.io_int_word);
1229                 rc = __do_deliver_io(vcpu, &io);
1230         }
1231 out:
1232         return rc;
1233 }
1234
1235 /* Check whether an external call is pending (deliverable or not) */
1236 int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
1237 {
1238         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1239
1240         if (!sclp.has_sigpif)
1241                 return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
1242
1243         return sca_ext_call_pending(vcpu, NULL);
1244 }
1245
1246 int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
1247 {
1248         if (deliverable_irqs(vcpu))
1249                 return 1;
1250
1251         if (kvm_cpu_has_pending_timer(vcpu))
1252                 return 1;
1253
1254         /* external call pending and deliverable */
1255         if (kvm_s390_ext_call_pending(vcpu) &&
1256             !psw_extint_disabled(vcpu) &&
1257             (vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
1258                 return 1;
1259
1260         if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu))
1261                 return 1;
1262         return 0;
1263 }
1264
1265 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
1266 {
1267         return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu);
1268 }
1269
1270 static u64 __calculate_sltime(struct kvm_vcpu *vcpu)
1271 {
1272         const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
1273         const u64 ckc = vcpu->arch.sie_block->ckc;
1274         u64 cputm, sltime = 0;
1275
1276         if (ckc_interrupts_enabled(vcpu)) {
1277                 if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
1278                         if ((s64)now < (s64)ckc)
1279                                 sltime = tod_to_ns((s64)ckc - (s64)now);
1280                 } else if (now < ckc) {
1281                         sltime = tod_to_ns(ckc - now);
1282                 }
1283                 /* already expired */
1284                 if (!sltime)
1285                         return 0;
1286                 if (cpu_timer_interrupts_enabled(vcpu)) {
1287                         cputm = kvm_s390_get_cpu_timer(vcpu);
1288                         /* already expired? */
1289                         if (cputm >> 63)
1290                                 return 0;
1291                         return min_t(u64, sltime, tod_to_ns(cputm));
1292                 }
1293         } else if (cpu_timer_interrupts_enabled(vcpu)) {
1294                 sltime = kvm_s390_get_cpu_timer(vcpu);
1295                 /* already expired? */
1296                 if (sltime >> 63)
1297                         return 0;
1298         }
1299         return sltime;
1300 }
1301
1302 int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
1303 {
1304         struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
1305         u64 sltime;
1306
1307         vcpu->stat.exit_wait_state++;
1308
1309         /* fast path */
1310         if (kvm_arch_vcpu_runnable(vcpu))
1311                 return 0;
1312
1313         if (psw_interrupts_disabled(vcpu)) {
1314                 VCPU_EVENT(vcpu, 3, "%s", "disabled wait");
1315                 return -EOPNOTSUPP; /* disabled wait */
1316         }
1317
1318         if (gi->origin &&
1319             (gisa_get_ipm_or_restore_iam(gi) &
1320              vcpu->arch.sie_block->gcr[6] >> 24))
1321                 return 0;
1322
1323         if (!ckc_interrupts_enabled(vcpu) &&
1324             !cpu_timer_interrupts_enabled(vcpu)) {
1325                 VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer");
1326                 __set_cpu_idle(vcpu);
1327                 goto no_timer;
1328         }
1329
1330         sltime = __calculate_sltime(vcpu);
1331         if (!sltime)
1332                 return 0;
1333
1334         __set_cpu_idle(vcpu);
1335         hrtimer_start(&vcpu->arch.ckc_timer, sltime, HRTIMER_MODE_REL);
1336         VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime);
1337 no_timer:
1338         kvm_vcpu_srcu_read_unlock(vcpu);
1339         kvm_vcpu_halt(vcpu);
1340         vcpu->valid_wakeup = false;
1341         __unset_cpu_idle(vcpu);
1342         kvm_vcpu_srcu_read_lock(vcpu);
1343
1344         hrtimer_cancel(&vcpu->arch.ckc_timer);
1345         return 0;
1346 }
1347
1348 void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu)
1349 {
1350         vcpu->valid_wakeup = true;
1351         kvm_vcpu_wake_up(vcpu);
1352
1353         /*
1354          * The VCPU might not be sleeping but rather executing VSIE. Let's
1355          * kick it, so it leaves the SIE to process the request.
1356          */
1357         kvm_s390_vsie_kick(vcpu);
1358 }
1359
1360 enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer)
1361 {
1362         struct kvm_vcpu *vcpu;
1363         u64 sltime;
1364
1365         vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer);
1366         sltime = __calculate_sltime(vcpu);
1367
1368         /*
1369          * If the monotonic clock runs faster than the tod clock we might be
1370          * woken up too early and have to go back to sleep to avoid deadlocks.
1371          */
1372         if (sltime && hrtimer_forward_now(timer, ns_to_ktime(sltime)))
1373                 return HRTIMER_RESTART;
1374         kvm_s390_vcpu_wakeup(vcpu);
1375         return HRTIMER_NORESTART;
1376 }
1377
1378 void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu)
1379 {
1380         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1381
1382         spin_lock(&li->lock);
1383         li->pending_irqs = 0;
1384         bitmap_zero(li->sigp_emerg_pending, KVM_MAX_VCPUS);
1385         memset(&li->irq, 0, sizeof(li->irq));
1386         spin_unlock(&li->lock);
1387
1388         sca_clear_ext_call(vcpu);
1389 }
1390
1391 int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
1392 {
1393         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1394         int rc = 0;
1395         bool delivered = false;
1396         unsigned long irq_type;
1397         unsigned long irqs;
1398
1399         __reset_intercept_indicators(vcpu);
1400
1401         /* pending ckc conditions might have been invalidated */
1402         clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1403         if (ckc_irq_pending(vcpu))
1404                 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1405
1406         /* pending cpu timer conditions might have been invalidated */
1407         clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1408         if (cpu_timer_irq_pending(vcpu))
1409                 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1410
1411         while ((irqs = deliverable_irqs(vcpu)) && !rc) {
1412                 /* bits are in the reverse order of interrupt priority */
1413                 irq_type = find_last_bit(&irqs, IRQ_PEND_COUNT);
1414                 switch (irq_type) {
1415                 case IRQ_PEND_IO_ISC_0:
1416                 case IRQ_PEND_IO_ISC_1:
1417                 case IRQ_PEND_IO_ISC_2:
1418                 case IRQ_PEND_IO_ISC_3:
1419                 case IRQ_PEND_IO_ISC_4:
1420                 case IRQ_PEND_IO_ISC_5:
1421                 case IRQ_PEND_IO_ISC_6:
1422                 case IRQ_PEND_IO_ISC_7:
1423                         rc = __deliver_io(vcpu, irq_type);
1424                         break;
1425                 case IRQ_PEND_MCHK_EX:
1426                 case IRQ_PEND_MCHK_REP:
1427                         rc = __deliver_machine_check(vcpu);
1428                         break;
1429                 case IRQ_PEND_PROG:
1430                         rc = __deliver_prog(vcpu);
1431                         break;
1432                 case IRQ_PEND_EXT_EMERGENCY:
1433                         rc = __deliver_emergency_signal(vcpu);
1434                         break;
1435                 case IRQ_PEND_EXT_EXTERNAL:
1436                         rc = __deliver_external_call(vcpu);
1437                         break;
1438                 case IRQ_PEND_EXT_CLOCK_COMP:
1439                         rc = __deliver_ckc(vcpu);
1440                         break;
1441                 case IRQ_PEND_EXT_CPU_TIMER:
1442                         rc = __deliver_cpu_timer(vcpu);
1443                         break;
1444                 case IRQ_PEND_RESTART:
1445                         rc = __deliver_restart(vcpu);
1446                         break;
1447                 case IRQ_PEND_SET_PREFIX:
1448                         rc = __deliver_set_prefix(vcpu);
1449                         break;
1450                 case IRQ_PEND_PFAULT_INIT:
1451                         rc = __deliver_pfault_init(vcpu);
1452                         break;
1453                 case IRQ_PEND_EXT_SERVICE:
1454                         rc = __deliver_service(vcpu);
1455                         break;
1456                 case IRQ_PEND_EXT_SERVICE_EV:
1457                         rc = __deliver_service_ev(vcpu);
1458                         break;
1459                 case IRQ_PEND_PFAULT_DONE:
1460                         rc = __deliver_pfault_done(vcpu);
1461                         break;
1462                 case IRQ_PEND_VIRTIO:
1463                         rc = __deliver_virtio(vcpu);
1464                         break;
1465                 default:
1466                         WARN_ONCE(1, "Unknown pending irq type %ld", irq_type);
1467                         clear_bit(irq_type, &li->pending_irqs);
1468                 }
1469                 delivered |= !rc;
1470         }
1471
1472         /*
1473          * We delivered at least one interrupt and modified the PC. Force a
1474          * singlestep event now.
1475          */
1476         if (delivered && guestdbg_sstep_enabled(vcpu)) {
1477                 struct kvm_debug_exit_arch *debug_exit = &vcpu->run->debug.arch;
1478
1479                 debug_exit->addr = vcpu->arch.sie_block->gpsw.addr;
1480                 debug_exit->type = KVM_SINGLESTEP;
1481                 vcpu->guest_debug |= KVM_GUESTDBG_EXIT_PENDING;
1482         }
1483
1484         set_intercept_indicators(vcpu);
1485
1486         return rc;
1487 }
1488
1489 static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1490 {
1491         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1492
1493         vcpu->stat.inject_program++;
1494         VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code);
1495         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
1496                                    irq->u.pgm.code, 0);
1497
1498         if (!(irq->u.pgm.flags & KVM_S390_PGM_FLAGS_ILC_VALID)) {
1499                 /* auto detection if no valid ILC was given */
1500                 irq->u.pgm.flags &= ~KVM_S390_PGM_FLAGS_ILC_MASK;
1501                 irq->u.pgm.flags |= kvm_s390_get_ilen(vcpu);
1502                 irq->u.pgm.flags |= KVM_S390_PGM_FLAGS_ILC_VALID;
1503         }
1504
1505         if (irq->u.pgm.code == PGM_PER) {
1506                 li->irq.pgm.code |= PGM_PER;
1507                 li->irq.pgm.flags = irq->u.pgm.flags;
1508                 /* only modify PER related information */
1509                 li->irq.pgm.per_address = irq->u.pgm.per_address;
1510                 li->irq.pgm.per_code = irq->u.pgm.per_code;
1511                 li->irq.pgm.per_atmid = irq->u.pgm.per_atmid;
1512                 li->irq.pgm.per_access_id = irq->u.pgm.per_access_id;
1513         } else if (!(irq->u.pgm.code & PGM_PER)) {
1514                 li->irq.pgm.code = (li->irq.pgm.code & PGM_PER) |
1515                                    irq->u.pgm.code;
1516                 li->irq.pgm.flags = irq->u.pgm.flags;
1517                 /* only modify non-PER information */
1518                 li->irq.pgm.trans_exc_code = irq->u.pgm.trans_exc_code;
1519                 li->irq.pgm.mon_code = irq->u.pgm.mon_code;
1520                 li->irq.pgm.data_exc_code = irq->u.pgm.data_exc_code;
1521                 li->irq.pgm.mon_class_nr = irq->u.pgm.mon_class_nr;
1522                 li->irq.pgm.exc_access_id = irq->u.pgm.exc_access_id;
1523                 li->irq.pgm.op_access_id = irq->u.pgm.op_access_id;
1524         } else {
1525                 li->irq.pgm = irq->u.pgm;
1526         }
1527         set_bit(IRQ_PEND_PROG, &li->pending_irqs);
1528         return 0;
1529 }
1530
1531 static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1532 {
1533         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1534
1535         vcpu->stat.inject_pfault_init++;
1536         VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx",
1537                    irq->u.ext.ext_params2);
1538         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT,
1539                                    irq->u.ext.ext_params,
1540                                    irq->u.ext.ext_params2);
1541
1542         li->irq.ext = irq->u.ext;
1543         set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
1544         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1545         return 0;
1546 }
1547
1548 static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1549 {
1550         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1551         struct kvm_s390_extcall_info *extcall = &li->irq.extcall;
1552         uint16_t src_id = irq->u.extcall.code;
1553
1554         vcpu->stat.inject_external_call++;
1555         VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u",
1556                    src_id);
1557         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL,
1558                                    src_id, 0);
1559
1560         /* sending vcpu invalid */
1561         if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL)
1562                 return -EINVAL;
1563
1564         if (sclp.has_sigpif && !kvm_s390_pv_cpu_get_handle(vcpu))
1565                 return sca_inject_ext_call(vcpu, src_id);
1566
1567         if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs))
1568                 return -EBUSY;
1569         *extcall = irq->u.extcall;
1570         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1571         return 0;
1572 }
1573
1574 static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1575 {
1576         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1577         struct kvm_s390_prefix_info *prefix = &li->irq.prefix;
1578
1579         vcpu->stat.inject_set_prefix++;
1580         VCPU_EVENT(vcpu, 3, "inject: set prefix to %x",
1581                    irq->u.prefix.address);
1582         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX,
1583                                    irq->u.prefix.address, 0);
1584
1585         if (!is_vcpu_stopped(vcpu))
1586                 return -EBUSY;
1587
1588         *prefix = irq->u.prefix;
1589         set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
1590         return 0;
1591 }
1592
1593 #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS)
1594 static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1595 {
1596         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1597         struct kvm_s390_stop_info *stop = &li->irq.stop;
1598         int rc = 0;
1599
1600         vcpu->stat.inject_stop_signal++;
1601         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0);
1602
1603         if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS)
1604                 return -EINVAL;
1605
1606         if (is_vcpu_stopped(vcpu)) {
1607                 if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS)
1608                         rc = kvm_s390_store_status_unloaded(vcpu,
1609                                                 KVM_S390_STORE_STATUS_NOADDR);
1610                 return rc;
1611         }
1612
1613         if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs))
1614                 return -EBUSY;
1615         stop->flags = irq->u.stop.flags;
1616         kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
1617         return 0;
1618 }
1619
1620 static int __inject_sigp_restart(struct kvm_vcpu *vcpu)
1621 {
1622         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1623
1624         vcpu->stat.inject_restart++;
1625         VCPU_EVENT(vcpu, 3, "%s", "inject: restart int");
1626         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
1627
1628         set_bit(IRQ_PEND_RESTART, &li->pending_irqs);
1629         return 0;
1630 }
1631
1632 static int __inject_sigp_emergency(struct kvm_vcpu *vcpu,
1633                                    struct kvm_s390_irq *irq)
1634 {
1635         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1636
1637         vcpu->stat.inject_emergency_signal++;
1638         VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u",
1639                    irq->u.emerg.code);
1640         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
1641                                    irq->u.emerg.code, 0);
1642
1643         /* sending vcpu invalid */
1644         if (kvm_get_vcpu_by_id(vcpu->kvm, irq->u.emerg.code) == NULL)
1645                 return -EINVAL;
1646
1647         set_bit(irq->u.emerg.code, li->sigp_emerg_pending);
1648         set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
1649         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1650         return 0;
1651 }
1652
1653 static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1654 {
1655         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1656         struct kvm_s390_mchk_info *mchk = &li->irq.mchk;
1657
1658         vcpu->stat.inject_mchk++;
1659         VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx",
1660                    irq->u.mchk.mcic);
1661         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0,
1662                                    irq->u.mchk.mcic);
1663
1664         /*
1665          * Because repressible machine checks can be indicated along with
1666          * exigent machine checks (PoP, Chapter 11, Interruption action)
1667          * we need to combine cr14, mcic and external damage code.
1668          * Failing storage address and the logout area should not be or'ed
1669          * together, we just indicate the last occurrence of the corresponding
1670          * machine check
1671          */
1672         mchk->cr14 |= irq->u.mchk.cr14;
1673         mchk->mcic |= irq->u.mchk.mcic;
1674         mchk->ext_damage_code |= irq->u.mchk.ext_damage_code;
1675         mchk->failing_storage_address = irq->u.mchk.failing_storage_address;
1676         memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout,
1677                sizeof(mchk->fixed_logout));
1678         if (mchk->mcic & MCHK_EX_MASK)
1679                 set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
1680         else if (mchk->mcic & MCHK_REP_MASK)
1681                 set_bit(IRQ_PEND_MCHK_REP,  &li->pending_irqs);
1682         return 0;
1683 }
1684
1685 static int __inject_ckc(struct kvm_vcpu *vcpu)
1686 {
1687         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1688
1689         vcpu->stat.inject_ckc++;
1690         VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external");
1691         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
1692                                    0, 0);
1693
1694         set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1695         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1696         return 0;
1697 }
1698
1699 static int __inject_cpu_timer(struct kvm_vcpu *vcpu)
1700 {
1701         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1702
1703         vcpu->stat.inject_cputm++;
1704         VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external");
1705         trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
1706                                    0, 0);
1707
1708         set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1709         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1710         return 0;
1711 }
1712
1713 static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
1714                                                   int isc, u32 schid)
1715 {
1716         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1717         struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1718         struct kvm_s390_interrupt_info *iter;
1719         u16 id = (schid & 0xffff0000U) >> 16;
1720         u16 nr = schid & 0x0000ffffU;
1721
1722         spin_lock(&fi->lock);
1723         list_for_each_entry(iter, isc_list, list) {
1724                 if (schid && (id != iter->io.subchannel_id ||
1725                               nr != iter->io.subchannel_nr))
1726                         continue;
1727                 /* found an appropriate entry */
1728                 list_del_init(&iter->list);
1729                 fi->counters[FIRQ_CNTR_IO] -= 1;
1730                 if (list_empty(isc_list))
1731                         clear_bit(isc_to_irq_type(isc), &fi->pending_irqs);
1732                 spin_unlock(&fi->lock);
1733                 return iter;
1734         }
1735         spin_unlock(&fi->lock);
1736         return NULL;
1737 }
1738
1739 static struct kvm_s390_interrupt_info *get_top_io_int(struct kvm *kvm,
1740                                                       u64 isc_mask, u32 schid)
1741 {
1742         struct kvm_s390_interrupt_info *inti = NULL;
1743         int isc;
1744
1745         for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
1746                 if (isc_mask & isc_to_isc_bits(isc))
1747                         inti = get_io_int(kvm, isc, schid);
1748         }
1749         return inti;
1750 }
1751
1752 static int get_top_gisa_isc(struct kvm *kvm, u64 isc_mask, u32 schid)
1753 {
1754         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1755         unsigned long active_mask;
1756         int isc;
1757
1758         if (schid)
1759                 goto out;
1760         if (!gi->origin)
1761                 goto out;
1762
1763         active_mask = (isc_mask & gisa_get_ipm(gi->origin) << 24) << 32;
1764         while (active_mask) {
1765                 isc = __fls(active_mask) ^ (BITS_PER_LONG - 1);
1766                 if (gisa_tac_ipm_gisc(gi->origin, isc))
1767                         return isc;
1768                 clear_bit_inv(isc, &active_mask);
1769         }
1770 out:
1771         return -EINVAL;
1772 }
1773
1774 /*
1775  * Dequeue and return an I/O interrupt matching any of the interruption
1776  * subclasses as designated by the isc mask in cr6 and the schid (if != 0).
1777  * Take into account the interrupts pending in the interrupt list and in GISA.
1778  *
1779  * Note that for a guest that does not enable I/O interrupts
1780  * but relies on TPI, a flood of classic interrupts may starve
1781  * out adapter interrupts on the same isc. Linux does not do
1782  * that, and it is possible to work around the issue by configuring
1783  * different iscs for classic and adapter interrupts in the guest,
1784  * but we may want to revisit this in the future.
1785  */
1786 struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
1787                                                     u64 isc_mask, u32 schid)
1788 {
1789         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1790         struct kvm_s390_interrupt_info *inti, *tmp_inti;
1791         int isc;
1792
1793         inti = get_top_io_int(kvm, isc_mask, schid);
1794
1795         isc = get_top_gisa_isc(kvm, isc_mask, schid);
1796         if (isc < 0)
1797                 /* no AI in GISA */
1798                 goto out;
1799
1800         if (!inti)
1801                 /* AI in GISA but no classical IO int */
1802                 goto gisa_out;
1803
1804         /* both types of interrupts present */
1805         if (int_word_to_isc(inti->io.io_int_word) <= isc) {
1806                 /* classical IO int with higher priority */
1807                 gisa_set_ipm_gisc(gi->origin, isc);
1808                 goto out;
1809         }
1810 gisa_out:
1811         tmp_inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
1812         if (tmp_inti) {
1813                 tmp_inti->type = KVM_S390_INT_IO(1, 0, 0, 0);
1814                 tmp_inti->io.io_int_word = isc_to_int_word(isc);
1815                 if (inti)
1816                         kvm_s390_reinject_io_int(kvm, inti);
1817                 inti = tmp_inti;
1818         } else
1819                 gisa_set_ipm_gisc(gi->origin, isc);
1820 out:
1821         return inti;
1822 }
1823
1824 static int __inject_service(struct kvm *kvm,
1825                              struct kvm_s390_interrupt_info *inti)
1826 {
1827         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1828
1829         kvm->stat.inject_service_signal++;
1830         spin_lock(&fi->lock);
1831         fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING;
1832
1833         /* We always allow events, track them separately from the sccb ints */
1834         if (fi->srv_signal.ext_params & SCCB_EVENT_PENDING)
1835                 set_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1836
1837         /*
1838          * Early versions of the QEMU s390 bios will inject several
1839          * service interrupts after another without handling a
1840          * condition code indicating busy.
1841          * We will silently ignore those superfluous sccb values.
1842          * A future version of QEMU will take care of serialization
1843          * of servc requests
1844          */
1845         if (fi->srv_signal.ext_params & SCCB_MASK)
1846                 goto out;
1847         fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK;
1848         set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1849 out:
1850         spin_unlock(&fi->lock);
1851         kfree(inti);
1852         return 0;
1853 }
1854
1855 static int __inject_virtio(struct kvm *kvm,
1856                             struct kvm_s390_interrupt_info *inti)
1857 {
1858         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1859
1860         kvm->stat.inject_virtio++;
1861         spin_lock(&fi->lock);
1862         if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) {
1863                 spin_unlock(&fi->lock);
1864                 return -EBUSY;
1865         }
1866         fi->counters[FIRQ_CNTR_VIRTIO] += 1;
1867         list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]);
1868         set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1869         spin_unlock(&fi->lock);
1870         return 0;
1871 }
1872
1873 static int __inject_pfault_done(struct kvm *kvm,
1874                                  struct kvm_s390_interrupt_info *inti)
1875 {
1876         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1877
1878         kvm->stat.inject_pfault_done++;
1879         spin_lock(&fi->lock);
1880         if (fi->counters[FIRQ_CNTR_PFAULT] >=
1881                 (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) {
1882                 spin_unlock(&fi->lock);
1883                 return -EBUSY;
1884         }
1885         fi->counters[FIRQ_CNTR_PFAULT] += 1;
1886         list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]);
1887         set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1888         spin_unlock(&fi->lock);
1889         return 0;
1890 }
1891
1892 #define CR_PENDING_SUBCLASS 28
1893 static int __inject_float_mchk(struct kvm *kvm,
1894                                 struct kvm_s390_interrupt_info *inti)
1895 {
1896         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1897
1898         kvm->stat.inject_float_mchk++;
1899         spin_lock(&fi->lock);
1900         fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS);
1901         fi->mchk.mcic |= inti->mchk.mcic;
1902         set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs);
1903         spin_unlock(&fi->lock);
1904         kfree(inti);
1905         return 0;
1906 }
1907
1908 static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1909 {
1910         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1911         struct kvm_s390_float_interrupt *fi;
1912         struct list_head *list;
1913         int isc;
1914
1915         kvm->stat.inject_io++;
1916         isc = int_word_to_isc(inti->io.io_int_word);
1917
1918         /*
1919          * We do not use the lock checking variant as this is just a
1920          * performance optimization and we do not hold the lock here.
1921          * This is ok as the code will pick interrupts from both "lists"
1922          * for delivery.
1923          */
1924         if (gi->origin && inti->type & KVM_S390_INT_IO_AI_MASK) {
1925                 VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc);
1926                 gisa_set_ipm_gisc(gi->origin, isc);
1927                 kfree(inti);
1928                 return 0;
1929         }
1930
1931         fi = &kvm->arch.float_int;
1932         spin_lock(&fi->lock);
1933         if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) {
1934                 spin_unlock(&fi->lock);
1935                 return -EBUSY;
1936         }
1937         fi->counters[FIRQ_CNTR_IO] += 1;
1938
1939         if (inti->type & KVM_S390_INT_IO_AI_MASK)
1940                 VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)");
1941         else
1942                 VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x",
1943                         inti->io.subchannel_id >> 8,
1944                         inti->io.subchannel_id >> 1 & 0x3,
1945                         inti->io.subchannel_nr);
1946         list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1947         list_add_tail(&inti->list, list);
1948         set_bit(isc_to_irq_type(isc), &fi->pending_irqs);
1949         spin_unlock(&fi->lock);
1950         return 0;
1951 }
1952
1953 /*
1954  * Find a destination VCPU for a floating irq and kick it.
1955  */
1956 static void __floating_irq_kick(struct kvm *kvm, u64 type)
1957 {
1958         struct kvm_vcpu *dst_vcpu;
1959         int sigcpu, online_vcpus, nr_tries = 0;
1960
1961         online_vcpus = atomic_read(&kvm->online_vcpus);
1962         if (!online_vcpus)
1963                 return;
1964
1965         /* find idle VCPUs first, then round robin */
1966         sigcpu = find_first_bit(kvm->arch.idle_mask, online_vcpus);
1967         if (sigcpu == online_vcpus) {
1968                 do {
1969                         sigcpu = kvm->arch.float_int.next_rr_cpu++;
1970                         kvm->arch.float_int.next_rr_cpu %= online_vcpus;
1971                         /* avoid endless loops if all vcpus are stopped */
1972                         if (nr_tries++ >= online_vcpus)
1973                                 return;
1974                 } while (is_vcpu_stopped(kvm_get_vcpu(kvm, sigcpu)));
1975         }
1976         dst_vcpu = kvm_get_vcpu(kvm, sigcpu);
1977
1978         /* make the VCPU drop out of the SIE, or wake it up if sleeping */
1979         switch (type) {
1980         case KVM_S390_MCHK:
1981                 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT);
1982                 break;
1983         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1984                 if (!(type & KVM_S390_INT_IO_AI_MASK &&
1985                       kvm->arch.gisa_int.origin) ||
1986                       kvm_s390_pv_cpu_get_handle(dst_vcpu))
1987                         kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT);
1988                 break;
1989         default:
1990                 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT);
1991                 break;
1992         }
1993         kvm_s390_vcpu_wakeup(dst_vcpu);
1994 }
1995
1996 static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1997 {
1998         u64 type = READ_ONCE(inti->type);
1999         int rc;
2000
2001         switch (type) {
2002         case KVM_S390_MCHK:
2003                 rc = __inject_float_mchk(kvm, inti);
2004                 break;
2005         case KVM_S390_INT_VIRTIO:
2006                 rc = __inject_virtio(kvm, inti);
2007                 break;
2008         case KVM_S390_INT_SERVICE:
2009                 rc = __inject_service(kvm, inti);
2010                 break;
2011         case KVM_S390_INT_PFAULT_DONE:
2012                 rc = __inject_pfault_done(kvm, inti);
2013                 break;
2014         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2015                 rc = __inject_io(kvm, inti);
2016                 break;
2017         default:
2018                 rc = -EINVAL;
2019         }
2020         if (rc)
2021                 return rc;
2022
2023         __floating_irq_kick(kvm, type);
2024         return 0;
2025 }
2026
2027 int kvm_s390_inject_vm(struct kvm *kvm,
2028                        struct kvm_s390_interrupt *s390int)
2029 {
2030         struct kvm_s390_interrupt_info *inti;
2031         int rc;
2032
2033         inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
2034         if (!inti)
2035                 return -ENOMEM;
2036
2037         inti->type = s390int->type;
2038         switch (inti->type) {
2039         case KVM_S390_INT_VIRTIO:
2040                 VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx",
2041                          s390int->parm, s390int->parm64);
2042                 inti->ext.ext_params = s390int->parm;
2043                 inti->ext.ext_params2 = s390int->parm64;
2044                 break;
2045         case KVM_S390_INT_SERVICE:
2046                 VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm);
2047                 inti->ext.ext_params = s390int->parm;
2048                 break;
2049         case KVM_S390_INT_PFAULT_DONE:
2050                 inti->ext.ext_params2 = s390int->parm64;
2051                 break;
2052         case KVM_S390_MCHK:
2053                 VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx",
2054                          s390int->parm64);
2055                 inti->mchk.cr14 = s390int->parm; /* upper bits are not used */
2056                 inti->mchk.mcic = s390int->parm64;
2057                 break;
2058         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2059                 inti->io.subchannel_id = s390int->parm >> 16;
2060                 inti->io.subchannel_nr = s390int->parm & 0x0000ffffu;
2061                 inti->io.io_int_parm = s390int->parm64 >> 32;
2062                 inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull;
2063                 break;
2064         default:
2065                 kfree(inti);
2066                 return -EINVAL;
2067         }
2068         trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64,
2069                                  2);
2070
2071         rc = __inject_vm(kvm, inti);
2072         if (rc)
2073                 kfree(inti);
2074         return rc;
2075 }
2076
2077 int kvm_s390_reinject_io_int(struct kvm *kvm,
2078                               struct kvm_s390_interrupt_info *inti)
2079 {
2080         return __inject_vm(kvm, inti);
2081 }
2082
2083 int s390int_to_s390irq(struct kvm_s390_interrupt *s390int,
2084                        struct kvm_s390_irq *irq)
2085 {
2086         irq->type = s390int->type;
2087         switch (irq->type) {
2088         case KVM_S390_PROGRAM_INT:
2089                 if (s390int->parm & 0xffff0000)
2090                         return -EINVAL;
2091                 irq->u.pgm.code = s390int->parm;
2092                 break;
2093         case KVM_S390_SIGP_SET_PREFIX:
2094                 irq->u.prefix.address = s390int->parm;
2095                 break;
2096         case KVM_S390_SIGP_STOP:
2097                 irq->u.stop.flags = s390int->parm;
2098                 break;
2099         case KVM_S390_INT_EXTERNAL_CALL:
2100                 if (s390int->parm & 0xffff0000)
2101                         return -EINVAL;
2102                 irq->u.extcall.code = s390int->parm;
2103                 break;
2104         case KVM_S390_INT_EMERGENCY:
2105                 if (s390int->parm & 0xffff0000)
2106                         return -EINVAL;
2107                 irq->u.emerg.code = s390int->parm;
2108                 break;
2109         case KVM_S390_MCHK:
2110                 irq->u.mchk.mcic = s390int->parm64;
2111                 break;
2112         case KVM_S390_INT_PFAULT_INIT:
2113                 irq->u.ext.ext_params = s390int->parm;
2114                 irq->u.ext.ext_params2 = s390int->parm64;
2115                 break;
2116         case KVM_S390_RESTART:
2117         case KVM_S390_INT_CLOCK_COMP:
2118         case KVM_S390_INT_CPU_TIMER:
2119                 break;
2120         default:
2121                 return -EINVAL;
2122         }
2123         return 0;
2124 }
2125
2126 int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu)
2127 {
2128         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2129
2130         return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
2131 }
2132
2133 int kvm_s390_is_restart_irq_pending(struct kvm_vcpu *vcpu)
2134 {
2135         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2136
2137         return test_bit(IRQ_PEND_RESTART, &li->pending_irqs);
2138 }
2139
2140 void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu)
2141 {
2142         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2143
2144         spin_lock(&li->lock);
2145         li->irq.stop.flags = 0;
2146         clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
2147         spin_unlock(&li->lock);
2148 }
2149
2150 static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
2151 {
2152         int rc;
2153
2154         switch (irq->type) {
2155         case KVM_S390_PROGRAM_INT:
2156                 rc = __inject_prog(vcpu, irq);
2157                 break;
2158         case KVM_S390_SIGP_SET_PREFIX:
2159                 rc = __inject_set_prefix(vcpu, irq);
2160                 break;
2161         case KVM_S390_SIGP_STOP:
2162                 rc = __inject_sigp_stop(vcpu, irq);
2163                 break;
2164         case KVM_S390_RESTART:
2165                 rc = __inject_sigp_restart(vcpu);
2166                 break;
2167         case KVM_S390_INT_CLOCK_COMP:
2168                 rc = __inject_ckc(vcpu);
2169                 break;
2170         case KVM_S390_INT_CPU_TIMER:
2171                 rc = __inject_cpu_timer(vcpu);
2172                 break;
2173         case KVM_S390_INT_EXTERNAL_CALL:
2174                 rc = __inject_extcall(vcpu, irq);
2175                 break;
2176         case KVM_S390_INT_EMERGENCY:
2177                 rc = __inject_sigp_emergency(vcpu, irq);
2178                 break;
2179         case KVM_S390_MCHK:
2180                 rc = __inject_mchk(vcpu, irq);
2181                 break;
2182         case KVM_S390_INT_PFAULT_INIT:
2183                 rc = __inject_pfault_init(vcpu, irq);
2184                 break;
2185         case KVM_S390_INT_VIRTIO:
2186         case KVM_S390_INT_SERVICE:
2187         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2188         default:
2189                 rc = -EINVAL;
2190         }
2191
2192         return rc;
2193 }
2194
2195 int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
2196 {
2197         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2198         int rc;
2199
2200         spin_lock(&li->lock);
2201         rc = do_inject_vcpu(vcpu, irq);
2202         spin_unlock(&li->lock);
2203         if (!rc)
2204                 kvm_s390_vcpu_wakeup(vcpu);
2205         return rc;
2206 }
2207
2208 static inline void clear_irq_list(struct list_head *_list)
2209 {
2210         struct kvm_s390_interrupt_info *inti, *n;
2211
2212         list_for_each_entry_safe(inti, n, _list, list) {
2213                 list_del(&inti->list);
2214                 kfree(inti);
2215         }
2216 }
2217
2218 static void inti_to_irq(struct kvm_s390_interrupt_info *inti,
2219                        struct kvm_s390_irq *irq)
2220 {
2221         irq->type = inti->type;
2222         switch (inti->type) {
2223         case KVM_S390_INT_PFAULT_INIT:
2224         case KVM_S390_INT_PFAULT_DONE:
2225         case KVM_S390_INT_VIRTIO:
2226                 irq->u.ext = inti->ext;
2227                 break;
2228         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2229                 irq->u.io = inti->io;
2230                 break;
2231         }
2232 }
2233
2234 void kvm_s390_clear_float_irqs(struct kvm *kvm)
2235 {
2236         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2237         int i;
2238
2239         mutex_lock(&kvm->lock);
2240         if (!kvm_s390_pv_is_protected(kvm))
2241                 fi->masked_irqs = 0;
2242         mutex_unlock(&kvm->lock);
2243         spin_lock(&fi->lock);
2244         fi->pending_irqs = 0;
2245         memset(&fi->srv_signal, 0, sizeof(fi->srv_signal));
2246         memset(&fi->mchk, 0, sizeof(fi->mchk));
2247         for (i = 0; i < FIRQ_LIST_COUNT; i++)
2248                 clear_irq_list(&fi->lists[i]);
2249         for (i = 0; i < FIRQ_MAX_COUNT; i++)
2250                 fi->counters[i] = 0;
2251         spin_unlock(&fi->lock);
2252         kvm_s390_gisa_clear(kvm);
2253 };
2254
2255 static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
2256 {
2257         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
2258         struct kvm_s390_interrupt_info *inti;
2259         struct kvm_s390_float_interrupt *fi;
2260         struct kvm_s390_irq *buf;
2261         struct kvm_s390_irq *irq;
2262         int max_irqs;
2263         int ret = 0;
2264         int n = 0;
2265         int i;
2266
2267         if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0)
2268                 return -EINVAL;
2269
2270         /*
2271          * We are already using -ENOMEM to signal
2272          * userspace it may retry with a bigger buffer,
2273          * so we need to use something else for this case
2274          */
2275         buf = vzalloc(len);
2276         if (!buf)
2277                 return -ENOBUFS;
2278
2279         max_irqs = len / sizeof(struct kvm_s390_irq);
2280
2281         if (gi->origin && gisa_get_ipm(gi->origin)) {
2282                 for (i = 0; i <= MAX_ISC; i++) {
2283                         if (n == max_irqs) {
2284                                 /* signal userspace to try again */
2285                                 ret = -ENOMEM;
2286                                 goto out_nolock;
2287                         }
2288                         if (gisa_tac_ipm_gisc(gi->origin, i)) {
2289                                 irq = (struct kvm_s390_irq *) &buf[n];
2290                                 irq->type = KVM_S390_INT_IO(1, 0, 0, 0);
2291                                 irq->u.io.io_int_word = isc_to_int_word(i);
2292                                 n++;
2293                         }
2294                 }
2295         }
2296         fi = &kvm->arch.float_int;
2297         spin_lock(&fi->lock);
2298         for (i = 0; i < FIRQ_LIST_COUNT; i++) {
2299                 list_for_each_entry(inti, &fi->lists[i], list) {
2300                         if (n == max_irqs) {
2301                                 /* signal userspace to try again */
2302                                 ret = -ENOMEM;
2303                                 goto out;
2304                         }
2305                         inti_to_irq(inti, &buf[n]);
2306                         n++;
2307                 }
2308         }
2309         if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs) ||
2310             test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs)) {
2311                 if (n == max_irqs) {
2312                         /* signal userspace to try again */
2313                         ret = -ENOMEM;
2314                         goto out;
2315                 }
2316                 irq = (struct kvm_s390_irq *) &buf[n];
2317                 irq->type = KVM_S390_INT_SERVICE;
2318                 irq->u.ext = fi->srv_signal;
2319                 n++;
2320         }
2321         if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
2322                 if (n == max_irqs) {
2323                                 /* signal userspace to try again */
2324                                 ret = -ENOMEM;
2325                                 goto out;
2326                 }
2327                 irq = (struct kvm_s390_irq *) &buf[n];
2328                 irq->type = KVM_S390_MCHK;
2329                 irq->u.mchk = fi->mchk;
2330                 n++;
2331 }
2332
2333 out:
2334         spin_unlock(&fi->lock);
2335 out_nolock:
2336         if (!ret && n > 0) {
2337                 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n))
2338                         ret = -EFAULT;
2339         }
2340         vfree(buf);
2341
2342         return ret < 0 ? ret : n;
2343 }
2344
2345 static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr)
2346 {
2347         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2348         struct kvm_s390_ais_all ais;
2349
2350         if (attr->attr < sizeof(ais))
2351                 return -EINVAL;
2352
2353         if (!test_kvm_facility(kvm, 72))
2354                 return -EOPNOTSUPP;
2355
2356         mutex_lock(&fi->ais_lock);
2357         ais.simm = fi->simm;
2358         ais.nimm = fi->nimm;
2359         mutex_unlock(&fi->ais_lock);
2360
2361         if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais)))
2362                 return -EFAULT;
2363
2364         return 0;
2365 }
2366
2367 static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2368 {
2369         int r;
2370
2371         switch (attr->group) {
2372         case KVM_DEV_FLIC_GET_ALL_IRQS:
2373                 r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr,
2374                                           attr->attr);
2375                 break;
2376         case KVM_DEV_FLIC_AISM_ALL:
2377                 r = flic_ais_mode_get_all(dev->kvm, attr);
2378                 break;
2379         default:
2380                 r = -EINVAL;
2381         }
2382
2383         return r;
2384 }
2385
2386 static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti,
2387                                      u64 addr)
2388 {
2389         struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr;
2390         void *target = NULL;
2391         void __user *source;
2392         u64 size;
2393
2394         if (get_user(inti->type, (u64 __user *)addr))
2395                 return -EFAULT;
2396
2397         switch (inti->type) {
2398         case KVM_S390_INT_PFAULT_INIT:
2399         case KVM_S390_INT_PFAULT_DONE:
2400         case KVM_S390_INT_VIRTIO:
2401         case KVM_S390_INT_SERVICE:
2402                 target = (void *) &inti->ext;
2403                 source = &uptr->u.ext;
2404                 size = sizeof(inti->ext);
2405                 break;
2406         case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2407                 target = (void *) &inti->io;
2408                 source = &uptr->u.io;
2409                 size = sizeof(inti->io);
2410                 break;
2411         case KVM_S390_MCHK:
2412                 target = (void *) &inti->mchk;
2413                 source = &uptr->u.mchk;
2414                 size = sizeof(inti->mchk);
2415                 break;
2416         default:
2417                 return -EINVAL;
2418         }
2419
2420         if (copy_from_user(target, source, size))
2421                 return -EFAULT;
2422
2423         return 0;
2424 }
2425
2426 static int enqueue_floating_irq(struct kvm_device *dev,
2427                                 struct kvm_device_attr *attr)
2428 {
2429         struct kvm_s390_interrupt_info *inti = NULL;
2430         int r = 0;
2431         int len = attr->attr;
2432
2433         if (len % sizeof(struct kvm_s390_irq) != 0)
2434                 return -EINVAL;
2435         else if (len > KVM_S390_FLIC_MAX_BUFFER)
2436                 return -EINVAL;
2437
2438         while (len >= sizeof(struct kvm_s390_irq)) {
2439                 inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
2440                 if (!inti)
2441                         return -ENOMEM;
2442
2443                 r = copy_irq_from_user(inti, attr->addr);
2444                 if (r) {
2445                         kfree(inti);
2446                         return r;
2447                 }
2448                 r = __inject_vm(dev->kvm, inti);
2449                 if (r) {
2450                         kfree(inti);
2451                         return r;
2452                 }
2453                 len -= sizeof(struct kvm_s390_irq);
2454                 attr->addr += sizeof(struct kvm_s390_irq);
2455         }
2456
2457         return r;
2458 }
2459
2460 static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id)
2461 {
2462         if (id >= MAX_S390_IO_ADAPTERS)
2463                 return NULL;
2464         id = array_index_nospec(id, MAX_S390_IO_ADAPTERS);
2465         return kvm->arch.adapters[id];
2466 }
2467
2468 static int register_io_adapter(struct kvm_device *dev,
2469                                struct kvm_device_attr *attr)
2470 {
2471         struct s390_io_adapter *adapter;
2472         struct kvm_s390_io_adapter adapter_info;
2473
2474         if (copy_from_user(&adapter_info,
2475                            (void __user *)attr->addr, sizeof(adapter_info)))
2476                 return -EFAULT;
2477
2478         if (adapter_info.id >= MAX_S390_IO_ADAPTERS)
2479                 return -EINVAL;
2480
2481         adapter_info.id = array_index_nospec(adapter_info.id,
2482                                              MAX_S390_IO_ADAPTERS);
2483
2484         if (dev->kvm->arch.adapters[adapter_info.id] != NULL)
2485                 return -EINVAL;
2486
2487         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL_ACCOUNT);
2488         if (!adapter)
2489                 return -ENOMEM;
2490
2491         adapter->id = adapter_info.id;
2492         adapter->isc = adapter_info.isc;
2493         adapter->maskable = adapter_info.maskable;
2494         adapter->masked = false;
2495         adapter->swap = adapter_info.swap;
2496         adapter->suppressible = (adapter_info.flags) &
2497                                 KVM_S390_ADAPTER_SUPPRESSIBLE;
2498         dev->kvm->arch.adapters[adapter->id] = adapter;
2499
2500         return 0;
2501 }
2502
2503 int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked)
2504 {
2505         int ret;
2506         struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2507
2508         if (!adapter || !adapter->maskable)
2509                 return -EINVAL;
2510         ret = adapter->masked;
2511         adapter->masked = masked;
2512         return ret;
2513 }
2514
2515 void kvm_s390_destroy_adapters(struct kvm *kvm)
2516 {
2517         int i;
2518
2519         for (i = 0; i < MAX_S390_IO_ADAPTERS; i++)
2520                 kfree(kvm->arch.adapters[i]);
2521 }
2522
2523 static int modify_io_adapter(struct kvm_device *dev,
2524                              struct kvm_device_attr *attr)
2525 {
2526         struct kvm_s390_io_adapter_req req;
2527         struct s390_io_adapter *adapter;
2528         int ret;
2529
2530         if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
2531                 return -EFAULT;
2532
2533         adapter = get_io_adapter(dev->kvm, req.id);
2534         if (!adapter)
2535                 return -EINVAL;
2536         switch (req.type) {
2537         case KVM_S390_IO_ADAPTER_MASK:
2538                 ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask);
2539                 if (ret > 0)
2540                         ret = 0;
2541                 break;
2542         /*
2543          * The following operations are no longer needed and therefore no-ops.
2544          * The gpa to hva translation is done when an IRQ route is set up. The
2545          * set_irq code uses get_user_pages_remote() to do the actual write.
2546          */
2547         case KVM_S390_IO_ADAPTER_MAP:
2548         case KVM_S390_IO_ADAPTER_UNMAP:
2549                 ret = 0;
2550                 break;
2551         default:
2552                 ret = -EINVAL;
2553         }
2554
2555         return ret;
2556 }
2557
2558 static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr)
2559
2560 {
2561         const u64 isc_mask = 0xffUL << 24; /* all iscs set */
2562         u32 schid;
2563
2564         if (attr->flags)
2565                 return -EINVAL;
2566         if (attr->attr != sizeof(schid))
2567                 return -EINVAL;
2568         if (copy_from_user(&schid, (void __user *) attr->addr, sizeof(schid)))
2569                 return -EFAULT;
2570         if (!schid)
2571                 return -EINVAL;
2572         kfree(kvm_s390_get_io_int(kvm, isc_mask, schid));
2573         /*
2574          * If userspace is conforming to the architecture, we can have at most
2575          * one pending I/O interrupt per subchannel, so this is effectively a
2576          * clear all.
2577          */
2578         return 0;
2579 }
2580
2581 static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr)
2582 {
2583         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2584         struct kvm_s390_ais_req req;
2585         int ret = 0;
2586
2587         if (!test_kvm_facility(kvm, 72))
2588                 return -EOPNOTSUPP;
2589
2590         if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
2591                 return -EFAULT;
2592
2593         if (req.isc > MAX_ISC)
2594                 return -EINVAL;
2595
2596         trace_kvm_s390_modify_ais_mode(req.isc,
2597                                        (fi->simm & AIS_MODE_MASK(req.isc)) ?
2598                                        (fi->nimm & AIS_MODE_MASK(req.isc)) ?
2599                                        2 : KVM_S390_AIS_MODE_SINGLE :
2600                                        KVM_S390_AIS_MODE_ALL, req.mode);
2601
2602         mutex_lock(&fi->ais_lock);
2603         switch (req.mode) {
2604         case KVM_S390_AIS_MODE_ALL:
2605                 fi->simm &= ~AIS_MODE_MASK(req.isc);
2606                 fi->nimm &= ~AIS_MODE_MASK(req.isc);
2607                 break;
2608         case KVM_S390_AIS_MODE_SINGLE:
2609                 fi->simm |= AIS_MODE_MASK(req.isc);
2610                 fi->nimm &= ~AIS_MODE_MASK(req.isc);
2611                 break;
2612         default:
2613                 ret = -EINVAL;
2614         }
2615         mutex_unlock(&fi->ais_lock);
2616
2617         return ret;
2618 }
2619
2620 static int kvm_s390_inject_airq(struct kvm *kvm,
2621                                 struct s390_io_adapter *adapter)
2622 {
2623         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2624         struct kvm_s390_interrupt s390int = {
2625                 .type = KVM_S390_INT_IO(1, 0, 0, 0),
2626                 .parm = 0,
2627                 .parm64 = isc_to_int_word(adapter->isc),
2628         };
2629         int ret = 0;
2630
2631         if (!test_kvm_facility(kvm, 72) || !adapter->suppressible)
2632                 return kvm_s390_inject_vm(kvm, &s390int);
2633
2634         mutex_lock(&fi->ais_lock);
2635         if (fi->nimm & AIS_MODE_MASK(adapter->isc)) {
2636                 trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc);
2637                 goto out;
2638         }
2639
2640         ret = kvm_s390_inject_vm(kvm, &s390int);
2641         if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) {
2642                 fi->nimm |= AIS_MODE_MASK(adapter->isc);
2643                 trace_kvm_s390_modify_ais_mode(adapter->isc,
2644                                                KVM_S390_AIS_MODE_SINGLE, 2);
2645         }
2646 out:
2647         mutex_unlock(&fi->ais_lock);
2648         return ret;
2649 }
2650
2651 static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr)
2652 {
2653         unsigned int id = attr->attr;
2654         struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2655
2656         if (!adapter)
2657                 return -EINVAL;
2658
2659         return kvm_s390_inject_airq(kvm, adapter);
2660 }
2661
2662 static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr)
2663 {
2664         struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2665         struct kvm_s390_ais_all ais;
2666
2667         if (!test_kvm_facility(kvm, 72))
2668                 return -EOPNOTSUPP;
2669
2670         if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais)))
2671                 return -EFAULT;
2672
2673         mutex_lock(&fi->ais_lock);
2674         fi->simm = ais.simm;
2675         fi->nimm = ais.nimm;
2676         mutex_unlock(&fi->ais_lock);
2677
2678         return 0;
2679 }
2680
2681 static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2682 {
2683         int r = 0;
2684         unsigned long i;
2685         struct kvm_vcpu *vcpu;
2686
2687         switch (attr->group) {
2688         case KVM_DEV_FLIC_ENQUEUE:
2689                 r = enqueue_floating_irq(dev, attr);
2690                 break;
2691         case KVM_DEV_FLIC_CLEAR_IRQS:
2692                 kvm_s390_clear_float_irqs(dev->kvm);
2693                 break;
2694         case KVM_DEV_FLIC_APF_ENABLE:
2695                 dev->kvm->arch.gmap->pfault_enabled = 1;
2696                 break;
2697         case KVM_DEV_FLIC_APF_DISABLE_WAIT:
2698                 dev->kvm->arch.gmap->pfault_enabled = 0;
2699                 /*
2700                  * Make sure no async faults are in transition when
2701                  * clearing the queues. So we don't need to worry
2702                  * about late coming workers.
2703                  */
2704                 synchronize_srcu(&dev->kvm->srcu);
2705                 kvm_for_each_vcpu(i, vcpu, dev->kvm)
2706                         kvm_clear_async_pf_completion_queue(vcpu);
2707                 break;
2708         case KVM_DEV_FLIC_ADAPTER_REGISTER:
2709                 r = register_io_adapter(dev, attr);
2710                 break;
2711         case KVM_DEV_FLIC_ADAPTER_MODIFY:
2712                 r = modify_io_adapter(dev, attr);
2713                 break;
2714         case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2715                 r = clear_io_irq(dev->kvm, attr);
2716                 break;
2717         case KVM_DEV_FLIC_AISM:
2718                 r = modify_ais_mode(dev->kvm, attr);
2719                 break;
2720         case KVM_DEV_FLIC_AIRQ_INJECT:
2721                 r = flic_inject_airq(dev->kvm, attr);
2722                 break;
2723         case KVM_DEV_FLIC_AISM_ALL:
2724                 r = flic_ais_mode_set_all(dev->kvm, attr);
2725                 break;
2726         default:
2727                 r = -EINVAL;
2728         }
2729
2730         return r;
2731 }
2732
2733 static int flic_has_attr(struct kvm_device *dev,
2734                              struct kvm_device_attr *attr)
2735 {
2736         switch (attr->group) {
2737         case KVM_DEV_FLIC_GET_ALL_IRQS:
2738         case KVM_DEV_FLIC_ENQUEUE:
2739         case KVM_DEV_FLIC_CLEAR_IRQS:
2740         case KVM_DEV_FLIC_APF_ENABLE:
2741         case KVM_DEV_FLIC_APF_DISABLE_WAIT:
2742         case KVM_DEV_FLIC_ADAPTER_REGISTER:
2743         case KVM_DEV_FLIC_ADAPTER_MODIFY:
2744         case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2745         case KVM_DEV_FLIC_AISM:
2746         case KVM_DEV_FLIC_AIRQ_INJECT:
2747         case KVM_DEV_FLIC_AISM_ALL:
2748                 return 0;
2749         }
2750         return -ENXIO;
2751 }
2752
2753 static int flic_create(struct kvm_device *dev, u32 type)
2754 {
2755         if (!dev)
2756                 return -EINVAL;
2757         if (dev->kvm->arch.flic)
2758                 return -EINVAL;
2759         dev->kvm->arch.flic = dev;
2760         return 0;
2761 }
2762
2763 static void flic_destroy(struct kvm_device *dev)
2764 {
2765         dev->kvm->arch.flic = NULL;
2766         kfree(dev);
2767 }
2768
2769 /* s390 floating irq controller (flic) */
2770 struct kvm_device_ops kvm_flic_ops = {
2771         .name = "kvm-flic",
2772         .get_attr = flic_get_attr,
2773         .set_attr = flic_set_attr,
2774         .has_attr = flic_has_attr,
2775         .create = flic_create,
2776         .destroy = flic_destroy,
2777 };
2778
2779 static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap)
2780 {
2781         unsigned long bit;
2782
2783         bit = bit_nr + (addr % PAGE_SIZE) * 8;
2784
2785         return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit;
2786 }
2787
2788 static struct page *get_map_page(struct kvm *kvm, u64 uaddr)
2789 {
2790         struct page *page = NULL;
2791
2792         mmap_read_lock(kvm->mm);
2793         get_user_pages_remote(kvm->mm, uaddr, 1, FOLL_WRITE,
2794                               &page, NULL);
2795         mmap_read_unlock(kvm->mm);
2796         return page;
2797 }
2798
2799 static int adapter_indicators_set(struct kvm *kvm,
2800                                   struct s390_io_adapter *adapter,
2801                                   struct kvm_s390_adapter_int *adapter_int)
2802 {
2803         unsigned long bit;
2804         int summary_set, idx;
2805         struct page *ind_page, *summary_page;
2806         void *map;
2807
2808         ind_page = get_map_page(kvm, adapter_int->ind_addr);
2809         if (!ind_page)
2810                 return -1;
2811         summary_page = get_map_page(kvm, adapter_int->summary_addr);
2812         if (!summary_page) {
2813                 put_page(ind_page);
2814                 return -1;
2815         }
2816
2817         idx = srcu_read_lock(&kvm->srcu);
2818         map = page_address(ind_page);
2819         bit = get_ind_bit(adapter_int->ind_addr,
2820                           adapter_int->ind_offset, adapter->swap);
2821         set_bit(bit, map);
2822         mark_page_dirty(kvm, adapter_int->ind_addr >> PAGE_SHIFT);
2823         set_page_dirty_lock(ind_page);
2824         map = page_address(summary_page);
2825         bit = get_ind_bit(adapter_int->summary_addr,
2826                           adapter_int->summary_offset, adapter->swap);
2827         summary_set = test_and_set_bit(bit, map);
2828         mark_page_dirty(kvm, adapter_int->summary_addr >> PAGE_SHIFT);
2829         set_page_dirty_lock(summary_page);
2830         srcu_read_unlock(&kvm->srcu, idx);
2831
2832         put_page(ind_page);
2833         put_page(summary_page);
2834         return summary_set ? 0 : 1;
2835 }
2836
2837 /*
2838  * < 0 - not injected due to error
2839  * = 0 - coalesced, summary indicator already active
2840  * > 0 - injected interrupt
2841  */
2842 static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e,
2843                            struct kvm *kvm, int irq_source_id, int level,
2844                            bool line_status)
2845 {
2846         int ret;
2847         struct s390_io_adapter *adapter;
2848
2849         /* We're only interested in the 0->1 transition. */
2850         if (!level)
2851                 return 0;
2852         adapter = get_io_adapter(kvm, e->adapter.adapter_id);
2853         if (!adapter)
2854                 return -1;
2855         ret = adapter_indicators_set(kvm, adapter, &e->adapter);
2856         if ((ret > 0) && !adapter->masked) {
2857                 ret = kvm_s390_inject_airq(kvm, adapter);
2858                 if (ret == 0)
2859                         ret = 1;
2860         }
2861         return ret;
2862 }
2863
2864 /*
2865  * Inject the machine check to the guest.
2866  */
2867 void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu,
2868                                      struct mcck_volatile_info *mcck_info)
2869 {
2870         struct kvm_s390_interrupt_info inti;
2871         struct kvm_s390_irq irq;
2872         struct kvm_s390_mchk_info *mchk;
2873         union mci mci;
2874         __u64 cr14 = 0;         /* upper bits are not used */
2875         int rc;
2876
2877         mci.val = mcck_info->mcic;
2878         if (mci.sr)
2879                 cr14 |= CR14_RECOVERY_SUBMASK;
2880         if (mci.dg)
2881                 cr14 |= CR14_DEGRADATION_SUBMASK;
2882         if (mci.w)
2883                 cr14 |= CR14_WARNING_SUBMASK;
2884
2885         mchk = mci.ck ? &inti.mchk : &irq.u.mchk;
2886         mchk->cr14 = cr14;
2887         mchk->mcic = mcck_info->mcic;
2888         mchk->ext_damage_code = mcck_info->ext_damage_code;
2889         mchk->failing_storage_address = mcck_info->failing_storage_address;
2890         if (mci.ck) {
2891                 /* Inject the floating machine check */
2892                 inti.type = KVM_S390_MCHK;
2893                 rc = __inject_vm(vcpu->kvm, &inti);
2894         } else {
2895                 /* Inject the machine check to specified vcpu */
2896                 irq.type = KVM_S390_MCHK;
2897                 rc = kvm_s390_inject_vcpu(vcpu, &irq);
2898         }
2899         WARN_ON_ONCE(rc);
2900 }
2901
2902 int kvm_set_routing_entry(struct kvm *kvm,
2903                           struct kvm_kernel_irq_routing_entry *e,
2904                           const struct kvm_irq_routing_entry *ue)
2905 {
2906         u64 uaddr;
2907
2908         switch (ue->type) {
2909         /* we store the userspace addresses instead of the guest addresses */
2910         case KVM_IRQ_ROUTING_S390_ADAPTER:
2911                 e->set = set_adapter_int;
2912                 uaddr =  gmap_translate(kvm->arch.gmap, ue->u.adapter.summary_addr);
2913                 if (uaddr == -EFAULT)
2914                         return -EFAULT;
2915                 e->adapter.summary_addr = uaddr;
2916                 uaddr =  gmap_translate(kvm->arch.gmap, ue->u.adapter.ind_addr);
2917                 if (uaddr == -EFAULT)
2918                         return -EFAULT;
2919                 e->adapter.ind_addr = uaddr;
2920                 e->adapter.summary_offset = ue->u.adapter.summary_offset;
2921                 e->adapter.ind_offset = ue->u.adapter.ind_offset;
2922                 e->adapter.adapter_id = ue->u.adapter.adapter_id;
2923                 return 0;
2924         default:
2925                 return -EINVAL;
2926         }
2927 }
2928
2929 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
2930                 int irq_source_id, int level, bool line_status)
2931 {
2932         return -EINVAL;
2933 }
2934
2935 int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len)
2936 {
2937         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2938         struct kvm_s390_irq *buf;
2939         int r = 0;
2940         int n;
2941
2942         buf = vmalloc(len);
2943         if (!buf)
2944                 return -ENOMEM;
2945
2946         if (copy_from_user((void *) buf, irqstate, len)) {
2947                 r = -EFAULT;
2948                 goto out_free;
2949         }
2950
2951         /*
2952          * Don't allow setting the interrupt state
2953          * when there are already interrupts pending
2954          */
2955         spin_lock(&li->lock);
2956         if (li->pending_irqs) {
2957                 r = -EBUSY;
2958                 goto out_unlock;
2959         }
2960
2961         for (n = 0; n < len / sizeof(*buf); n++) {
2962                 r = do_inject_vcpu(vcpu, &buf[n]);
2963                 if (r)
2964                         break;
2965         }
2966
2967 out_unlock:
2968         spin_unlock(&li->lock);
2969 out_free:
2970         vfree(buf);
2971
2972         return r;
2973 }
2974
2975 static void store_local_irq(struct kvm_s390_local_interrupt *li,
2976                             struct kvm_s390_irq *irq,
2977                             unsigned long irq_type)
2978 {
2979         switch (irq_type) {
2980         case IRQ_PEND_MCHK_EX:
2981         case IRQ_PEND_MCHK_REP:
2982                 irq->type = KVM_S390_MCHK;
2983                 irq->u.mchk = li->irq.mchk;
2984                 break;
2985         case IRQ_PEND_PROG:
2986                 irq->type = KVM_S390_PROGRAM_INT;
2987                 irq->u.pgm = li->irq.pgm;
2988                 break;
2989         case IRQ_PEND_PFAULT_INIT:
2990                 irq->type = KVM_S390_INT_PFAULT_INIT;
2991                 irq->u.ext = li->irq.ext;
2992                 break;
2993         case IRQ_PEND_EXT_EXTERNAL:
2994                 irq->type = KVM_S390_INT_EXTERNAL_CALL;
2995                 irq->u.extcall = li->irq.extcall;
2996                 break;
2997         case IRQ_PEND_EXT_CLOCK_COMP:
2998                 irq->type = KVM_S390_INT_CLOCK_COMP;
2999                 break;
3000         case IRQ_PEND_EXT_CPU_TIMER:
3001                 irq->type = KVM_S390_INT_CPU_TIMER;
3002                 break;
3003         case IRQ_PEND_SIGP_STOP:
3004                 irq->type = KVM_S390_SIGP_STOP;
3005                 irq->u.stop = li->irq.stop;
3006                 break;
3007         case IRQ_PEND_RESTART:
3008                 irq->type = KVM_S390_RESTART;
3009                 break;
3010         case IRQ_PEND_SET_PREFIX:
3011                 irq->type = KVM_S390_SIGP_SET_PREFIX;
3012                 irq->u.prefix = li->irq.prefix;
3013                 break;
3014         }
3015 }
3016
3017 int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len)
3018 {
3019         int scn;
3020         DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS);
3021         struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
3022         unsigned long pending_irqs;
3023         struct kvm_s390_irq irq;
3024         unsigned long irq_type;
3025         int cpuaddr;
3026         int n = 0;
3027
3028         spin_lock(&li->lock);
3029         pending_irqs = li->pending_irqs;
3030         memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending,
3031                sizeof(sigp_emerg_pending));
3032         spin_unlock(&li->lock);
3033
3034         for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) {
3035                 memset(&irq, 0, sizeof(irq));
3036                 if (irq_type == IRQ_PEND_EXT_EMERGENCY)
3037                         continue;
3038                 if (n + sizeof(irq) > len)
3039                         return -ENOBUFS;
3040                 store_local_irq(&vcpu->arch.local_int, &irq, irq_type);
3041                 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3042                         return -EFAULT;
3043                 n += sizeof(irq);
3044         }
3045
3046         if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) {
3047                 for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) {
3048                         memset(&irq, 0, sizeof(irq));
3049                         if (n + sizeof(irq) > len)
3050                                 return -ENOBUFS;
3051                         irq.type = KVM_S390_INT_EMERGENCY;
3052                         irq.u.emerg.code = cpuaddr;
3053                         if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3054                                 return -EFAULT;
3055                         n += sizeof(irq);
3056                 }
3057         }
3058
3059         if (sca_ext_call_pending(vcpu, &scn)) {
3060                 if (n + sizeof(irq) > len)
3061                         return -ENOBUFS;
3062                 memset(&irq, 0, sizeof(irq));
3063                 irq.type = KVM_S390_INT_EXTERNAL_CALL;
3064                 irq.u.extcall.code = scn;
3065                 if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3066                         return -EFAULT;
3067                 n += sizeof(irq);
3068         }
3069
3070         return n;
3071 }
3072
3073 static void __airqs_kick_single_vcpu(struct kvm *kvm, u8 deliverable_mask)
3074 {
3075         int vcpu_idx, online_vcpus = atomic_read(&kvm->online_vcpus);
3076         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3077         struct kvm_vcpu *vcpu;
3078         u8 vcpu_isc_mask;
3079
3080         for_each_set_bit(vcpu_idx, kvm->arch.idle_mask, online_vcpus) {
3081                 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
3082                 if (psw_ioint_disabled(vcpu))
3083                         continue;
3084                 vcpu_isc_mask = (u8)(vcpu->arch.sie_block->gcr[6] >> 24);
3085                 if (deliverable_mask & vcpu_isc_mask) {
3086                         /* lately kicked but not yet running */
3087                         if (test_and_set_bit(vcpu_idx, gi->kicked_mask))
3088                                 return;
3089                         kvm_s390_vcpu_wakeup(vcpu);
3090                         return;
3091                 }
3092         }
3093 }
3094
3095 static enum hrtimer_restart gisa_vcpu_kicker(struct hrtimer *timer)
3096 {
3097         struct kvm_s390_gisa_interrupt *gi =
3098                 container_of(timer, struct kvm_s390_gisa_interrupt, timer);
3099         struct kvm *kvm =
3100                 container_of(gi->origin, struct sie_page2, gisa)->kvm;
3101         u8 pending_mask;
3102
3103         pending_mask = gisa_get_ipm_or_restore_iam(gi);
3104         if (pending_mask) {
3105                 __airqs_kick_single_vcpu(kvm, pending_mask);
3106                 hrtimer_forward_now(timer, ns_to_ktime(gi->expires));
3107                 return HRTIMER_RESTART;
3108         }
3109
3110         return HRTIMER_NORESTART;
3111 }
3112
3113 #define NULL_GISA_ADDR 0x00000000UL
3114 #define NONE_GISA_ADDR 0x00000001UL
3115 #define GISA_ADDR_MASK 0xfffff000UL
3116
3117 static void process_gib_alert_list(void)
3118 {
3119         struct kvm_s390_gisa_interrupt *gi;
3120         u32 final, gisa_phys, origin = 0UL;
3121         struct kvm_s390_gisa *gisa;
3122         struct kvm *kvm;
3123
3124         do {
3125                 /*
3126                  * If the NONE_GISA_ADDR is still stored in the alert list
3127                  * origin, we will leave the outer loop. No further GISA has
3128                  * been added to the alert list by millicode while processing
3129                  * the current alert list.
3130                  */
3131                 final = (origin & NONE_GISA_ADDR);
3132                 /*
3133                  * Cut off the alert list and store the NONE_GISA_ADDR in the
3134                  * alert list origin to avoid further GAL interruptions.
3135                  * A new alert list can be build up by millicode in parallel
3136                  * for guests not in the yet cut-off alert list. When in the
3137                  * final loop, store the NULL_GISA_ADDR instead. This will re-
3138                  * enable GAL interruptions on the host again.
3139                  */
3140                 origin = xchg(&gib->alert_list_origin,
3141                               (!final) ? NONE_GISA_ADDR : NULL_GISA_ADDR);
3142                 /*
3143                  * Loop through the just cut-off alert list and start the
3144                  * gisa timers to kick idle vcpus to consume the pending
3145                  * interruptions asap.
3146                  */
3147                 while (origin & GISA_ADDR_MASK) {
3148                         gisa_phys = origin;
3149                         gisa = phys_to_virt(gisa_phys);
3150                         origin = gisa->next_alert;
3151                         gisa->next_alert = gisa_phys;
3152                         kvm = container_of(gisa, struct sie_page2, gisa)->kvm;
3153                         gi = &kvm->arch.gisa_int;
3154                         if (hrtimer_active(&gi->timer))
3155                                 hrtimer_cancel(&gi->timer);
3156                         hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL);
3157                 }
3158         } while (!final);
3159
3160 }
3161
3162 void kvm_s390_gisa_clear(struct kvm *kvm)
3163 {
3164         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3165
3166         if (!gi->origin)
3167                 return;
3168         gisa_clear_ipm(gi->origin);
3169         VM_EVENT(kvm, 3, "gisa 0x%pK cleared", gi->origin);
3170 }
3171
3172 void kvm_s390_gisa_init(struct kvm *kvm)
3173 {
3174         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3175
3176         if (!css_general_characteristics.aiv)
3177                 return;
3178         gi->origin = &kvm->arch.sie_page2->gisa;
3179         gi->alert.mask = 0;
3180         spin_lock_init(&gi->alert.ref_lock);
3181         gi->expires = 50 * 1000; /* 50 usec */
3182         hrtimer_init(&gi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3183         gi->timer.function = gisa_vcpu_kicker;
3184         memset(gi->origin, 0, sizeof(struct kvm_s390_gisa));
3185         gi->origin->next_alert = (u32)virt_to_phys(gi->origin);
3186         VM_EVENT(kvm, 3, "gisa 0x%pK initialized", gi->origin);
3187 }
3188
3189 void kvm_s390_gisa_enable(struct kvm *kvm)
3190 {
3191         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3192         struct kvm_vcpu *vcpu;
3193         unsigned long i;
3194         u32 gisa_desc;
3195
3196         if (gi->origin)
3197                 return;
3198         kvm_s390_gisa_init(kvm);
3199         gisa_desc = kvm_s390_get_gisa_desc(kvm);
3200         if (!gisa_desc)
3201                 return;
3202         kvm_for_each_vcpu(i, vcpu, kvm) {
3203                 mutex_lock(&vcpu->mutex);
3204                 vcpu->arch.sie_block->gd = gisa_desc;
3205                 vcpu->arch.sie_block->eca |= ECA_AIV;
3206                 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3207                            vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3208                 mutex_unlock(&vcpu->mutex);
3209         }
3210 }
3211
3212 void kvm_s390_gisa_destroy(struct kvm *kvm)
3213 {
3214         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3215         struct kvm_s390_gisa *gisa = gi->origin;
3216
3217         if (!gi->origin)
3218                 return;
3219         if (gi->alert.mask)
3220                 KVM_EVENT(3, "vm 0x%pK has unexpected iam 0x%02x",
3221                           kvm, gi->alert.mask);
3222         while (gisa_in_alert_list(gi->origin))
3223                 cpu_relax();
3224         hrtimer_cancel(&gi->timer);
3225         gi->origin = NULL;
3226         VM_EVENT(kvm, 3, "gisa 0x%pK destroyed", gisa);
3227 }
3228
3229 void kvm_s390_gisa_disable(struct kvm *kvm)
3230 {
3231         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3232         struct kvm_vcpu *vcpu;
3233         unsigned long i;
3234
3235         if (!gi->origin)
3236                 return;
3237         kvm_for_each_vcpu(i, vcpu, kvm) {
3238                 mutex_lock(&vcpu->mutex);
3239                 vcpu->arch.sie_block->eca &= ~ECA_AIV;
3240                 vcpu->arch.sie_block->gd = 0U;
3241                 mutex_unlock(&vcpu->mutex);
3242                 VCPU_EVENT(vcpu, 3, "AIV disabled for cpu %03u", vcpu->vcpu_id);
3243         }
3244         kvm_s390_gisa_destroy(kvm);
3245 }
3246
3247 /**
3248  * kvm_s390_gisc_register - register a guest ISC
3249  *
3250  * @kvm:  the kernel vm to work with
3251  * @gisc: the guest interruption sub class to register
3252  *
3253  * The function extends the vm specific alert mask to use.
3254  * The effective IAM mask in the GISA is updated as well
3255  * in case the GISA is not part of the GIB alert list.
3256  * It will be updated latest when the IAM gets restored
3257  * by gisa_get_ipm_or_restore_iam().
3258  *
3259  * Returns: the nonspecific ISC (NISC) the gib alert mechanism
3260  *          has registered with the channel subsystem.
3261  *          -ENODEV in case the vm uses no GISA
3262  *          -ERANGE in case the guest ISC is invalid
3263  */
3264 int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc)
3265 {
3266         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3267
3268         if (!gi->origin)
3269                 return -ENODEV;
3270         if (gisc > MAX_ISC)
3271                 return -ERANGE;
3272
3273         spin_lock(&gi->alert.ref_lock);
3274         gi->alert.ref_count[gisc]++;
3275         if (gi->alert.ref_count[gisc] == 1) {
3276                 gi->alert.mask |= 0x80 >> gisc;
3277                 gisa_set_iam(gi->origin, gi->alert.mask);
3278         }
3279         spin_unlock(&gi->alert.ref_lock);
3280
3281         return gib->nisc;
3282 }
3283 EXPORT_SYMBOL_GPL(kvm_s390_gisc_register);
3284
3285 /**
3286  * kvm_s390_gisc_unregister - unregister a guest ISC
3287  *
3288  * @kvm:  the kernel vm to work with
3289  * @gisc: the guest interruption sub class to register
3290  *
3291  * The function reduces the vm specific alert mask to use.
3292  * The effective IAM mask in the GISA is updated as well
3293  * in case the GISA is not part of the GIB alert list.
3294  * It will be updated latest when the IAM gets restored
3295  * by gisa_get_ipm_or_restore_iam().
3296  *
3297  * Returns: the nonspecific ISC (NISC) the gib alert mechanism
3298  *          has registered with the channel subsystem.
3299  *          -ENODEV in case the vm uses no GISA
3300  *          -ERANGE in case the guest ISC is invalid
3301  *          -EINVAL in case the guest ISC is not registered
3302  */
3303 int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc)
3304 {
3305         struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3306         int rc = 0;
3307
3308         if (!gi->origin)
3309                 return -ENODEV;
3310         if (gisc > MAX_ISC)
3311                 return -ERANGE;
3312
3313         spin_lock(&gi->alert.ref_lock);
3314         if (gi->alert.ref_count[gisc] == 0) {
3315                 rc = -EINVAL;
3316                 goto out;
3317         }
3318         gi->alert.ref_count[gisc]--;
3319         if (gi->alert.ref_count[gisc] == 0) {
3320                 gi->alert.mask &= ~(0x80 >> gisc);
3321                 gisa_set_iam(gi->origin, gi->alert.mask);
3322         }
3323 out:
3324         spin_unlock(&gi->alert.ref_lock);
3325
3326         return rc;
3327 }
3328 EXPORT_SYMBOL_GPL(kvm_s390_gisc_unregister);
3329
3330 static void aen_host_forward(unsigned long si)
3331 {
3332         struct kvm_s390_gisa_interrupt *gi;
3333         struct zpci_gaite *gaite;
3334         struct kvm *kvm;
3335
3336         gaite = (struct zpci_gaite *)aift->gait +
3337                 (si * sizeof(struct zpci_gaite));
3338         if (gaite->count == 0)
3339                 return;
3340         if (gaite->aisb != 0)
3341                 set_bit_inv(gaite->aisbo, phys_to_virt(gaite->aisb));
3342
3343         kvm = kvm_s390_pci_si_to_kvm(aift, si);
3344         if (!kvm)
3345                 return;
3346         gi = &kvm->arch.gisa_int;
3347
3348         if (!(gi->origin->g1.simm & AIS_MODE_MASK(gaite->gisc)) ||
3349             !(gi->origin->g1.nimm & AIS_MODE_MASK(gaite->gisc))) {
3350                 gisa_set_ipm_gisc(gi->origin, gaite->gisc);
3351                 if (hrtimer_active(&gi->timer))
3352                         hrtimer_cancel(&gi->timer);
3353                 hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL);
3354                 kvm->stat.aen_forward++;
3355         }
3356 }
3357
3358 static void aen_process_gait(u8 isc)
3359 {
3360         bool found = false, first = true;
3361         union zpci_sic_iib iib = {{0}};
3362         unsigned long si, flags;
3363
3364         spin_lock_irqsave(&aift->gait_lock, flags);
3365
3366         if (!aift->gait) {
3367                 spin_unlock_irqrestore(&aift->gait_lock, flags);
3368                 return;
3369         }
3370
3371         for (si = 0;;) {
3372                 /* Scan adapter summary indicator bit vector */
3373                 si = airq_iv_scan(aift->sbv, si, airq_iv_end(aift->sbv));
3374                 if (si == -1UL) {
3375                         if (first || found) {
3376                                 /* Re-enable interrupts. */
3377                                 zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, isc,
3378                                                   &iib);
3379                                 first = found = false;
3380                         } else {
3381                                 /* Interrupts on and all bits processed */
3382                                 break;
3383                         }
3384                         found = false;
3385                         si = 0;
3386                         /* Scan again after re-enabling interrupts */
3387                         continue;
3388                 }
3389                 found = true;
3390                 aen_host_forward(si);
3391         }
3392
3393         spin_unlock_irqrestore(&aift->gait_lock, flags);
3394 }
3395
3396 static void gib_alert_irq_handler(struct airq_struct *airq,
3397                                   struct tpi_info *tpi_info)
3398 {
3399         struct tpi_adapter_info *info = (struct tpi_adapter_info *)tpi_info;
3400
3401         inc_irq_stat(IRQIO_GAL);
3402
3403         if ((info->forward || info->error) &&
3404             IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
3405                 aen_process_gait(info->isc);
3406                 if (info->aism != 0)
3407                         process_gib_alert_list();
3408         } else {
3409                 process_gib_alert_list();
3410         }
3411 }
3412
3413 static struct airq_struct gib_alert_irq = {
3414         .handler = gib_alert_irq_handler,
3415 };
3416
3417 void kvm_s390_gib_destroy(void)
3418 {
3419         if (!gib)
3420                 return;
3421         if (kvm_s390_pci_interp_allowed() && aift) {
3422                 mutex_lock(&aift->aift_lock);
3423                 kvm_s390_pci_aen_exit();
3424                 mutex_unlock(&aift->aift_lock);
3425         }
3426         chsc_sgib(0);
3427         unregister_adapter_interrupt(&gib_alert_irq);
3428         free_page((unsigned long)gib);
3429         gib = NULL;
3430 }
3431
3432 int __init kvm_s390_gib_init(u8 nisc)
3433 {
3434         u32 gib_origin;
3435         int rc = 0;
3436
3437         if (!css_general_characteristics.aiv) {
3438                 KVM_EVENT(3, "%s", "gib not initialized, no AIV facility");
3439                 goto out;
3440         }
3441
3442         gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
3443         if (!gib) {
3444                 rc = -ENOMEM;
3445                 goto out;
3446         }
3447
3448         gib_alert_irq.isc = nisc;
3449         if (register_adapter_interrupt(&gib_alert_irq)) {
3450                 pr_err("Registering the GIB alert interruption handler failed\n");
3451                 rc = -EIO;
3452                 goto out_free_gib;
3453         }
3454         /* adapter interrupts used for AP (applicable here) don't use the LSI */
3455         *gib_alert_irq.lsi_ptr = 0xff;
3456
3457         gib->nisc = nisc;
3458         gib_origin = virt_to_phys(gib);
3459         if (chsc_sgib(gib_origin)) {
3460                 pr_err("Associating the GIB with the AIV facility failed\n");
3461                 free_page((unsigned long)gib);
3462                 gib = NULL;
3463                 rc = -EIO;
3464                 goto out_unreg_gal;
3465         }
3466
3467         if (kvm_s390_pci_interp_allowed()) {
3468                 if (kvm_s390_pci_aen_init(nisc)) {
3469                         pr_err("Initializing AEN for PCI failed\n");
3470                         rc = -EIO;
3471                         goto out_unreg_gal;
3472                 }
3473         }
3474
3475         KVM_EVENT(3, "gib 0x%pK (nisc=%d) initialized", gib, gib->nisc);
3476         goto out;
3477
3478 out_unreg_gal:
3479         unregister_adapter_interrupt(&gib_alert_irq);
3480 out_free_gib:
3481         free_page((unsigned long)gib);
3482         gib = NULL;
3483 out:
3484         return rc;
3485 }