drivers/gpu/drm/i915/display/intel_display_irq.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2023 Intel Corporation
   4  */
   5
   6 #include "gt/intel_rps.h"
   7 #include "i915_drv.h"
   8 #include "i915_irq.h"
   9 #include "i915_reg.h"
  10 #include "icl_dsi_regs.h"
  11 #include "intel_crtc.h"
  12 #include "intel_de.h"
  13 #include "intel_display_irq.h"
  14 #include "intel_display_trace.h"
  15 #include "intel_display_types.h"
  16 #include "intel_dp_aux.h"
  17 #include "intel_fdi_regs.h"
  18 #include "intel_fifo_underrun.h"
  19 #include "intel_gmbus.h"
  20 #include "intel_hotplug_irq.h"
  21 #include "intel_pmdemand.h"
  22 #include "intel_psr.h"
  23 #include "intel_psr_regs.h"
  24
  25 static void
  26 intel_handle_vblank(struct drm_i915_private *dev_priv, enum pipe pipe)
  27 {
  28         struct intel_crtc *crtc = intel_crtc_for_pipe(dev_priv, pipe);
  29
  30         drm_crtc_handle_vblank(&crtc->base);
  31 }
  32
  33 /**
  34  * ilk_update_display_irq - update DEIMR
  35  * @dev_priv: driver private
  36  * @interrupt_mask: mask of interrupt bits to update
  37  * @enabled_irq_mask: mask of interrupt bits to enable
  38  */
  39 void ilk_update_display_irq(struct drm_i915_private *dev_priv,
  40                             u32 interrupt_mask, u32 enabled_irq_mask)
  41 {
  42         u32 new_val;
  43
  44         lockdep_assert_held(&dev_priv->irq_lock);
  45         drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
  46
  47         new_val = dev_priv->irq_mask;
  48         new_val &= ~interrupt_mask;
  49         new_val |= (~enabled_irq_mask & interrupt_mask);
  50
  51         if (new_val != dev_priv->irq_mask &&
  52             !drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv))) {
  53                 dev_priv->irq_mask = new_val;
  54                 intel_uncore_write(&dev_priv->uncore, DEIMR, dev_priv->irq_mask);
  55                 intel_uncore_posting_read(&dev_priv->uncore, DEIMR);
  56         }
  57 }
  58
  59 void ilk_enable_display_irq(struct drm_i915_private *i915, u32 bits)
  60 {
  61         ilk_update_display_irq(i915, bits, bits);
  62 }
  63
  64 void ilk_disable_display_irq(struct drm_i915_private *i915, u32 bits)
  65 {
  66         ilk_update_display_irq(i915, bits, 0);
  67 }
  68
  69 /**
  70  * bdw_update_port_irq - update DE port interrupt
  71  * @dev_priv: driver private
  72  * @interrupt_mask: mask of interrupt bits to update
  73  * @enabled_irq_mask: mask of interrupt bits to enable
  74  */
  75 void bdw_update_port_irq(struct drm_i915_private *dev_priv,
  76                          u32 interrupt_mask, u32 enabled_irq_mask)
  77 {
  78         u32 new_val;
  79         u32 old_val;
  80
  81         lockdep_assert_held(&dev_priv->irq_lock);
  82
  83         drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
  84
  85         if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
  86                 return;
  87
  88         old_val = intel_uncore_read(&dev_priv->uncore, GEN8_DE_PORT_IMR);
  89
  90         new_val = old_val;
  91         new_val &= ~interrupt_mask;
  92         new_val |= (~enabled_irq_mask & interrupt_mask);
  93
  94         if (new_val != old_val) {
  95                 intel_uncore_write(&dev_priv->uncore, GEN8_DE_PORT_IMR, new_val);
  96                 intel_uncore_posting_read(&dev_priv->uncore, GEN8_DE_PORT_IMR);
  97         }
  98 }
  99
 100 /**
 101  * bdw_update_pipe_irq - update DE pipe interrupt
 102  * @dev_priv: driver private
 103  * @pipe: pipe whose interrupt to update
 104  * @interrupt_mask: mask of interrupt bits to update
 105  * @enabled_irq_mask: mask of interrupt bits to enable
 106  */
 107 static void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
 108                                 enum pipe pipe, u32 interrupt_mask,
 109                                 u32 enabled_irq_mask)
 110 {
 111         u32 new_val;
 112
 113         lockdep_assert_held(&dev_priv->irq_lock);
 114
 115         drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
 116
 117         if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
 118                 return;
 119
 120         new_val = dev_priv->display.irq.de_irq_mask[pipe];
 121         new_val &= ~interrupt_mask;
 122         new_val |= (~enabled_irq_mask & interrupt_mask);
 123
 124         if (new_val != dev_priv->display.irq.de_irq_mask[pipe]) {
 125                 dev_priv->display.irq.de_irq_mask[pipe] = new_val;
 126                 intel_uncore_write(&dev_priv->uncore, GEN8_DE_PIPE_IMR(pipe),
 127                                    dev_priv->display.irq.de_irq_mask[pipe]);
 128                 intel_uncore_posting_read(&dev_priv->uncore, GEN8_DE_PIPE_IMR(pipe));
 129         }
 130 }
 131
 132 void bdw_enable_pipe_irq(struct drm_i915_private *i915,
 133                          enum pipe pipe, u32 bits)
 134 {
 135         bdw_update_pipe_irq(i915, pipe, bits, bits);
 136 }
 137
 138 void bdw_disable_pipe_irq(struct drm_i915_private *i915,
 139                           enum pipe pipe, u32 bits)
 140 {
 141         bdw_update_pipe_irq(i915, pipe, bits, 0);
 142 }
 143
 144 /**
 145  * ibx_display_interrupt_update - update SDEIMR
 146  * @dev_priv: driver private
 147  * @interrupt_mask: mask of interrupt bits to update
 148  * @enabled_irq_mask: mask of interrupt bits to enable
 149  */
 150 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
 151                                   u32 interrupt_mask,
 152                                   u32 enabled_irq_mask)
 153 {
 154         u32 sdeimr = intel_uncore_read(&dev_priv->uncore, SDEIMR);
 155
 156         sdeimr &= ~interrupt_mask;
 157         sdeimr |= (~enabled_irq_mask & interrupt_mask);
 158
 159         drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
 160
 161         lockdep_assert_held(&dev_priv->irq_lock);
 162
 163         if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
 164                 return;
 165
 166         intel_uncore_write(&dev_priv->uncore, SDEIMR, sdeimr);
 167         intel_uncore_posting_read(&dev_priv->uncore, SDEIMR);
 168 }
 169
 170 void ibx_enable_display_interrupt(struct drm_i915_private *i915, u32 bits)
 171 {
 172         ibx_display_interrupt_update(i915, bits, bits);
 173 }
 174
 175 void ibx_disable_display_interrupt(struct drm_i915_private *i915, u32 bits)
 176 {
 177         ibx_display_interrupt_update(i915, bits, 0);
 178 }
 179
 180 u32 i915_pipestat_enable_mask(struct drm_i915_private *dev_priv,
 181                               enum pipe pipe)
 182 {
 183         u32 status_mask = dev_priv->display.irq.pipestat_irq_mask[pipe];
 184         u32 enable_mask = status_mask << 16;
 185
 186         lockdep_assert_held(&dev_priv->irq_lock);
 187
 188         if (DISPLAY_VER(dev_priv) < 5)
 189                 goto out;
 190
 191         /*
 192          * On pipe A we don't support the PSR interrupt yet,
 193          * on pipe B and C the same bit MBZ.
 194          */
 195         if (drm_WARN_ON_ONCE(&dev_priv->drm,
 196                              status_mask & PIPE_A_PSR_STATUS_VLV))
 197                 return 0;
 198         /*
 199          * On pipe B and C we don't support the PSR interrupt yet, on pipe
 200          * A the same bit is for perf counters which we don't use either.
 201          */
 202         if (drm_WARN_ON_ONCE(&dev_priv->drm,
 203                              status_mask & PIPE_B_PSR_STATUS_VLV))
 204                 return 0;
 205
 206         enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
 207                          SPRITE0_FLIP_DONE_INT_EN_VLV |
 208                          SPRITE1_FLIP_DONE_INT_EN_VLV);
 209         if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
 210                 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
 211         if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
 212                 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
 213
 214 out:
 215         drm_WARN_ONCE(&dev_priv->drm,
 216                       enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
 217                       status_mask & ~PIPESTAT_INT_STATUS_MASK,
 218                       "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
 219                       pipe_name(pipe), enable_mask, status_mask);
 220
 221         return enable_mask;
 222 }
 223
 224 void i915_enable_pipestat(struct drm_i915_private *dev_priv,
 225                           enum pipe pipe, u32 status_mask)
 226 {
 227         i915_reg_t reg = PIPESTAT(pipe);
 228         u32 enable_mask;
 229
 230         drm_WARN_ONCE(&dev_priv->drm, status_mask & ~PIPESTAT_INT_STATUS_MASK,
 231                       "pipe %c: status_mask=0x%x\n",
 232                       pipe_name(pipe), status_mask);
 233
 234         lockdep_assert_held(&dev_priv->irq_lock);
 235         drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv));
 236
 237         if ((dev_priv->display.irq.pipestat_irq_mask[pipe] & status_mask) == status_mask)
 238                 return;
 239
 240         dev_priv->display.irq.pipestat_irq_mask[pipe] |= status_mask;
 241         enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
 242
 243         intel_uncore_write(&dev_priv->uncore, reg, enable_mask | status_mask);
 244         intel_uncore_posting_read(&dev_priv->uncore, reg);
 245 }
 246
 247 void i915_disable_pipestat(struct drm_i915_private *dev_priv,
 248                            enum pipe pipe, u32 status_mask)
 249 {
 250         i915_reg_t reg = PIPESTAT(pipe);
 251         u32 enable_mask;
 252
 253         drm_WARN_ONCE(&dev_priv->drm, status_mask & ~PIPESTAT_INT_STATUS_MASK,
 254                       "pipe %c: status_mask=0x%x\n",
 255                       pipe_name(pipe), status_mask);
 256
 257         lockdep_assert_held(&dev_priv->irq_lock);
 258         drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv));
 259
 260         if ((dev_priv->display.irq.pipestat_irq_mask[pipe] & status_mask) == 0)
 261                 return;
 262
 263         dev_priv->display.irq.pipestat_irq_mask[pipe] &= ~status_mask;
 264         enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
 265
 266         intel_uncore_write(&dev_priv->uncore, reg, enable_mask | status_mask);
 267         intel_uncore_posting_read(&dev_priv->uncore, reg);
 268 }
 269
 270 static bool i915_has_asle(struct drm_i915_private *i915)
 271 {
 272         if (!IS_PINEVIEW(i915) && !IS_MOBILE(i915))
 273                 return false;
 274
 275         return intel_opregion_asle_present(i915);
 276 }
 277
 278 /**
 279  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
 280  * @dev_priv: i915 device private
 281  */
 282 void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
 283 {
 284         if (!i915_has_asle(dev_priv))
 285                 return;
 286
 287         spin_lock_irq(&dev_priv->irq_lock);
 288
 289         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
 290         if (DISPLAY_VER(dev_priv) >= 4)
 291                 i915_enable_pipestat(dev_priv, PIPE_A,
 292                                      PIPE_LEGACY_BLC_EVENT_STATUS);
 293
 294         spin_unlock_irq(&dev_priv->irq_lock);
 295 }
 296
 297 #if defined(CONFIG_DEBUG_FS)
 298 static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
 299                                          enum pipe pipe,
 300                                          u32 crc0, u32 crc1,
 301                                          u32 crc2, u32 crc3,
 302                                          u32 crc4)
 303 {
 304         struct intel_crtc *crtc = intel_crtc_for_pipe(dev_priv, pipe);
 305         struct intel_pipe_crc *pipe_crc = &crtc->pipe_crc;
 306         u32 crcs[5] = { crc0, crc1, crc2, crc3, crc4 };
 307
 308         trace_intel_pipe_crc(crtc, crcs);
 309
 310         spin_lock(&pipe_crc->lock);
 311         /*
 312          * For some not yet identified reason, the first CRC is
 313          * bonkers. So let's just wait for the next vblank and read
 314          * out the buggy result.
 315          *
 316          * On GEN8+ sometimes the second CRC is bonkers as well, so
 317          * don't trust that one either.
 318          */
 319         if (pipe_crc->skipped <= 0 ||
 320             (DISPLAY_VER(dev_priv) >= 8 && pipe_crc->skipped == 1)) {
 321                 pipe_crc->skipped++;
 322                 spin_unlock(&pipe_crc->lock);
 323                 return;
 324         }
 325         spin_unlock(&pipe_crc->lock);
 326
 327         drm_crtc_add_crc_entry(&crtc->base, true,
 328                                drm_crtc_accurate_vblank_count(&crtc->base),
 329                                crcs);
 330 }
 331 #else
 332 static inline void
 333 display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
 334                              enum pipe pipe,
 335                              u32 crc0, u32 crc1,
 336                              u32 crc2, u32 crc3,
 337                              u32 crc4) {}
 338 #endif
 339
 340 static void flip_done_handler(struct drm_i915_private *i915,
 341                               enum pipe pipe)
 342 {
 343         struct intel_crtc *crtc = intel_crtc_for_pipe(i915, pipe);
 344
 345         spin_lock(&i915->drm.event_lock);
 346
 347         if (crtc->flip_done_event) {
 348                 drm_crtc_send_vblank_event(&crtc->base, crtc->flip_done_event);
 349                 crtc->flip_done_event = NULL;
 350         }
 351
 352         spin_unlock(&i915->drm.event_lock);
 353 }
 354
 355 static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
 356                                      enum pipe pipe)
 357 {
 358         display_pipe_crc_irq_handler(dev_priv, pipe,
 359                                      intel_uncore_read(&dev_priv->uncore, PIPE_CRC_RES_1_IVB(pipe)),
 360                                      0, 0, 0, 0);
 361 }
 362
 363 static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
 364                                      enum pipe pipe)
 365 {
 366         display_pipe_crc_irq_handler(dev_priv, pipe,
 367                                      intel_uncore_read(&dev_priv->uncore, PIPE_CRC_RES_1_IVB(pipe)),
 368                                      intel_uncore_read(&dev_priv->uncore, PIPE_CRC_RES_2_IVB(pipe)),
 369                                      intel_uncore_read(&dev_priv->uncore, PIPE_CRC_RES_3_IVB(pipe)),
 370                                      intel_uncore_read(&dev_priv->uncore, PIPE_CRC_RES_4_IVB(pipe)),
 371                                      intel_uncore_read(&dev_priv->uncore, PIPE_CRC_RES_5_IVB(pipe)));
 372 }
 373
 374 static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
 375                                       enum pipe pipe)
 376 {
 377         u32 res1, res2;
 378
 379         if (DISPLAY_VER(dev_priv) >= 3)
 380                 res1 = intel_uncore_read(&dev_priv->uncore, PIPE_CRC_RES_RES1_I915(pipe));
 381         else
 382                 res1 = 0;
 383
 384         if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv))
 385                 res2 = intel_uncore_read(&dev_priv->uncore, PIPE_CRC_RES_RES2_G4X(pipe));
 386         else
 387                 res2 = 0;
 388
 389         display_pipe_crc_irq_handler(dev_priv, pipe,
 390                                      intel_uncore_read(&dev_priv->uncore, PIPE_CRC_RES_RED(pipe)),
 391                                      intel_uncore_read(&dev_priv->uncore, PIPE_CRC_RES_GREEN(pipe)),
 392                                      intel_uncore_read(&dev_priv->uncore, PIPE_CRC_RES_BLUE(pipe)),
 393                                      res1, res2);
 394 }
 395
 396 void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv)
 397 {
 398         enum pipe pipe;
 399
 400         for_each_pipe(dev_priv, pipe) {
 401                 intel_uncore_write(&dev_priv->uncore, PIPESTAT(pipe),
 402                                    PIPESTAT_INT_STATUS_MASK |
 403                                    PIPE_FIFO_UNDERRUN_STATUS);
 404
 405                 dev_priv->display.irq.pipestat_irq_mask[pipe] = 0;
 406         }
 407 }
 408
 409 void i9xx_pipestat_irq_ack(struct drm_i915_private *dev_priv,
 410                            u32 iir, u32 pipe_stats[I915_MAX_PIPES])
 411 {
 412         enum pipe pipe;
 413
 414         spin_lock(&dev_priv->irq_lock);
 415
 416         if (!dev_priv->display.irq.display_irqs_enabled) {
 417                 spin_unlock(&dev_priv->irq_lock);
 418                 return;
 419         }
 420
 421         for_each_pipe(dev_priv, pipe) {
 422                 i915_reg_t reg;
 423                 u32 status_mask, enable_mask, iir_bit = 0;
 424
 425                 /*
 426                  * PIPESTAT bits get signalled even when the interrupt is
 427                  * disabled with the mask bits, and some of the status bits do
 428                  * not generate interrupts at all (like the underrun bit). Hence
 429                  * we need to be careful that we only handle what we want to
 430                  * handle.
 431                  */
 432
 433                 /* fifo underruns are filterered in the underrun handler. */
 434                 status_mask = PIPE_FIFO_UNDERRUN_STATUS;
 435
 436                 switch (pipe) {
 437                 default:
 438                 case PIPE_A:
 439                         iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
 440                         break;
 441                 case PIPE_B:
 442                         iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
 443                         break;
 444                 case PIPE_C:
 445                         iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
 446                         break;
 447                 }
 448                 if (iir & iir_bit)
 449                         status_mask |= dev_priv->display.irq.pipestat_irq_mask[pipe];
 450
 451                 if (!status_mask)
 452                         continue;
 453
 454                 reg = PIPESTAT(pipe);
 455                 pipe_stats[pipe] = intel_uncore_read(&dev_priv->uncore, reg) & status_mask;
 456                 enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
 457
 458                 /*
 459                  * Clear the PIPE*STAT regs before the IIR
 460                  *
 461                  * Toggle the enable bits to make sure we get an
 462                  * edge in the ISR pipe event bit if we don't clear
 463                  * all the enabled status bits. Otherwise the edge
 464                  * triggered IIR on i965/g4x wouldn't notice that
 465                  * an interrupt is still pending.
 466                  */
 467                 if (pipe_stats[pipe]) {
 468                         intel_uncore_write(&dev_priv->uncore, reg, pipe_stats[pipe]);
 469                         intel_uncore_write(&dev_priv->uncore, reg, enable_mask);
 470                 }
 471         }
 472         spin_unlock(&dev_priv->irq_lock);
 473 }
 474
 475 void i8xx_pipestat_irq_handler(struct drm_i915_private *dev_priv,
 476                                u16 iir, u32 pipe_stats[I915_MAX_PIPES])
 477 {
 478         enum pipe pipe;
 479
 480         for_each_pipe(dev_priv, pipe) {
 481                 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
 482                         intel_handle_vblank(dev_priv, pipe);
 483
 484                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
 485                         i9xx_pipe_crc_irq_handler(dev_priv, pipe);
 486
 487                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
 488                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
 489         }
 490 }
 491
 492 void i915_pipestat_irq_handler(struct drm_i915_private *dev_priv,
 493                                u32 iir, u32 pipe_stats[I915_MAX_PIPES])
 494 {
 495         bool blc_event = false;
 496         enum pipe pipe;
 497
 498         for_each_pipe(dev_priv, pipe) {
 499                 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
 500                         intel_handle_vblank(dev_priv, pipe);
 501
 502                 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
 503                         blc_event = true;
 504
 505                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
 506                         i9xx_pipe_crc_irq_handler(dev_priv, pipe);
 507
 508                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
 509                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
 510         }
 511
 512         if (blc_event || (iir & I915_ASLE_INTERRUPT))
 513                 intel_opregion_asle_intr(dev_priv);
 514 }
 515
 516 void i965_pipestat_irq_handler(struct drm_i915_private *dev_priv,
 517                                u32 iir, u32 pipe_stats[I915_MAX_PIPES])
 518 {
 519         bool blc_event = false;
 520         enum pipe pipe;
 521
 522         for_each_pipe(dev_priv, pipe) {
 523                 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
 524                         intel_handle_vblank(dev_priv, pipe);
 525
 526                 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
 527                         blc_event = true;
 528
 529                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
 530                         i9xx_pipe_crc_irq_handler(dev_priv, pipe);
 531
 532                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
 533                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
 534         }
 535
 536         if (blc_event || (iir & I915_ASLE_INTERRUPT))
 537                 intel_opregion_asle_intr(dev_priv);
 538
 539         if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
 540                 intel_gmbus_irq_handler(dev_priv);
 541 }
 542
 543 void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
 544                                      u32 pipe_stats[I915_MAX_PIPES])
 545 {
 546         enum pipe pipe;
 547
 548         for_each_pipe(dev_priv, pipe) {
 549                 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
 550                         intel_handle_vblank(dev_priv, pipe);
 551
 552                 if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV)
 553                         flip_done_handler(dev_priv, pipe);
 554
 555                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
 556                         i9xx_pipe_crc_irq_handler(dev_priv, pipe);
 557
 558                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
 559                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
 560         }
 561
 562         if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
 563                 intel_gmbus_irq_handler(dev_priv);
 564 }
 565
 566 static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
 567 {
 568         enum pipe pipe;
 569         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
 570
 571         ibx_hpd_irq_handler(dev_priv, hotplug_trigger);
 572
 573         if (pch_iir & SDE_AUDIO_POWER_MASK) {
 574                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
 575                                SDE_AUDIO_POWER_SHIFT);
 576                 drm_dbg(&dev_priv->drm, "PCH audio power change on port %d\n",
 577                         port_name(port));
 578         }
 579
 580         if (pch_iir & SDE_AUX_MASK)
 581                 intel_dp_aux_irq_handler(dev_priv);
 582
 583         if (pch_iir & SDE_GMBUS)
 584                 intel_gmbus_irq_handler(dev_priv);
 585
 586         if (pch_iir & SDE_AUDIO_HDCP_MASK)
 587                 drm_dbg(&dev_priv->drm, "PCH HDCP audio interrupt\n");
 588
 589         if (pch_iir & SDE_AUDIO_TRANS_MASK)
 590                 drm_dbg(&dev_priv->drm, "PCH transcoder audio interrupt\n");
 591
 592         if (pch_iir & SDE_POISON)
 593                 drm_err(&dev_priv->drm, "PCH poison interrupt\n");
 594
 595         if (pch_iir & SDE_FDI_MASK) {
 596                 for_each_pipe(dev_priv, pipe)
 597                         drm_dbg(&dev_priv->drm, "  pipe %c FDI IIR: 0x%08x\n",
 598                                 pipe_name(pipe),
 599                                 intel_uncore_read(&dev_priv->uncore, FDI_RX_IIR(pipe)));
 600         }
 601
 602         if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
 603                 drm_dbg(&dev_priv->drm, "PCH transcoder CRC done interrupt\n");
 604
 605         if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
 606                 drm_dbg(&dev_priv->drm,
 607                         "PCH transcoder CRC error interrupt\n");
 608
 609         if (pch_iir & SDE_TRANSA_FIFO_UNDER)
 610                 intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_A);
 611
 612         if (pch_iir & SDE_TRANSB_FIFO_UNDER)
 613                 intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_B);
 614 }
 615
 616 static void ivb_err_int_handler(struct drm_i915_private *dev_priv)
 617 {
 618         u32 err_int = intel_uncore_read(&dev_priv->uncore, GEN7_ERR_INT);
 619         enum pipe pipe;
 620
 621         if (err_int & ERR_INT_POISON)
 622                 drm_err(&dev_priv->drm, "Poison interrupt\n");
 623
 624         for_each_pipe(dev_priv, pipe) {
 625                 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
 626                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
 627
 628                 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
 629                         if (IS_IVYBRIDGE(dev_priv))
 630                                 ivb_pipe_crc_irq_handler(dev_priv, pipe);
 631                         else
 632                                 hsw_pipe_crc_irq_handler(dev_priv, pipe);
 633                 }
 634         }
 635
 636         intel_uncore_write(&dev_priv->uncore, GEN7_ERR_INT, err_int);
 637 }
 638
 639 static void cpt_serr_int_handler(struct drm_i915_private *dev_priv)
 640 {
 641         u32 serr_int = intel_uncore_read(&dev_priv->uncore, SERR_INT);
 642         enum pipe pipe;
 643
 644         if (serr_int & SERR_INT_POISON)
 645                 drm_err(&dev_priv->drm, "PCH poison interrupt\n");
 646
 647         for_each_pipe(dev_priv, pipe)
 648                 if (serr_int & SERR_INT_TRANS_FIFO_UNDERRUN(pipe))
 649                         intel_pch_fifo_underrun_irq_handler(dev_priv, pipe);
 650
 651         intel_uncore_write(&dev_priv->uncore, SERR_INT, serr_int);
 652 }
 653
 654 static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
 655 {
 656         enum pipe pipe;
 657         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
 658
 659         ibx_hpd_irq_handler(dev_priv, hotplug_trigger);
 660
 661         if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
 662                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
 663                                SDE_AUDIO_POWER_SHIFT_CPT);
 664                 drm_dbg(&dev_priv->drm, "PCH audio power change on port %c\n",
 665                         port_name(port));
 666         }
 667
 668         if (pch_iir & SDE_AUX_MASK_CPT)
 669                 intel_dp_aux_irq_handler(dev_priv);
 670
 671         if (pch_iir & SDE_GMBUS_CPT)
 672                 intel_gmbus_irq_handler(dev_priv);
 673
 674         if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
 675                 drm_dbg(&dev_priv->drm, "Audio CP request interrupt\n");
 676
 677         if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
 678                 drm_dbg(&dev_priv->drm, "Audio CP change interrupt\n");
 679
 680         if (pch_iir & SDE_FDI_MASK_CPT) {
 681                 for_each_pipe(dev_priv, pipe)
 682                         drm_dbg(&dev_priv->drm, "  pipe %c FDI IIR: 0x%08x\n",
 683                                 pipe_name(pipe),
 684                                 intel_uncore_read(&dev_priv->uncore, FDI_RX_IIR(pipe)));
 685         }
 686
 687         if (pch_iir & SDE_ERROR_CPT)
 688                 cpt_serr_int_handler(dev_priv);
 689 }
 690
 691 void ilk_display_irq_handler(struct drm_i915_private *dev_priv, u32 de_iir)
 692 {
 693         enum pipe pipe;
 694         u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
 695
 696         if (hotplug_trigger)
 697                 ilk_hpd_irq_handler(dev_priv, hotplug_trigger);
 698
 699         if (de_iir & DE_AUX_CHANNEL_A)
 700                 intel_dp_aux_irq_handler(dev_priv);
 701
 702         if (de_iir & DE_GSE)
 703                 intel_opregion_asle_intr(dev_priv);
 704
 705         if (de_iir & DE_POISON)
 706                 drm_err(&dev_priv->drm, "Poison interrupt\n");
 707
 708         for_each_pipe(dev_priv, pipe) {
 709                 if (de_iir & DE_PIPE_VBLANK(pipe))
 710                         intel_handle_vblank(dev_priv, pipe);
 711
 712                 if (de_iir & DE_PLANE_FLIP_DONE(pipe))
 713                         flip_done_handler(dev_priv, pipe);
 714
 715                 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
 716                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
 717
 718                 if (de_iir & DE_PIPE_CRC_DONE(pipe))
 719                         i9xx_pipe_crc_irq_handler(dev_priv, pipe);
 720         }
 721
 722         /* check event from PCH */
 723         if (de_iir & DE_PCH_EVENT) {
 724                 u32 pch_iir = intel_uncore_read(&dev_priv->uncore, SDEIIR);
 725
 726                 if (HAS_PCH_CPT(dev_priv))
 727                         cpt_irq_handler(dev_priv, pch_iir);
 728                 else
 729                         ibx_irq_handler(dev_priv, pch_iir);
 730
 731                 /* should clear PCH hotplug event before clear CPU irq */
 732                 intel_uncore_write(&dev_priv->uncore, SDEIIR, pch_iir);
 733         }
 734
 735         if (DISPLAY_VER(dev_priv) == 5 && de_iir & DE_PCU_EVENT)
 736                 gen5_rps_irq_handler(&to_gt(dev_priv)->rps);
 737 }
 738
 739 void ivb_display_irq_handler(struct drm_i915_private *dev_priv, u32 de_iir)
 740 {
 741         enum pipe pipe;
 742         u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
 743
 744         if (hotplug_trigger)
 745                 ilk_hpd_irq_handler(dev_priv, hotplug_trigger);
 746
 747         if (de_iir & DE_ERR_INT_IVB)
 748                 ivb_err_int_handler(dev_priv);
 749
 750         if (de_iir & DE_EDP_PSR_INT_HSW) {
 751                 struct intel_encoder *encoder;
 752
 753                 for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
 754                         struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 755                         u32 psr_iir;
 756
 757                         psr_iir = intel_uncore_rmw(&dev_priv->uncore,
 758                                                    EDP_PSR_IIR, 0, 0);
 759                         intel_psr_irq_handler(intel_dp, psr_iir);
 760                         break;
 761                 }
 762         }
 763
 764         if (de_iir & DE_AUX_CHANNEL_A_IVB)
 765                 intel_dp_aux_irq_handler(dev_priv);
 766
 767         if (de_iir & DE_GSE_IVB)
 768                 intel_opregion_asle_intr(dev_priv);
 769
 770         for_each_pipe(dev_priv, pipe) {
 771                 if (de_iir & DE_PIPE_VBLANK_IVB(pipe))
 772                         intel_handle_vblank(dev_priv, pipe);
 773
 774                 if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe))
 775                         flip_done_handler(dev_priv, pipe);
 776         }
 777
 778         /* check event from PCH */
 779         if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) {
 780                 u32 pch_iir = intel_uncore_read(&dev_priv->uncore, SDEIIR);
 781
 782                 cpt_irq_handler(dev_priv, pch_iir);
 783
 784                 /* clear PCH hotplug event before clear CPU irq */
 785                 intel_uncore_write(&dev_priv->uncore, SDEIIR, pch_iir);
 786         }
 787 }
 788
 789 static u32 gen8_de_port_aux_mask(struct drm_i915_private *dev_priv)
 790 {
 791         u32 mask;
 792
 793         if (DISPLAY_VER(dev_priv) >= 20)
 794                 return 0;
 795         else if (DISPLAY_VER(dev_priv) >= 14)
 796                 return TGL_DE_PORT_AUX_DDIA |
 797                         TGL_DE_PORT_AUX_DDIB;
 798         else if (DISPLAY_VER(dev_priv) >= 13)
 799                 return TGL_DE_PORT_AUX_DDIA |
 800                         TGL_DE_PORT_AUX_DDIB |
 801                         TGL_DE_PORT_AUX_DDIC |
 802                         XELPD_DE_PORT_AUX_DDID |
 803                         XELPD_DE_PORT_AUX_DDIE |
 804                         TGL_DE_PORT_AUX_USBC1 |
 805                         TGL_DE_PORT_AUX_USBC2 |
 806                         TGL_DE_PORT_AUX_USBC3 |
 807                         TGL_DE_PORT_AUX_USBC4;
 808         else if (DISPLAY_VER(dev_priv) >= 12)
 809                 return TGL_DE_PORT_AUX_DDIA |
 810                         TGL_DE_PORT_AUX_DDIB |
 811                         TGL_DE_PORT_AUX_DDIC |
 812                         TGL_DE_PORT_AUX_USBC1 |
 813                         TGL_DE_PORT_AUX_USBC2 |
 814                         TGL_DE_PORT_AUX_USBC3 |
 815                         TGL_DE_PORT_AUX_USBC4 |
 816                         TGL_DE_PORT_AUX_USBC5 |
 817                         TGL_DE_PORT_AUX_USBC6;
 818
 819         mask = GEN8_AUX_CHANNEL_A;
 820         if (DISPLAY_VER(dev_priv) >= 9)
 821                 mask |= GEN9_AUX_CHANNEL_B |
 822                         GEN9_AUX_CHANNEL_C |
 823                         GEN9_AUX_CHANNEL_D;
 824
 825         if (DISPLAY_VER(dev_priv) == 11) {
 826                 mask |= ICL_AUX_CHANNEL_F;
 827                 mask |= ICL_AUX_CHANNEL_E;
 828         }
 829
 830         return mask;
 831 }
 832
 833 static u32 gen8_de_pipe_fault_mask(struct drm_i915_private *dev_priv)
 834 {
 835         if (DISPLAY_VER(dev_priv) >= 13 || HAS_D12_PLANE_MINIMIZATION(dev_priv))
 836                 return RKL_DE_PIPE_IRQ_FAULT_ERRORS;
 837         else if (DISPLAY_VER(dev_priv) >= 11)
 838                 return GEN11_DE_PIPE_IRQ_FAULT_ERRORS;
 839         else if (DISPLAY_VER(dev_priv) >= 9)
 840                 return GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
 841         else
 842                 return GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
 843 }
 844
 845 static void intel_pmdemand_irq_handler(struct drm_i915_private *dev_priv)
 846 {
 847         wake_up_all(&dev_priv->display.pmdemand.waitqueue);
 848 }
 849
 850 static void
 851 gen8_de_misc_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
 852 {
 853         bool found = false;
 854
 855         if (DISPLAY_VER(dev_priv) >= 14) {
 856                 if (iir & (XELPDP_PMDEMAND_RSP |
 857                            XELPDP_PMDEMAND_RSPTOUT_ERR)) {
 858                         if (iir & XELPDP_PMDEMAND_RSPTOUT_ERR)
 859                                 drm_dbg(&dev_priv->drm,
 860                                         "Error waiting for Punit PM Demand Response\n");
 861
 862                         intel_pmdemand_irq_handler(dev_priv);
 863                         found = true;
 864                 }
 865         } else if (iir & GEN8_DE_MISC_GSE) {
 866                 intel_opregion_asle_intr(dev_priv);
 867                 found = true;
 868         }
 869
 870         if (iir & GEN8_DE_EDP_PSR) {
 871                 struct intel_encoder *encoder;
 872                 u32 psr_iir;
 873                 i915_reg_t iir_reg;
 874
 875                 for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
 876                         struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 877
 878                         if (DISPLAY_VER(dev_priv) >= 12)
 879                                 iir_reg = TRANS_PSR_IIR(intel_dp->psr.transcoder);
 880                         else
 881                                 iir_reg = EDP_PSR_IIR;
 882
 883                         psr_iir = intel_uncore_rmw(&dev_priv->uncore, iir_reg, 0, 0);
 884
 885                         if (psr_iir)
 886                                 found = true;
 887
 888                         intel_psr_irq_handler(intel_dp, psr_iir);
 889
 890                         /* prior GEN12 only have one EDP PSR */
 891                         if (DISPLAY_VER(dev_priv) < 12)
 892                                 break;
 893                 }
 894         }
 895
 896         if (!found)
 897                 drm_err(&dev_priv->drm, "Unexpected DE Misc interrupt: 0x%08x\n", iir);
 898 }
 899
 900 static void gen11_dsi_te_interrupt_handler(struct drm_i915_private *dev_priv,
 901                                            u32 te_trigger)
 902 {
 903         enum pipe pipe = INVALID_PIPE;
 904         enum transcoder dsi_trans;
 905         enum port port;
 906         u32 val;
 907
 908         /*
 909          * Incase of dual link, TE comes from DSI_1
 910          * this is to check if dual link is enabled
 911          */
 912         val = intel_uncore_read(&dev_priv->uncore, TRANS_DDI_FUNC_CTL2(TRANSCODER_DSI_0));
 913         val &= PORT_SYNC_MODE_ENABLE;
 914
 915         /*
 916          * if dual link is enabled, then read DSI_0
 917          * transcoder registers
 918          */
 919         port = ((te_trigger & DSI1_TE && val) || (te_trigger & DSI0_TE)) ?
 920                                                   PORT_A : PORT_B;
 921         dsi_trans = (port == PORT_A) ? TRANSCODER_DSI_0 : TRANSCODER_DSI_1;
 922
 923         /* Check if DSI configured in command mode */
 924         val = intel_uncore_read(&dev_priv->uncore, DSI_TRANS_FUNC_CONF(dsi_trans));
 925         val = val & OP_MODE_MASK;
 926
 927         if (val != CMD_MODE_NO_GATE && val != CMD_MODE_TE_GATE) {
 928                 drm_err(&dev_priv->drm, "DSI trancoder not configured in command mode\n");
 929                 return;
 930         }
 931
 932         /* Get PIPE for handling VBLANK event */
 933         val = intel_uncore_read(&dev_priv->uncore, TRANS_DDI_FUNC_CTL(dsi_trans));
 934         switch (val & TRANS_DDI_EDP_INPUT_MASK) {
 935         case TRANS_DDI_EDP_INPUT_A_ON:
 936                 pipe = PIPE_A;
 937                 break;
 938         case TRANS_DDI_EDP_INPUT_B_ONOFF:
 939                 pipe = PIPE_B;
 940                 break;
 941         case TRANS_DDI_EDP_INPUT_C_ONOFF:
 942                 pipe = PIPE_C;
 943                 break;
 944         default:
 945                 drm_err(&dev_priv->drm, "Invalid PIPE\n");
 946                 return;
 947         }
 948
 949         intel_handle_vblank(dev_priv, pipe);
 950
 951         /* clear TE in dsi IIR */
 952         port = (te_trigger & DSI1_TE) ? PORT_B : PORT_A;
 953         intel_uncore_rmw(&dev_priv->uncore, DSI_INTR_IDENT_REG(port), 0, 0);
 954 }
 955
 956 static u32 gen8_de_pipe_flip_done_mask(struct drm_i915_private *i915)
 957 {
 958         if (DISPLAY_VER(i915) >= 9)
 959                 return GEN9_PIPE_PLANE1_FLIP_DONE;
 960         else
 961                 return GEN8_PIPE_PRIMARY_FLIP_DONE;
 962 }
 963
 964 u32 gen8_de_pipe_underrun_mask(struct drm_i915_private *dev_priv)
 965 {
 966         u32 mask = GEN8_PIPE_FIFO_UNDERRUN;
 967
 968         if (DISPLAY_VER(dev_priv) >= 13)
 969                 mask |= XELPD_PIPE_SOFT_UNDERRUN |
 970                         XELPD_PIPE_HARD_UNDERRUN;
 971
 972         return mask;
 973 }
 974
 975 static void gen8_read_and_ack_pch_irqs(struct drm_i915_private *i915, u32 *pch_iir, u32 *pica_iir)
 976 {
 977         u32 pica_ier = 0;
 978
 979         *pica_iir = 0;
 980         *pch_iir = intel_de_read(i915, SDEIIR);
 981         if (!*pch_iir)
 982                 return;
 983
 984         /**
 985          * PICA IER must be disabled/re-enabled around clearing PICA IIR and
 986          * SDEIIR, to avoid losing PICA IRQs and to ensure that such IRQs set
 987          * their flags both in the PICA and SDE IIR.
 988          */
 989         if (*pch_iir & SDE_PICAINTERRUPT) {
 990                 drm_WARN_ON(&i915->drm, INTEL_PCH_TYPE(i915) < PCH_MTL);
 991
 992                 pica_ier = intel_de_rmw(i915, PICAINTERRUPT_IER, ~0, 0);
 993                 *pica_iir = intel_de_read(i915, PICAINTERRUPT_IIR);
 994                 intel_de_write(i915, PICAINTERRUPT_IIR, *pica_iir);
 995         }
 996
 997         intel_de_write(i915, SDEIIR, *pch_iir);
 998
 999         if (pica_ier)
1000                 intel_de_write(i915, PICAINTERRUPT_IER, pica_ier);
1001 }
1002
1003 void gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
1004 {
1005         u32 iir;
1006         enum pipe pipe;
1007
1008         drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_DISPLAY(dev_priv));
1009
1010         if (master_ctl & GEN8_DE_MISC_IRQ) {
1011                 iir = intel_uncore_read(&dev_priv->uncore, GEN8_DE_MISC_IIR);
1012                 if (iir) {
1013                         intel_uncore_write(&dev_priv->uncore, GEN8_DE_MISC_IIR, iir);
1014                         gen8_de_misc_irq_handler(dev_priv, iir);
1015                 } else {
1016                         drm_err_ratelimited(&dev_priv->drm,
1017                                             "The master control interrupt lied (DE MISC)!\n");
1018                 }
1019         }
1020
1021         if (DISPLAY_VER(dev_priv) >= 11 && (master_ctl & GEN11_DE_HPD_IRQ)) {
1022                 iir = intel_uncore_read(&dev_priv->uncore, GEN11_DE_HPD_IIR);
1023                 if (iir) {
1024                         intel_uncore_write(&dev_priv->uncore, GEN11_DE_HPD_IIR, iir);
1025                         gen11_hpd_irq_handler(dev_priv, iir);
1026                 } else {
1027                         drm_err_ratelimited(&dev_priv->drm,
1028                                             "The master control interrupt lied, (DE HPD)!\n");
1029                 }
1030         }
1031
1032         if (master_ctl & GEN8_DE_PORT_IRQ) {
1033                 iir = intel_uncore_read(&dev_priv->uncore, GEN8_DE_PORT_IIR);
1034                 if (iir) {
1035                         bool found = false;
1036
1037                         intel_uncore_write(&dev_priv->uncore, GEN8_DE_PORT_IIR, iir);
1038
1039                         if (iir & gen8_de_port_aux_mask(dev_priv)) {
1040                                 intel_dp_aux_irq_handler(dev_priv);
1041                                 found = true;
1042                         }
1043
1044                         if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
1045                                 u32 hotplug_trigger = iir & BXT_DE_PORT_HOTPLUG_MASK;
1046
1047                                 if (hotplug_trigger) {
1048                                         bxt_hpd_irq_handler(dev_priv, hotplug_trigger);
1049                                         found = true;
1050                                 }
1051                         } else if (IS_BROADWELL(dev_priv)) {
1052                                 u32 hotplug_trigger = iir & BDW_DE_PORT_HOTPLUG_MASK;
1053
1054                                 if (hotplug_trigger) {
1055                                         ilk_hpd_irq_handler(dev_priv, hotplug_trigger);
1056                                         found = true;
1057                                 }
1058                         }
1059
1060                         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
1061                             (iir & BXT_DE_PORT_GMBUS)) {
1062                                 intel_gmbus_irq_handler(dev_priv);
1063                                 found = true;
1064                         }
1065
1066                         if (DISPLAY_VER(dev_priv) >= 11) {
1067                                 u32 te_trigger = iir & (DSI0_TE | DSI1_TE);
1068
1069                                 if (te_trigger) {
1070                                         gen11_dsi_te_interrupt_handler(dev_priv, te_trigger);
1071                                         found = true;
1072                                 }
1073                         }
1074
1075                         if (!found)
1076                                 drm_err_ratelimited(&dev_priv->drm,
1077                                                     "Unexpected DE Port interrupt\n");
1078                 } else {
1079                         drm_err_ratelimited(&dev_priv->drm,
1080                                             "The master control interrupt lied (DE PORT)!\n");
1081                 }
1082         }
1083
1084         for_each_pipe(dev_priv, pipe) {
1085                 u32 fault_errors;
1086
1087                 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
1088                         continue;
1089
1090                 iir = intel_uncore_read(&dev_priv->uncore, GEN8_DE_PIPE_IIR(pipe));
1091                 if (!iir) {
1092                         drm_err_ratelimited(&dev_priv->drm,
1093                                             "The master control interrupt lied (DE PIPE)!\n");
1094                         continue;
1095                 }
1096
1097                 intel_uncore_write(&dev_priv->uncore, GEN8_DE_PIPE_IIR(pipe), iir);
1098
1099                 if (iir & GEN8_PIPE_VBLANK)
1100                         intel_handle_vblank(dev_priv, pipe);
1101
1102                 if (iir & gen8_de_pipe_flip_done_mask(dev_priv))
1103                         flip_done_handler(dev_priv, pipe);
1104
1105                 if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
1106                         hsw_pipe_crc_irq_handler(dev_priv, pipe);
1107
1108                 if (iir & gen8_de_pipe_underrun_mask(dev_priv))
1109                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1110
1111                 fault_errors = iir & gen8_de_pipe_fault_mask(dev_priv);
1112                 if (fault_errors)
1113                         drm_err_ratelimited(&dev_priv->drm,
1114                                             "Fault errors on pipe %c: 0x%08x\n",
1115                                             pipe_name(pipe),
1116                                             fault_errors);
1117         }
1118
1119         if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) &&
1120             master_ctl & GEN8_DE_PCH_IRQ) {
1121                 u32 pica_iir;
1122
1123                 /*
1124                  * FIXME(BDW): Assume for now that the new interrupt handling
1125                  * scheme also closed the SDE interrupt handling race we've seen
1126                  * on older pch-split platforms. But this needs testing.
1127                  */
1128                 gen8_read_and_ack_pch_irqs(dev_priv, &iir, &pica_iir);
1129                 if (iir) {
1130                         if (pica_iir)
1131                                 xelpdp_pica_irq_handler(dev_priv, pica_iir);
1132
1133                         if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
1134                                 icp_irq_handler(dev_priv, iir);
1135                         else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
1136                                 spt_irq_handler(dev_priv, iir);
1137                         else
1138                                 cpt_irq_handler(dev_priv, iir);
1139                 } else {
1140                         /*
1141                          * Like on previous PCH there seems to be something
1142                          * fishy going on with forwarding PCH interrupts.
1143                          */
1144                         drm_dbg(&dev_priv->drm,
1145                                 "The master control interrupt lied (SDE)!\n");
1146                 }
1147         }
1148 }
1149
1150 u32 gen11_gu_misc_irq_ack(struct drm_i915_private *i915, const u32 master_ctl)
1151 {
1152         void __iomem * const regs = intel_uncore_regs(&i915->uncore);
1153         u32 iir;
1154
1155         if (!(master_ctl & GEN11_GU_MISC_IRQ))
1156                 return 0;
1157
1158         iir = raw_reg_read(regs, GEN11_GU_MISC_IIR);
1159         if (likely(iir))
1160                 raw_reg_write(regs, GEN11_GU_MISC_IIR, iir);
1161
1162         return iir;
1163 }
1164
1165 void gen11_gu_misc_irq_handler(struct drm_i915_private *i915, const u32 iir)
1166 {
1167         if (iir & GEN11_GU_MISC_GSE)
1168                 intel_opregion_asle_intr(i915);
1169 }
1170
1171 void gen11_display_irq_handler(struct drm_i915_private *i915)
1172 {
1173         void __iomem * const regs = intel_uncore_regs(&i915->uncore);
1174         const u32 disp_ctl = raw_reg_read(regs, GEN11_DISPLAY_INT_CTL);
1175
1176         disable_rpm_wakeref_asserts(&i915->runtime_pm);
1177         /*
1178          * GEN11_DISPLAY_INT_CTL has same format as GEN8_MASTER_IRQ
1179          * for the display related bits.
1180          */
1181         raw_reg_write(regs, GEN11_DISPLAY_INT_CTL, 0x0);
1182         gen8_de_irq_handler(i915, disp_ctl);
1183         raw_reg_write(regs, GEN11_DISPLAY_INT_CTL,
1184                       GEN11_DISPLAY_IRQ_ENABLE);
1185
1186         enable_rpm_wakeref_asserts(&i915->runtime_pm);
1187 }
1188
1189 /* Called from drm generic code, passed 'crtc' which
1190  * we use as a pipe index
1191  */
1192 int i8xx_enable_vblank(struct drm_crtc *crtc)
1193 {
1194         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
1195         enum pipe pipe = to_intel_crtc(crtc)->pipe;
1196         unsigned long irqflags;
1197
1198         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1199         i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
1200         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1201
1202         return 0;
1203 }
1204
1205 int i915gm_enable_vblank(struct drm_crtc *crtc)
1206 {
1207         struct drm_i915_private *i915 = to_i915(crtc->dev);
1208
1209         /*
1210          * Vblank interrupts fail to wake the device up from C2+.
1211          * Disabling render clock gating during C-states avoids
1212          * the problem. There is a small power cost so we do this
1213          * only when vblank interrupts are actually enabled.
1214          */
1215         if (i915->display.irq.vblank_enabled++ == 0)
1216                 intel_uncore_write(&i915->uncore, SCPD0, _MASKED_BIT_ENABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
1217
1218         return i8xx_enable_vblank(crtc);
1219 }
1220
1221 int i965_enable_vblank(struct drm_crtc *crtc)
1222 {
1223         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
1224         enum pipe pipe = to_intel_crtc(crtc)->pipe;
1225         unsigned long irqflags;
1226
1227         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1228         i915_enable_pipestat(dev_priv, pipe,
1229                              PIPE_START_VBLANK_INTERRUPT_STATUS);
1230         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1231
1232         return 0;
1233 }
1234
1235 int ilk_enable_vblank(struct drm_crtc *crtc)
1236 {
1237         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
1238         enum pipe pipe = to_intel_crtc(crtc)->pipe;
1239         unsigned long irqflags;
1240         u32 bit = DISPLAY_VER(dev_priv) >= 7 ?
1241                 DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
1242
1243         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1244         ilk_enable_display_irq(dev_priv, bit);
1245         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1246
1247         /* Even though there is no DMC, frame counter can get stuck when
1248          * PSR is active as no frames are generated.
1249          */
1250         if (HAS_PSR(dev_priv))
1251                 drm_crtc_vblank_restore(crtc);
1252
1253         return 0;
1254 }
1255
1256 static bool gen11_dsi_configure_te(struct intel_crtc *intel_crtc,
1257                                    bool enable)
1258 {
1259         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
1260         enum port port;
1261
1262         if (!(intel_crtc->mode_flags &
1263             (I915_MODE_FLAG_DSI_USE_TE1 | I915_MODE_FLAG_DSI_USE_TE0)))
1264                 return false;
1265
1266         /* for dual link cases we consider TE from slave */
1267         if (intel_crtc->mode_flags & I915_MODE_FLAG_DSI_USE_TE1)
1268                 port = PORT_B;
1269         else
1270                 port = PORT_A;
1271
1272         intel_uncore_rmw(&dev_priv->uncore, DSI_INTR_MASK_REG(port), DSI_TE_EVENT,
1273                          enable ? 0 : DSI_TE_EVENT);
1274
1275         intel_uncore_rmw(&dev_priv->uncore, DSI_INTR_IDENT_REG(port), 0, 0);
1276
1277         return true;
1278 }
1279
1280 int bdw_enable_vblank(struct drm_crtc *_crtc)
1281 {
1282         struct intel_crtc *crtc = to_intel_crtc(_crtc);
1283         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1284         enum pipe pipe = crtc->pipe;
1285         unsigned long irqflags;
1286
1287         if (gen11_dsi_configure_te(crtc, true))
1288                 return 0;
1289
1290         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1291         bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
1292         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1293
1294         /* Even if there is no DMC, frame counter can get stuck when
1295          * PSR is active as no frames are generated, so check only for PSR.
1296          */
1297         if (HAS_PSR(dev_priv))
1298                 drm_crtc_vblank_restore(&crtc->base);
1299
1300         return 0;
1301 }
1302
1303 /* Called from drm generic code, passed 'crtc' which
1304  * we use as a pipe index
1305  */
1306 void i8xx_disable_vblank(struct drm_crtc *crtc)
1307 {
1308         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
1309         enum pipe pipe = to_intel_crtc(crtc)->pipe;
1310         unsigned long irqflags;
1311
1312         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1313         i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
1314         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1315 }
1316
1317 void i915gm_disable_vblank(struct drm_crtc *crtc)
1318 {
1319         struct drm_i915_private *i915 = to_i915(crtc->dev);
1320
1321         i8xx_disable_vblank(crtc);
1322
1323         if (--i915->display.irq.vblank_enabled == 0)
1324                 intel_uncore_write(&i915->uncore, SCPD0, _MASKED_BIT_DISABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
1325 }
1326
1327 void i965_disable_vblank(struct drm_crtc *crtc)
1328 {
1329         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
1330         enum pipe pipe = to_intel_crtc(crtc)->pipe;
1331         unsigned long irqflags;
1332
1333         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1334         i915_disable_pipestat(dev_priv, pipe,
1335                               PIPE_START_VBLANK_INTERRUPT_STATUS);
1336         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1337 }
1338
1339 void ilk_disable_vblank(struct drm_crtc *crtc)
1340 {
1341         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
1342         enum pipe pipe = to_intel_crtc(crtc)->pipe;
1343         unsigned long irqflags;
1344         u32 bit = DISPLAY_VER(dev_priv) >= 7 ?
1345                 DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
1346
1347         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1348         ilk_disable_display_irq(dev_priv, bit);
1349         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1350 }
1351
1352 void bdw_disable_vblank(struct drm_crtc *_crtc)
1353 {
1354         struct intel_crtc *crtc = to_intel_crtc(_crtc);
1355         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1356         enum pipe pipe = crtc->pipe;
1357         unsigned long irqflags;
1358
1359         if (gen11_dsi_configure_te(crtc, false))
1360                 return;
1361
1362         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1363         bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
1364         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1365 }
1366
1367 void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
1368 {
1369         struct intel_uncore *uncore = &dev_priv->uncore;
1370
1371         if (IS_CHERRYVIEW(dev_priv))
1372                 intel_uncore_write(uncore, DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
1373         else
1374                 intel_uncore_write(uncore, DPINVGTT, DPINVGTT_STATUS_MASK_VLV);
1375
1376         i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0);
1377         intel_uncore_rmw(uncore, PORT_HOTPLUG_STAT, 0, 0);
1378
1379         i9xx_pipestat_irq_reset(dev_priv);
1380
1381         GEN3_IRQ_RESET(uncore, VLV_);
1382         dev_priv->irq_mask = ~0u;
1383 }
1384
1385 void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
1386 {
1387         struct intel_uncore *uncore = &dev_priv->uncore;
1388
1389         u32 pipestat_mask;
1390         u32 enable_mask;
1391         enum pipe pipe;
1392
1393         pipestat_mask = PIPE_CRC_DONE_INTERRUPT_STATUS;
1394
1395         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
1396         for_each_pipe(dev_priv, pipe)
1397                 i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
1398
1399         enable_mask = I915_DISPLAY_PORT_INTERRUPT |
1400                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
1401                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
1402                 I915_LPE_PIPE_A_INTERRUPT |
1403                 I915_LPE_PIPE_B_INTERRUPT;
1404
1405         if (IS_CHERRYVIEW(dev_priv))
1406                 enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT |
1407                         I915_LPE_PIPE_C_INTERRUPT;
1408
1409         drm_WARN_ON(&dev_priv->drm, dev_priv->irq_mask != ~0u);
1410
1411         dev_priv->irq_mask = ~enable_mask;
1412
1413         GEN3_IRQ_INIT(uncore, VLV_, dev_priv->irq_mask, enable_mask);
1414 }
1415
1416 void gen8_display_irq_reset(struct drm_i915_private *dev_priv)
1417 {
1418         struct intel_uncore *uncore = &dev_priv->uncore;
1419         enum pipe pipe;
1420
1421         if (!HAS_DISPLAY(dev_priv))
1422                 return;
1423
1424         intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
1425         intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
1426
1427         for_each_pipe(dev_priv, pipe)
1428                 if (intel_display_power_is_enabled(dev_priv,
1429                                                    POWER_DOMAIN_PIPE(pipe)))
1430                         GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
1431
1432         GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_);
1433         GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_);
1434 }
1435
1436 void gen11_display_irq_reset(struct drm_i915_private *dev_priv)
1437 {
1438         struct intel_uncore *uncore = &dev_priv->uncore;
1439         enum pipe pipe;
1440         u32 trans_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
1441                 BIT(TRANSCODER_C) | BIT(TRANSCODER_D);
1442
1443         if (!HAS_DISPLAY(dev_priv))
1444                 return;
1445
1446         intel_uncore_write(uncore, GEN11_DISPLAY_INT_CTL, 0);
1447
1448         if (DISPLAY_VER(dev_priv) >= 12) {
1449                 enum transcoder trans;
1450
1451                 for_each_cpu_transcoder_masked(dev_priv, trans, trans_mask) {
1452                         enum intel_display_power_domain domain;
1453
1454                         domain = POWER_DOMAIN_TRANSCODER(trans);
1455                         if (!intel_display_power_is_enabled(dev_priv, domain))
1456                                 continue;
1457
1458                         intel_uncore_write(uncore, TRANS_PSR_IMR(trans), 0xffffffff);
1459                         intel_uncore_write(uncore, TRANS_PSR_IIR(trans), 0xffffffff);
1460                 }
1461         } else {
1462                 intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
1463                 intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
1464         }
1465
1466         for_each_pipe(dev_priv, pipe)
1467                 if (intel_display_power_is_enabled(dev_priv,
1468                                                    POWER_DOMAIN_PIPE(pipe)))
1469                         GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
1470
1471         GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_);
1472         GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_);
1473
1474         if (DISPLAY_VER(dev_priv) >= 14)
1475                 GEN3_IRQ_RESET(uncore, PICAINTERRUPT_);
1476         else
1477                 GEN3_IRQ_RESET(uncore, GEN11_DE_HPD_);
1478
1479         if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
1480                 GEN3_IRQ_RESET(uncore, SDE);
1481 }
1482
1483 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
1484                                      u8 pipe_mask)
1485 {
1486         struct intel_uncore *uncore = &dev_priv->uncore;
1487         u32 extra_ier = GEN8_PIPE_VBLANK |
1488                 gen8_de_pipe_underrun_mask(dev_priv) |
1489                 gen8_de_pipe_flip_done_mask(dev_priv);
1490         enum pipe pipe;
1491
1492         spin_lock_irq(&dev_priv->irq_lock);
1493
1494         if (!intel_irqs_enabled(dev_priv)) {
1495                 spin_unlock_irq(&dev_priv->irq_lock);
1496                 return;
1497         }
1498
1499         for_each_pipe_masked(dev_priv, pipe, pipe_mask)
1500                 GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
1501                                   dev_priv->display.irq.de_irq_mask[pipe],
1502                                   ~dev_priv->display.irq.de_irq_mask[pipe] | extra_ier);
1503
1504         spin_unlock_irq(&dev_priv->irq_lock);
1505 }
1506
1507 void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
1508                                      u8 pipe_mask)
1509 {
1510         struct intel_uncore *uncore = &dev_priv->uncore;
1511         enum pipe pipe;
1512
1513         spin_lock_irq(&dev_priv->irq_lock);
1514
1515         if (!intel_irqs_enabled(dev_priv)) {
1516                 spin_unlock_irq(&dev_priv->irq_lock);
1517                 return;
1518         }
1519
1520         for_each_pipe_masked(dev_priv, pipe, pipe_mask)
1521                 GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
1522
1523         spin_unlock_irq(&dev_priv->irq_lock);
1524
1525         /* make sure we're done processing display irqs */
1526         intel_synchronize_irq(dev_priv);
1527 }
1528
1529 /*
1530  * SDEIER is also touched by the interrupt handler to work around missed PCH
1531  * interrupts. Hence we can't update it after the interrupt handler is enabled -
1532  * instead we unconditionally enable all PCH interrupt sources here, but then
1533  * only unmask them as needed with SDEIMR.
1534  *
1535  * Note that we currently do this after installing the interrupt handler,
1536  * but before we enable the master interrupt. That should be sufficient
1537  * to avoid races with the irq handler, assuming we have MSI. Shared legacy
1538  * interrupts could still race.
1539  */
1540 static void ibx_irq_postinstall(struct drm_i915_private *dev_priv)
1541 {
1542         struct intel_uncore *uncore = &dev_priv->uncore;
1543         u32 mask;
1544
1545         if (HAS_PCH_NOP(dev_priv))
1546                 return;
1547
1548         if (HAS_PCH_IBX(dev_priv))
1549                 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
1550         else if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
1551                 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
1552         else
1553                 mask = SDE_GMBUS_CPT;
1554
1555         GEN3_IRQ_INIT(uncore, SDE, ~mask, 0xffffffff);
1556 }
1557
1558 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
1559 {
1560         lockdep_assert_held(&dev_priv->irq_lock);
1561
1562         if (dev_priv->display.irq.display_irqs_enabled)
1563                 return;
1564
1565         dev_priv->display.irq.display_irqs_enabled = true;
1566
1567         if (intel_irqs_enabled(dev_priv)) {
1568                 vlv_display_irq_reset(dev_priv);
1569                 vlv_display_irq_postinstall(dev_priv);
1570         }
1571 }
1572
1573 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
1574 {
1575         lockdep_assert_held(&dev_priv->irq_lock);
1576
1577         if (!dev_priv->display.irq.display_irqs_enabled)
1578                 return;
1579
1580         dev_priv->display.irq.display_irqs_enabled = false;
1581
1582         if (intel_irqs_enabled(dev_priv))
1583                 vlv_display_irq_reset(dev_priv);
1584 }
1585
1586 void ilk_de_irq_postinstall(struct drm_i915_private *i915)
1587 {
1588         struct intel_uncore *uncore = &i915->uncore;
1589         u32 display_mask, extra_mask;
1590
1591         if (DISPLAY_VER(i915) >= 7) {
1592                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
1593                                 DE_PCH_EVENT_IVB | DE_AUX_CHANNEL_A_IVB);
1594                 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
1595                               DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
1596                               DE_PLANE_FLIP_DONE_IVB(PLANE_C) |
1597                               DE_PLANE_FLIP_DONE_IVB(PLANE_B) |
1598                               DE_PLANE_FLIP_DONE_IVB(PLANE_A) |
1599                               DE_DP_A_HOTPLUG_IVB);
1600         } else {
1601                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
1602                                 DE_AUX_CHANNEL_A | DE_PIPEB_CRC_DONE |
1603                                 DE_PIPEA_CRC_DONE | DE_POISON);
1604                 extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK |
1605                               DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
1606                               DE_PLANE_FLIP_DONE(PLANE_A) |
1607                               DE_PLANE_FLIP_DONE(PLANE_B) |
1608                               DE_DP_A_HOTPLUG);
1609         }
1610
1611         if (IS_HASWELL(i915)) {
1612                 gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR);
1613                 display_mask |= DE_EDP_PSR_INT_HSW;
1614         }
1615
1616         if (IS_IRONLAKE_M(i915))
1617                 extra_mask |= DE_PCU_EVENT;
1618
1619         i915->irq_mask = ~display_mask;
1620
1621         ibx_irq_postinstall(i915);
1622
1623         GEN3_IRQ_INIT(uncore, DE, i915->irq_mask,
1624                       display_mask | extra_mask);
1625 }
1626
1627 static void mtp_irq_postinstall(struct drm_i915_private *i915);
1628 static void icp_irq_postinstall(struct drm_i915_private *i915);
1629
1630 void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
1631 {
1632         struct intel_uncore *uncore = &dev_priv->uncore;
1633
1634         u32 de_pipe_masked = gen8_de_pipe_fault_mask(dev_priv) |
1635                 GEN8_PIPE_CDCLK_CRC_DONE;
1636         u32 de_pipe_enables;
1637         u32 de_port_masked = gen8_de_port_aux_mask(dev_priv);
1638         u32 de_port_enables;
1639         u32 de_misc_masked = GEN8_DE_EDP_PSR;
1640         u32 trans_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
1641                 BIT(TRANSCODER_C) | BIT(TRANSCODER_D);
1642         enum pipe pipe;
1643
1644         if (!HAS_DISPLAY(dev_priv))
1645                 return;
1646
1647         if (DISPLAY_VER(dev_priv) >= 14)
1648                 mtp_irq_postinstall(dev_priv);
1649         else if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
1650                 icp_irq_postinstall(dev_priv);
1651         else if (HAS_PCH_SPLIT(dev_priv))
1652                 ibx_irq_postinstall(dev_priv);
1653
1654         if (DISPLAY_VER(dev_priv) < 11)
1655                 de_misc_masked |= GEN8_DE_MISC_GSE;
1656
1657         if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
1658                 de_port_masked |= BXT_DE_PORT_GMBUS;
1659
1660         if (DISPLAY_VER(dev_priv) >= 14) {
1661                 de_misc_masked |= XELPDP_PMDEMAND_RSPTOUT_ERR |
1662                                   XELPDP_PMDEMAND_RSP;
1663         } else if (DISPLAY_VER(dev_priv) >= 11) {
1664                 enum port port;
1665
1666                 if (intel_bios_is_dsi_present(dev_priv, &port))
1667                         de_port_masked |= DSI0_TE | DSI1_TE;
1668         }
1669
1670         de_pipe_enables = de_pipe_masked |
1671                 GEN8_PIPE_VBLANK |
1672                 gen8_de_pipe_underrun_mask(dev_priv) |
1673                 gen8_de_pipe_flip_done_mask(dev_priv);
1674
1675         de_port_enables = de_port_masked;
1676         if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
1677                 de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
1678         else if (IS_BROADWELL(dev_priv))
1679                 de_port_enables |= BDW_DE_PORT_HOTPLUG_MASK;
1680
1681         if (DISPLAY_VER(dev_priv) >= 12) {
1682                 enum transcoder trans;
1683
1684                 for_each_cpu_transcoder_masked(dev_priv, trans, trans_mask) {
1685                         enum intel_display_power_domain domain;
1686
1687                         domain = POWER_DOMAIN_TRANSCODER(trans);
1688                         if (!intel_display_power_is_enabled(dev_priv, domain))
1689                                 continue;
1690
1691                         gen3_assert_iir_is_zero(uncore, TRANS_PSR_IIR(trans));
1692                 }
1693         } else {
1694                 gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR);
1695         }
1696
1697         for_each_pipe(dev_priv, pipe) {
1698                 dev_priv->display.irq.de_irq_mask[pipe] = ~de_pipe_masked;
1699
1700                 if (intel_display_power_is_enabled(dev_priv,
1701                                                    POWER_DOMAIN_PIPE(pipe)))
1702                         GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
1703                                           dev_priv->display.irq.de_irq_mask[pipe],
1704                                           de_pipe_enables);
1705         }
1706
1707         GEN3_IRQ_INIT(uncore, GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
1708         GEN3_IRQ_INIT(uncore, GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
1709
1710         if (IS_DISPLAY_VER(dev_priv, 11, 13)) {
1711                 u32 de_hpd_masked = 0;
1712                 u32 de_hpd_enables = GEN11_DE_TC_HOTPLUG_MASK |
1713                                      GEN11_DE_TBT_HOTPLUG_MASK;
1714
1715                 GEN3_IRQ_INIT(uncore, GEN11_DE_HPD_, ~de_hpd_masked,
1716                               de_hpd_enables);
1717         }
1718 }
1719
1720 static void mtp_irq_postinstall(struct drm_i915_private *i915)
1721 {
1722         struct intel_uncore *uncore = &i915->uncore;
1723         u32 sde_mask = SDE_GMBUS_ICP | SDE_PICAINTERRUPT;
1724         u32 de_hpd_mask = XELPDP_AUX_TC_MASK;
1725         u32 de_hpd_enables = de_hpd_mask | XELPDP_DP_ALT_HOTPLUG_MASK |
1726                              XELPDP_TBT_HOTPLUG_MASK;
1727
1728         GEN3_IRQ_INIT(uncore, PICAINTERRUPT_, ~de_hpd_mask,
1729                       de_hpd_enables);
1730
1731         GEN3_IRQ_INIT(uncore, SDE, ~sde_mask, 0xffffffff);
1732 }
1733
1734 static void icp_irq_postinstall(struct drm_i915_private *dev_priv)
1735 {
1736         struct intel_uncore *uncore = &dev_priv->uncore;
1737         u32 mask = SDE_GMBUS_ICP;
1738
1739         GEN3_IRQ_INIT(uncore, SDE, ~mask, 0xffffffff);
1740 }
1741
1742 void gen11_de_irq_postinstall(struct drm_i915_private *dev_priv)
1743 {
1744         if (!HAS_DISPLAY(dev_priv))
1745                 return;
1746
1747         gen8_de_irq_postinstall(dev_priv);
1748
1749         intel_uncore_write(&dev_priv->uncore, GEN11_DISPLAY_INT_CTL,
1750                            GEN11_DISPLAY_IRQ_ENABLE);
1751 }
1752
1753 void dg1_de_irq_postinstall(struct drm_i915_private *i915)
1754 {
1755         if (!HAS_DISPLAY(i915))
1756                 return;
1757
1758         gen8_de_irq_postinstall(i915);
1759         intel_uncore_write(&i915->uncore, GEN11_DISPLAY_INT_CTL,
1760                            GEN11_DISPLAY_IRQ_ENABLE);
1761 }
1762
1763 void intel_display_irq_init(struct drm_i915_private *i915)
1764 {
1765         i915->drm.vblank_disable_immediate = true;
1766
1767         /*
1768          * Most platforms treat the display irq block as an always-on power
1769          * domain. vlv/chv can disable it at runtime and need special care to
1770          * avoid writing any of the display block registers outside of the power
1771          * domain. We defer setting up the display irqs in this case to the
1772          * runtime pm.
1773          */
1774         i915->display.irq.display_irqs_enabled = true;
1775         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
1776                 i915->display.irq.display_irqs_enabled = false;
1777
1778         intel_hotplug_irq_init(i915);
1779 }