drivers/gpu/drm/i915/intel_runtime_pm.c

   1 /*
   2  * Copyright © 2012-2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <[email protected]>
  25  *    Daniel Vetter <[email protected]>
  26  *
  27  */
  28
  29 #include <linux/pm_runtime.h>
  30 #include <linux/vgaarb.h>
  31
  32 #include "i915_drv.h"
  33 #include "intel_drv.h"
  34
  35 /**
  36  * DOC: runtime pm
  37  *
  38  * The i915 driver supports dynamic enabling and disabling of entire hardware
  39  * blocks at runtime. This is especially important on the display side where
  40  * software is supposed to control many power gates manually on recent hardware,
  41  * since on the GT side a lot of the power management is done by the hardware.
  42  * But even there some manual control at the device level is required.
  43  *
  44  * Since i915 supports a diverse set of platforms with a unified codebase and
  45  * hardware engineers just love to shuffle functionality around between power
  46  * domains there's a sizeable amount of indirection required. This file provides
  47  * generic functions to the driver for grabbing and releasing references for
  48  * abstract power domains. It then maps those to the actual power wells
  49  * present for a given platform.
  50  */
  51
  52 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
  53                                          enum i915_power_well_id power_well_id);
  54
  55 static struct i915_power_well *
  56 lookup_power_well(struct drm_i915_private *dev_priv,
  57                   enum i915_power_well_id power_well_id);
  58
  59 const char *
  60 intel_display_power_domain_str(enum intel_display_power_domain domain)
  61 {
  62         switch (domain) {
  63         case POWER_DOMAIN_PIPE_A:
  64                 return "PIPE_A";
  65         case POWER_DOMAIN_PIPE_B:
  66                 return "PIPE_B";
  67         case POWER_DOMAIN_PIPE_C:
  68                 return "PIPE_C";
  69         case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
  70                 return "PIPE_A_PANEL_FITTER";
  71         case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
  72                 return "PIPE_B_PANEL_FITTER";
  73         case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
  74                 return "PIPE_C_PANEL_FITTER";
  75         case POWER_DOMAIN_TRANSCODER_A:
  76                 return "TRANSCODER_A";
  77         case POWER_DOMAIN_TRANSCODER_B:
  78                 return "TRANSCODER_B";
  79         case POWER_DOMAIN_TRANSCODER_C:
  80                 return "TRANSCODER_C";
  81         case POWER_DOMAIN_TRANSCODER_EDP:
  82                 return "TRANSCODER_EDP";
  83         case POWER_DOMAIN_TRANSCODER_DSI_A:
  84                 return "TRANSCODER_DSI_A";
  85         case POWER_DOMAIN_TRANSCODER_DSI_C:
  86                 return "TRANSCODER_DSI_C";
  87         case POWER_DOMAIN_PORT_DDI_A_LANES:
  88                 return "PORT_DDI_A_LANES";
  89         case POWER_DOMAIN_PORT_DDI_B_LANES:
  90                 return "PORT_DDI_B_LANES";
  91         case POWER_DOMAIN_PORT_DDI_C_LANES:
  92                 return "PORT_DDI_C_LANES";
  93         case POWER_DOMAIN_PORT_DDI_D_LANES:
  94                 return "PORT_DDI_D_LANES";
  95         case POWER_DOMAIN_PORT_DDI_E_LANES:
  96                 return "PORT_DDI_E_LANES";
  97         case POWER_DOMAIN_PORT_DDI_A_IO:
  98                 return "PORT_DDI_A_IO";
  99         case POWER_DOMAIN_PORT_DDI_B_IO:
 100                 return "PORT_DDI_B_IO";
 101         case POWER_DOMAIN_PORT_DDI_C_IO:
 102                 return "PORT_DDI_C_IO";
 103         case POWER_DOMAIN_PORT_DDI_D_IO:
 104                 return "PORT_DDI_D_IO";
 105         case POWER_DOMAIN_PORT_DDI_E_IO:
 106                 return "PORT_DDI_E_IO";
 107         case POWER_DOMAIN_PORT_DSI:
 108                 return "PORT_DSI";
 109         case POWER_DOMAIN_PORT_CRT:
 110                 return "PORT_CRT";
 111         case POWER_DOMAIN_PORT_OTHER:
 112                 return "PORT_OTHER";
 113         case POWER_DOMAIN_VGA:
 114                 return "VGA";
 115         case POWER_DOMAIN_AUDIO:
 116                 return "AUDIO";
 117         case POWER_DOMAIN_PLLS:
 118                 return "PLLS";
 119         case POWER_DOMAIN_AUX_A:
 120                 return "AUX_A";
 121         case POWER_DOMAIN_AUX_B:
 122                 return "AUX_B";
 123         case POWER_DOMAIN_AUX_C:
 124                 return "AUX_C";
 125         case POWER_DOMAIN_AUX_D:
 126                 return "AUX_D";
 127         case POWER_DOMAIN_GMBUS:
 128                 return "GMBUS";
 129         case POWER_DOMAIN_INIT:
 130                 return "INIT";
 131         case POWER_DOMAIN_MODESET:
 132                 return "MODESET";
 133         case POWER_DOMAIN_GT_IRQ:
 134                 return "GT_IRQ";
 135         default:
 136                 MISSING_CASE(domain);
 137                 return "?";
 138         }
 139 }
 140
 141 static void intel_power_well_enable(struct drm_i915_private *dev_priv,
 142                                     struct i915_power_well *power_well)
 143 {
 144         DRM_DEBUG_KMS("enabling %s\n", power_well->name);
 145         power_well->ops->enable(dev_priv, power_well);
 146         power_well->hw_enabled = true;
 147 }
 148
 149 static void intel_power_well_disable(struct drm_i915_private *dev_priv,
 150                                      struct i915_power_well *power_well)
 151 {
 152         DRM_DEBUG_KMS("disabling %s\n", power_well->name);
 153         power_well->hw_enabled = false;
 154         power_well->ops->disable(dev_priv, power_well);
 155 }
 156
 157 static void intel_power_well_get(struct drm_i915_private *dev_priv,
 158                                  struct i915_power_well *power_well)
 159 {
 160         if (!power_well->count++)
 161                 intel_power_well_enable(dev_priv, power_well);
 162 }
 163
 164 static void intel_power_well_put(struct drm_i915_private *dev_priv,
 165                                  struct i915_power_well *power_well)
 166 {
 167         WARN(!power_well->count, "Use count on power well %s is already zero",
 168              power_well->name);
 169
 170         if (!--power_well->count)
 171                 intel_power_well_disable(dev_priv, power_well);
 172 }
 173
 174 /**
 175  * __intel_display_power_is_enabled - unlocked check for a power domain
 176  * @dev_priv: i915 device instance
 177  * @domain: power domain to check
 178  *
 179  * This is the unlocked version of intel_display_power_is_enabled() and should
 180  * only be used from error capture and recovery code where deadlocks are
 181  * possible.
 182  *
 183  * Returns:
 184  * True when the power domain is enabled, false otherwise.
 185  */
 186 bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 187                                       enum intel_display_power_domain domain)
 188 {
 189         struct i915_power_well *power_well;
 190         bool is_enabled;
 191
 192         if (dev_priv->runtime_pm.suspended)
 193                 return false;
 194
 195         is_enabled = true;
 196
 197         for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain)) {
 198                 if (power_well->always_on)
 199                         continue;
 200
 201                 if (!power_well->hw_enabled) {
 202                         is_enabled = false;
 203                         break;
 204                 }
 205         }
 206
 207         return is_enabled;
 208 }
 209
 210 /**
 211  * intel_display_power_is_enabled - check for a power domain
 212  * @dev_priv: i915 device instance
 213  * @domain: power domain to check
 214  *
 215  * This function can be used to check the hw power domain state. It is mostly
 216  * used in hardware state readout functions. Everywhere else code should rely
 217  * upon explicit power domain reference counting to ensure that the hardware
 218  * block is powered up before accessing it.
 219  *
 220  * Callers must hold the relevant modesetting locks to ensure that concurrent
 221  * threads can't disable the power well while the caller tries to read a few
 222  * registers.
 223  *
 224  * Returns:
 225  * True when the power domain is enabled, false otherwise.
 226  */
 227 bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 228                                     enum intel_display_power_domain domain)
 229 {
 230         struct i915_power_domains *power_domains;
 231         bool ret;
 232
 233         power_domains = &dev_priv->power_domains;
 234
 235         mutex_lock(&power_domains->lock);
 236         ret = __intel_display_power_is_enabled(dev_priv, domain);
 237         mutex_unlock(&power_domains->lock);
 238
 239         return ret;
 240 }
 241
 242 /**
 243  * intel_display_set_init_power - set the initial power domain state
 244  * @dev_priv: i915 device instance
 245  * @enable: whether to enable or disable the initial power domain state
 246  *
 247  * For simplicity our driver load/unload and system suspend/resume code assumes
 248  * that all power domains are always enabled. This functions controls the state
 249  * of this little hack. While the initial power domain state is enabled runtime
 250  * pm is effectively disabled.
 251  */
 252 void intel_display_set_init_power(struct drm_i915_private *dev_priv,
 253                                   bool enable)
 254 {
 255         if (dev_priv->power_domains.init_power_on == enable)
 256                 return;
 257
 258         if (enable)
 259                 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
 260         else
 261                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
 262
 263         dev_priv->power_domains.init_power_on = enable;
 264 }
 265
 266 /*
 267  * Starting with Haswell, we have a "Power Down Well" that can be turned off
 268  * when not needed anymore. We have 4 registers that can request the power well
 269  * to be enabled, and it will only be disabled if none of the registers is
 270  * requesting it to be enabled.
 271  */
 272 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv,
 273                                        u8 irq_pipe_mask, bool has_vga)
 274 {
 275         struct pci_dev *pdev = dev_priv->drm.pdev;
 276
 277         /*
 278          * After we re-enable the power well, if we touch VGA register 0x3d5
 279          * we'll get unclaimed register interrupts. This stops after we write
 280          * anything to the VGA MSR register. The vgacon module uses this
 281          * register all the time, so if we unbind our driver and, as a
 282          * consequence, bind vgacon, we'll get stuck in an infinite loop at
 283          * console_unlock(). So make here we touch the VGA MSR register, making
 284          * sure vgacon can keep working normally without triggering interrupts
 285          * and error messages.
 286          */
 287         if (has_vga) {
 288                 vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
 289                 outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
 290                 vga_put(pdev, VGA_RSRC_LEGACY_IO);
 291         }
 292
 293         if (irq_pipe_mask)
 294                 gen8_irq_power_well_post_enable(dev_priv, irq_pipe_mask);
 295 }
 296
 297 static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv,
 298                                        u8 irq_pipe_mask)
 299 {
 300         if (irq_pipe_mask)
 301                 gen8_irq_power_well_pre_disable(dev_priv, irq_pipe_mask);
 302 }
 303
 304
 305 static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv,
 306                                            struct i915_power_well *power_well)
 307 {
 308         enum i915_power_well_id id = power_well->id;
 309
 310         /* Timeout for PW1:10 us, AUX:not specified, other PWs:20 us. */
 311         WARN_ON(intel_wait_for_register(dev_priv,
 312                                         HSW_PWR_WELL_CTL_DRIVER(id),
 313                                         HSW_PWR_WELL_CTL_STATE(id),
 314                                         HSW_PWR_WELL_CTL_STATE(id),
 315                                         1));
 316 }
 317
 318 static u32 hsw_power_well_requesters(struct drm_i915_private *dev_priv,
 319                                      enum i915_power_well_id id)
 320 {
 321         u32 req_mask = HSW_PWR_WELL_CTL_REQ(id);
 322         u32 ret;
 323
 324         ret = I915_READ(HSW_PWR_WELL_CTL_BIOS(id)) & req_mask ? 1 : 0;
 325         ret |= I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) & req_mask ? 2 : 0;
 326         ret |= I915_READ(HSW_PWR_WELL_CTL_KVMR) & req_mask ? 4 : 0;
 327         ret |= I915_READ(HSW_PWR_WELL_CTL_DEBUG(id)) & req_mask ? 8 : 0;
 328
 329         return ret;
 330 }
 331
 332 static void hsw_wait_for_power_well_disable(struct drm_i915_private *dev_priv,
 333                                             struct i915_power_well *power_well)
 334 {
 335         enum i915_power_well_id id = power_well->id;
 336         bool disabled;
 337         u32 reqs;
 338
 339         /*
 340          * Bspec doesn't require waiting for PWs to get disabled, but still do
 341          * this for paranoia. The known cases where a PW will be forced on:
 342          * - a KVMR request on any power well via the KVMR request register
 343          * - a DMC request on PW1 and MISC_IO power wells via the BIOS and
 344          *   DEBUG request registers
 345          * Skip the wait in case any of the request bits are set and print a
 346          * diagnostic message.
 347          */
 348         wait_for((disabled = !(I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) &
 349                                HSW_PWR_WELL_CTL_STATE(id))) ||
 350                  (reqs = hsw_power_well_requesters(dev_priv, id)), 1);
 351         if (disabled)
 352                 return;
 353
 354         DRM_DEBUG_KMS("%s forced on (bios:%d driver:%d kvmr:%d debug:%d)\n",
 355                       power_well->name,
 356                       !!(reqs & 1), !!(reqs & 2), !!(reqs & 4), !!(reqs & 8));
 357 }
 358
 359 static void gen9_wait_for_power_well_fuses(struct drm_i915_private *dev_priv,
 360                                            enum skl_power_gate pg)
 361 {
 362         /* Timeout 5us for PG#0, for other PGs 1us */
 363         WARN_ON(intel_wait_for_register(dev_priv, SKL_FUSE_STATUS,
 364                                         SKL_FUSE_PG_DIST_STATUS(pg),
 365                                         SKL_FUSE_PG_DIST_STATUS(pg), 1));
 366 }
 367
 368 static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
 369                                   struct i915_power_well *power_well)
 370 {
 371         enum i915_power_well_id id = power_well->id;
 372         bool wait_fuses = power_well->hsw.has_fuses;
 373         enum skl_power_gate uninitialized_var(pg);
 374         u32 val;
 375
 376         if (wait_fuses) {
 377                 pg = SKL_PW_TO_PG(id);
 378                 /*
 379                  * For PW1 we have to wait both for the PW0/PG0 fuse state
 380                  * before enabling the power well and PW1/PG1's own fuse
 381                  * state after the enabling. For all other power wells with
 382                  * fuses we only have to wait for that PW/PG's fuse state
 383                  * after the enabling.
 384                  */
 385                 if (pg == SKL_PG1)
 386                         gen9_wait_for_power_well_fuses(dev_priv, SKL_PG0);
 387         }
 388
 389         val = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
 390         I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id), val | HSW_PWR_WELL_CTL_REQ(id));
 391         hsw_wait_for_power_well_enable(dev_priv, power_well);
 392
 393         if (wait_fuses)
 394                 gen9_wait_for_power_well_fuses(dev_priv, pg);
 395
 396         hsw_power_well_post_enable(dev_priv, power_well->hsw.irq_pipe_mask,
 397                                    power_well->hsw.has_vga);
 398 }
 399
 400 static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 401                                    struct i915_power_well *power_well)
 402 {
 403         enum i915_power_well_id id = power_well->id;
 404         u32 val;
 405
 406         hsw_power_well_pre_disable(dev_priv, power_well->hsw.irq_pipe_mask);
 407
 408         val = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
 409         I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id),
 410                    val & ~HSW_PWR_WELL_CTL_REQ(id));
 411         hsw_wait_for_power_well_disable(dev_priv, power_well);
 412 }
 413
 414 /*
 415  * We should only use the power well if we explicitly asked the hardware to
 416  * enable it, so check if it's enabled and also check if we've requested it to
 417  * be enabled.
 418  */
 419 static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
 420                                    struct i915_power_well *power_well)
 421 {
 422         enum i915_power_well_id id = power_well->id;
 423         u32 mask = HSW_PWR_WELL_CTL_REQ(id) | HSW_PWR_WELL_CTL_STATE(id);
 424
 425         return (I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) & mask) == mask;
 426 }
 427
 428 static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
 429 {
 430         enum i915_power_well_id id = SKL_DISP_PW_2;
 431
 432         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_DC9),
 433                   "DC9 already programmed to be enabled.\n");
 434         WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 435                   "DC5 still not disabled to enable DC9.\n");
 436         WARN_ONCE(I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) &
 437                   HSW_PWR_WELL_CTL_REQ(id),
 438                   "Power well 2 on.\n");
 439         WARN_ONCE(intel_irqs_enabled(dev_priv),
 440                   "Interrupts not disabled yet.\n");
 441
 442          /*
 443           * TODO: check for the following to verify the conditions to enter DC9
 444           * state are satisfied:
 445           * 1] Check relevant display engine registers to verify if mode set
 446           * disable sequence was followed.
 447           * 2] Check if display uninitialize sequence is initialized.
 448           */
 449 }
 450
 451 static void assert_can_disable_dc9(struct drm_i915_private *dev_priv)
 452 {
 453         WARN_ONCE(intel_irqs_enabled(dev_priv),
 454                   "Interrupts not disabled yet.\n");
 455         WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 456                   "DC5 still not disabled.\n");
 457
 458          /*
 459           * TODO: check for the following to verify DC9 state was indeed
 460           * entered before programming to disable it:
 461           * 1] Check relevant display engine registers to verify if mode
 462           *  set disable sequence was followed.
 463           * 2] Check if display uninitialize sequence is initialized.
 464           */
 465 }
 466
 467 static void gen9_write_dc_state(struct drm_i915_private *dev_priv,
 468                                 u32 state)
 469 {
 470         int rewrites = 0;
 471         int rereads = 0;
 472         u32 v;
 473
 474         I915_WRITE(DC_STATE_EN, state);
 475
 476         /* It has been observed that disabling the dc6 state sometimes
 477          * doesn't stick and dmc keeps returning old value. Make sure
 478          * the write really sticks enough times and also force rewrite until
 479          * we are confident that state is exactly what we want.
 480          */
 481         do  {
 482                 v = I915_READ(DC_STATE_EN);
 483
 484                 if (v != state) {
 485                         I915_WRITE(DC_STATE_EN, state);
 486                         rewrites++;
 487                         rereads = 0;
 488                 } else if (rereads++ > 5) {
 489                         break;
 490                 }
 491
 492         } while (rewrites < 100);
 493
 494         if (v != state)
 495                 DRM_ERROR("Writing dc state to 0x%x failed, now 0x%x\n",
 496                           state, v);
 497
 498         /* Most of the times we need one retry, avoid spam */
 499         if (rewrites > 1)
 500                 DRM_DEBUG_KMS("Rewrote dc state to 0x%x %d times\n",
 501                               state, rewrites);
 502 }
 503
 504 static u32 gen9_dc_mask(struct drm_i915_private *dev_priv)
 505 {
 506         u32 mask;
 507
 508         mask = DC_STATE_EN_UPTO_DC5;
 509         if (IS_GEN9_LP(dev_priv))
 510                 mask |= DC_STATE_EN_DC9;
 511         else
 512                 mask |= DC_STATE_EN_UPTO_DC6;
 513
 514         return mask;
 515 }
 516
 517 void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv)
 518 {
 519         u32 val;
 520
 521         val = I915_READ(DC_STATE_EN) & gen9_dc_mask(dev_priv);
 522
 523         DRM_DEBUG_KMS("Resetting DC state tracking from %02x to %02x\n",
 524                       dev_priv->csr.dc_state, val);
 525         dev_priv->csr.dc_state = val;
 526 }
 527
 528 static void gen9_set_dc_state(struct drm_i915_private *dev_priv, uint32_t state)
 529 {
 530         uint32_t val;
 531         uint32_t mask;
 532
 533         if (WARN_ON_ONCE(state & ~dev_priv->csr.allowed_dc_mask))
 534                 state &= dev_priv->csr.allowed_dc_mask;
 535
 536         val = I915_READ(DC_STATE_EN);
 537         mask = gen9_dc_mask(dev_priv);
 538         DRM_DEBUG_KMS("Setting DC state from %02x to %02x\n",
 539                       val & mask, state);
 540
 541         /* Check if DMC is ignoring our DC state requests */
 542         if ((val & mask) != dev_priv->csr.dc_state)
 543                 DRM_ERROR("DC state mismatch (0x%x -> 0x%x)\n",
 544                           dev_priv->csr.dc_state, val & mask);
 545
 546         val &= ~mask;
 547         val |= state;
 548
 549         gen9_write_dc_state(dev_priv, val);
 550
 551         dev_priv->csr.dc_state = val & mask;
 552 }
 553
 554 void bxt_enable_dc9(struct drm_i915_private *dev_priv)
 555 {
 556         assert_can_enable_dc9(dev_priv);
 557
 558         DRM_DEBUG_KMS("Enabling DC9\n");
 559
 560         intel_power_sequencer_reset(dev_priv);
 561         gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9);
 562 }
 563
 564 void bxt_disable_dc9(struct drm_i915_private *dev_priv)
 565 {
 566         assert_can_disable_dc9(dev_priv);
 567
 568         DRM_DEBUG_KMS("Disabling DC9\n");
 569
 570         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 571
 572         intel_pps_unlock_regs_wa(dev_priv);
 573 }
 574
 575 static void assert_csr_loaded(struct drm_i915_private *dev_priv)
 576 {
 577         WARN_ONCE(!I915_READ(CSR_PROGRAM(0)),
 578                   "CSR program storage start is NULL\n");
 579         WARN_ONCE(!I915_READ(CSR_SSP_BASE), "CSR SSP Base Not fine\n");
 580         WARN_ONCE(!I915_READ(CSR_HTP_SKL), "CSR HTP Not fine\n");
 581 }
 582
 583 static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
 584 {
 585         bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv,
 586                                         SKL_DISP_PW_2);
 587
 588         WARN_ONCE(pg2_enabled, "PG2 not disabled to enable DC5.\n");
 589
 590         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5),
 591                   "DC5 already programmed to be enabled.\n");
 592         assert_rpm_wakelock_held(dev_priv);
 593
 594         assert_csr_loaded(dev_priv);
 595 }
 596
 597 void gen9_enable_dc5(struct drm_i915_private *dev_priv)
 598 {
 599         assert_can_enable_dc5(dev_priv);
 600
 601         DRM_DEBUG_KMS("Enabling DC5\n");
 602
 603         /* Wa Display #1183: skl,kbl,cfl */
 604         if (IS_GEN9_BC(dev_priv))
 605                 I915_WRITE(GEN8_CHICKEN_DCPR_1, I915_READ(GEN8_CHICKEN_DCPR_1) |
 606                            SKL_SELECT_ALTERNATE_DC_EXIT);
 607
 608         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
 609 }
 610
 611 static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
 612 {
 613         WARN_ONCE(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
 614                   "Backlight is not disabled.\n");
 615         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
 616                   "DC6 already programmed to be enabled.\n");
 617
 618         assert_csr_loaded(dev_priv);
 619 }
 620
 621 void skl_enable_dc6(struct drm_i915_private *dev_priv)
 622 {
 623         assert_can_enable_dc6(dev_priv);
 624
 625         DRM_DEBUG_KMS("Enabling DC6\n");
 626
 627         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 628
 629 }
 630
 631 void skl_disable_dc6(struct drm_i915_private *dev_priv)
 632 {
 633         DRM_DEBUG_KMS("Disabling DC6\n");
 634
 635         /* Wa Display #1183: skl,kbl,cfl */
 636         if (IS_GEN9_BC(dev_priv))
 637                 I915_WRITE(GEN8_CHICKEN_DCPR_1, I915_READ(GEN8_CHICKEN_DCPR_1) |
 638                            SKL_SELECT_ALTERNATE_DC_EXIT);
 639
 640         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 641 }
 642
 643 static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
 644                                    struct i915_power_well *power_well)
 645 {
 646         enum i915_power_well_id id = power_well->id;
 647         u32 mask = HSW_PWR_WELL_CTL_REQ(id);
 648         u32 bios_req = I915_READ(HSW_PWR_WELL_CTL_BIOS(id));
 649
 650         /* Take over the request bit if set by BIOS. */
 651         if (bios_req & mask) {
 652                 u32 drv_req = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
 653
 654                 if (!(drv_req & mask))
 655                         I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id), drv_req | mask);
 656                 I915_WRITE(HSW_PWR_WELL_CTL_BIOS(id), bios_req & ~mask);
 657         }
 658 }
 659
 660 static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 661                                            struct i915_power_well *power_well)
 662 {
 663         bxt_ddi_phy_init(dev_priv, power_well->bxt.phy);
 664 }
 665
 666 static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 667                                             struct i915_power_well *power_well)
 668 {
 669         bxt_ddi_phy_uninit(dev_priv, power_well->bxt.phy);
 670 }
 671
 672 static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
 673                                             struct i915_power_well *power_well)
 674 {
 675         return bxt_ddi_phy_is_enabled(dev_priv, power_well->bxt.phy);
 676 }
 677
 678 static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
 679 {
 680         struct i915_power_well *power_well;
 681
 682         power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A);
 683         if (power_well->count > 0)
 684                 bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 685
 686         power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_BC);
 687         if (power_well->count > 0)
 688                 bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 689
 690         if (IS_GEMINILAKE(dev_priv)) {
 691                 power_well = lookup_power_well(dev_priv, GLK_DPIO_CMN_C);
 692                 if (power_well->count > 0)
 693                         bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 694         }
 695 }
 696
 697 static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
 698                                            struct i915_power_well *power_well)
 699 {
 700         return (I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0;
 701 }
 702
 703 static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
 704 {
 705         u32 tmp = I915_READ(DBUF_CTL);
 706
 707         WARN((tmp & (DBUF_POWER_STATE | DBUF_POWER_REQUEST)) !=
 708              (DBUF_POWER_STATE | DBUF_POWER_REQUEST),
 709              "Unexpected DBuf power power state (0x%08x)\n", tmp);
 710 }
 711
 712 static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
 713                                           struct i915_power_well *power_well)
 714 {
 715         struct intel_cdclk_state cdclk_state = {};
 716
 717         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 718
 719         dev_priv->display.get_cdclk(dev_priv, &cdclk_state);
 720         /* Can't read out voltage_level so can't use intel_cdclk_changed() */
 721         WARN_ON(intel_cdclk_needs_modeset(&dev_priv->cdclk.hw, &cdclk_state));
 722
 723         gen9_assert_dbuf_enabled(dev_priv);
 724
 725         if (IS_GEN9_LP(dev_priv))
 726                 bxt_verify_ddi_phy_power_wells(dev_priv);
 727 }
 728
 729 static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
 730                                            struct i915_power_well *power_well)
 731 {
 732         if (!dev_priv->csr.dmc_payload)
 733                 return;
 734
 735         if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC6)
 736                 skl_enable_dc6(dev_priv);
 737         else if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)
 738                 gen9_enable_dc5(dev_priv);
 739 }
 740
 741 static void i9xx_power_well_sync_hw_noop(struct drm_i915_private *dev_priv,
 742                                          struct i915_power_well *power_well)
 743 {
 744 }
 745
 746 static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
 747                                            struct i915_power_well *power_well)
 748 {
 749 }
 750
 751 static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
 752                                              struct i915_power_well *power_well)
 753 {
 754         return true;
 755 }
 756
 757 static void i830_pipes_power_well_enable(struct drm_i915_private *dev_priv,
 758                                          struct i915_power_well *power_well)
 759 {
 760         if ((I915_READ(PIPECONF(PIPE_A)) & PIPECONF_ENABLE) == 0)
 761                 i830_enable_pipe(dev_priv, PIPE_A);
 762         if ((I915_READ(PIPECONF(PIPE_B)) & PIPECONF_ENABLE) == 0)
 763                 i830_enable_pipe(dev_priv, PIPE_B);
 764 }
 765
 766 static void i830_pipes_power_well_disable(struct drm_i915_private *dev_priv,
 767                                           struct i915_power_well *power_well)
 768 {
 769         i830_disable_pipe(dev_priv, PIPE_B);
 770         i830_disable_pipe(dev_priv, PIPE_A);
 771 }
 772
 773 static bool i830_pipes_power_well_enabled(struct drm_i915_private *dev_priv,
 774                                           struct i915_power_well *power_well)
 775 {
 776         return I915_READ(PIPECONF(PIPE_A)) & PIPECONF_ENABLE &&
 777                 I915_READ(PIPECONF(PIPE_B)) & PIPECONF_ENABLE;
 778 }
 779
 780 static void i830_pipes_power_well_sync_hw(struct drm_i915_private *dev_priv,
 781                                           struct i915_power_well *power_well)
 782 {
 783         if (power_well->count > 0)
 784                 i830_pipes_power_well_enable(dev_priv, power_well);
 785         else
 786                 i830_pipes_power_well_disable(dev_priv, power_well);
 787 }
 788
 789 static void vlv_set_power_well(struct drm_i915_private *dev_priv,
 790                                struct i915_power_well *power_well, bool enable)
 791 {
 792         enum i915_power_well_id power_well_id = power_well->id;
 793         u32 mask;
 794         u32 state;
 795         u32 ctrl;
 796
 797         mask = PUNIT_PWRGT_MASK(power_well_id);
 798         state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) :
 799                          PUNIT_PWRGT_PWR_GATE(power_well_id);
 800
 801         mutex_lock(&dev_priv->pcu_lock);
 802
 803 #define COND \
 804         ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
 805
 806         if (COND)
 807                 goto out;
 808
 809         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
 810         ctrl &= ~mask;
 811         ctrl |= state;
 812         vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
 813
 814         if (wait_for(COND, 100))
 815                 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
 816                           state,
 817                           vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
 818
 819 #undef COND
 820
 821 out:
 822         mutex_unlock(&dev_priv->pcu_lock);
 823 }
 824
 825 static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
 826                                   struct i915_power_well *power_well)
 827 {
 828         vlv_set_power_well(dev_priv, power_well, true);
 829 }
 830
 831 static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
 832                                    struct i915_power_well *power_well)
 833 {
 834         vlv_set_power_well(dev_priv, power_well, false);
 835 }
 836
 837 static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
 838                                    struct i915_power_well *power_well)
 839 {
 840         enum i915_power_well_id power_well_id = power_well->id;
 841         bool enabled = false;
 842         u32 mask;
 843         u32 state;
 844         u32 ctrl;
 845
 846         mask = PUNIT_PWRGT_MASK(power_well_id);
 847         ctrl = PUNIT_PWRGT_PWR_ON(power_well_id);
 848
 849         mutex_lock(&dev_priv->pcu_lock);
 850
 851         state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
 852         /*
 853          * We only ever set the power-on and power-gate states, anything
 854          * else is unexpected.
 855          */
 856         WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) &&
 857                 state != PUNIT_PWRGT_PWR_GATE(power_well_id));
 858         if (state == ctrl)
 859                 enabled = true;
 860
 861         /*
 862          * A transient state at this point would mean some unexpected party
 863          * is poking at the power controls too.
 864          */
 865         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
 866         WARN_ON(ctrl != state);
 867
 868         mutex_unlock(&dev_priv->pcu_lock);
 869
 870         return enabled;
 871 }
 872
 873 static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
 874 {
 875         u32 val;
 876
 877         /*
 878          * On driver load, a pipe may be active and driving a DSI display.
 879          * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
 880          * (and never recovering) in this case. intel_dsi_post_disable() will
 881          * clear it when we turn off the display.
 882          */
 883         val = I915_READ(DSPCLK_GATE_D);
 884         val &= DPOUNIT_CLOCK_GATE_DISABLE;
 885         val |= VRHUNIT_CLOCK_GATE_DISABLE;
 886         I915_WRITE(DSPCLK_GATE_D, val);
 887
 888         /*
 889          * Disable trickle feed and enable pnd deadline calculation
 890          */
 891         I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
 892         I915_WRITE(CBR1_VLV, 0);
 893
 894         WARN_ON(dev_priv->rawclk_freq == 0);
 895
 896         I915_WRITE(RAWCLK_FREQ_VLV,
 897                    DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 1000));
 898 }
 899
 900 static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
 901 {
 902         struct intel_encoder *encoder;
 903         enum pipe pipe;
 904
 905         /*
 906          * Enable the CRI clock source so we can get at the
 907          * display and the reference clock for VGA
 908          * hotplug / manual detection. Supposedly DSI also
 909          * needs the ref clock up and running.
 910          *
 911          * CHV DPLL B/C have some issues if VGA mode is enabled.
 912          */
 913         for_each_pipe(dev_priv, pipe) {
 914                 u32 val = I915_READ(DPLL(pipe));
 915
 916                 val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
 917                 if (pipe != PIPE_A)
 918                         val |= DPLL_INTEGRATED_CRI_CLK_VLV;
 919
 920                 I915_WRITE(DPLL(pipe), val);
 921         }
 922
 923         vlv_init_display_clock_gating(dev_priv);
 924
 925         spin_lock_irq(&dev_priv->irq_lock);
 926         valleyview_enable_display_irqs(dev_priv);
 927         spin_unlock_irq(&dev_priv->irq_lock);
 928
 929         /*
 930          * During driver initialization/resume we can avoid restoring the
 931          * part of the HW/SW state that will be inited anyway explicitly.
 932          */
 933         if (dev_priv->power_domains.initializing)
 934                 return;
 935
 936         intel_hpd_init(dev_priv);
 937
 938         /* Re-enable the ADPA, if we have one */
 939         for_each_intel_encoder(&dev_priv->drm, encoder) {
 940                 if (encoder->type == INTEL_OUTPUT_ANALOG)
 941                         intel_crt_reset(&encoder->base);
 942         }
 943
 944         i915_redisable_vga_power_on(dev_priv);
 945
 946         intel_pps_unlock_regs_wa(dev_priv);
 947 }
 948
 949 static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
 950 {
 951         spin_lock_irq(&dev_priv->irq_lock);
 952         valleyview_disable_display_irqs(dev_priv);
 953         spin_unlock_irq(&dev_priv->irq_lock);
 954
 955         /* make sure we're done processing display irqs */
 956         synchronize_irq(dev_priv->drm.irq);
 957
 958         intel_power_sequencer_reset(dev_priv);
 959
 960         /* Prevent us from re-enabling polling on accident in late suspend */
 961         if (!dev_priv->drm.dev->power.is_suspended)
 962                 intel_hpd_poll_init(dev_priv);
 963 }
 964
 965 static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
 966                                           struct i915_power_well *power_well)
 967 {
 968         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
 969
 970         vlv_set_power_well(dev_priv, power_well, true);
 971
 972         vlv_display_power_well_init(dev_priv);
 973 }
 974
 975 static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
 976                                            struct i915_power_well *power_well)
 977 {
 978         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
 979
 980         vlv_display_power_well_deinit(dev_priv);
 981
 982         vlv_set_power_well(dev_priv, power_well, false);
 983 }
 984
 985 static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 986                                            struct i915_power_well *power_well)
 987 {
 988         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
 989
 990         /* since ref/cri clock was enabled */
 991         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
 992
 993         vlv_set_power_well(dev_priv, power_well, true);
 994
 995         /*
 996          * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
 997          *  6.  De-assert cmn_reset/side_reset. Same as VLV X0.
 998          *   a. GUnit 0x2110 bit[0] set to 1 (def 0)
 999          *   b. The other bits such as sfr settings / modesel may all
1000          *      be set to 0.
1001          *
1002          * This should only be done on init and resume from S3 with
1003          * both PLLs disabled, or we risk losing DPIO and PLL
1004          * synchronization.
1005          */
1006         I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) | DPIO_CMNRST);
1007 }
1008
1009 static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1010                                             struct i915_power_well *power_well)
1011 {
1012         enum pipe pipe;
1013
1014         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
1015
1016         for_each_pipe(dev_priv, pipe)
1017                 assert_pll_disabled(dev_priv, pipe);
1018
1019         /* Assert common reset */
1020         I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) & ~DPIO_CMNRST);
1021
1022         vlv_set_power_well(dev_priv, power_well, false);
1023 }
1024
1025 #define POWER_DOMAIN_MASK (GENMASK_ULL(POWER_DOMAIN_NUM - 1, 0))
1026
1027 static struct i915_power_well *
1028 lookup_power_well(struct drm_i915_private *dev_priv,
1029                   enum i915_power_well_id power_well_id)
1030 {
1031         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1032         int i;
1033
1034         for (i = 0; i < power_domains->power_well_count; i++) {
1035                 struct i915_power_well *power_well;
1036
1037                 power_well = &power_domains->power_wells[i];
1038                 if (power_well->id == power_well_id)
1039                         return power_well;
1040         }
1041
1042         return NULL;
1043 }
1044
1045 #define BITS_SET(val, bits) (((val) & (bits)) == (bits))
1046
1047 static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
1048 {
1049         struct i915_power_well *cmn_bc =
1050                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
1051         struct i915_power_well *cmn_d =
1052                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
1053         u32 phy_control = dev_priv->chv_phy_control;
1054         u32 phy_status = 0;
1055         u32 phy_status_mask = 0xffffffff;
1056
1057         /*
1058          * The BIOS can leave the PHY is some weird state
1059          * where it doesn't fully power down some parts.
1060          * Disable the asserts until the PHY has been fully
1061          * reset (ie. the power well has been disabled at
1062          * least once).
1063          */
1064         if (!dev_priv->chv_phy_assert[DPIO_PHY0])
1065                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
1066                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
1067                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
1068                                      PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) |
1069                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
1070                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));
1071
1072         if (!dev_priv->chv_phy_assert[DPIO_PHY1])
1073                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
1074                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
1075                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));
1076
1077         if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
1078                 phy_status |= PHY_POWERGOOD(DPIO_PHY0);
1079
1080                 /* this assumes override is only used to enable lanes */
1081                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0)
1082                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0);
1083
1084                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0)
1085                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1);
1086
1087                 /* CL1 is on whenever anything is on in either channel */
1088                 if (BITS_SET(phy_control,
1089                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) |
1090                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)))
1091                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0);
1092
1093                 /*
1094                  * The DPLLB check accounts for the pipe B + port A usage
1095                  * with CL2 powered up but all the lanes in the second channel
1096                  * powered down.
1097                  */
1098                 if (BITS_SET(phy_control,
1099                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) &&
1100                     (I915_READ(DPLL(PIPE_B)) & DPLL_VCO_ENABLE) == 0)
1101                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1);
1102
1103                 if (BITS_SET(phy_control,
1104                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0)))
1105                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0);
1106                 if (BITS_SET(phy_control,
1107                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0)))
1108                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1);
1109
1110                 if (BITS_SET(phy_control,
1111                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1)))
1112                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0);
1113                 if (BITS_SET(phy_control,
1114                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1)))
1115                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1);
1116         }
1117
1118         if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
1119                 phy_status |= PHY_POWERGOOD(DPIO_PHY1);
1120
1121                 /* this assumes override is only used to enable lanes */
1122                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0)
1123                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0);
1124
1125                 if (BITS_SET(phy_control,
1126                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0)))
1127                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0);
1128
1129                 if (BITS_SET(phy_control,
1130                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0)))
1131                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0);
1132                 if (BITS_SET(phy_control,
1133                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0)))
1134                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1);
1135         }
1136
1137         phy_status &= phy_status_mask;
1138
1139         /*
1140          * The PHY may be busy with some initial calibration and whatnot,
1141          * so the power state can take a while to actually change.
1142          */
1143         if (intel_wait_for_register(dev_priv,
1144                                     DISPLAY_PHY_STATUS,
1145                                     phy_status_mask,
1146                                     phy_status,
1147                                     10))
1148                 DRM_ERROR("Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
1149                           I915_READ(DISPLAY_PHY_STATUS) & phy_status_mask,
1150                            phy_status, dev_priv->chv_phy_control);
1151 }
1152
1153 #undef BITS_SET
1154
1155 static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1156                                            struct i915_power_well *power_well)
1157 {
1158         enum dpio_phy phy;
1159         enum pipe pipe;
1160         uint32_t tmp;
1161
1162         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
1163                      power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
1164
1165         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1166                 pipe = PIPE_A;
1167                 phy = DPIO_PHY0;
1168         } else {
1169                 pipe = PIPE_C;
1170                 phy = DPIO_PHY1;
1171         }
1172
1173         /* since ref/cri clock was enabled */
1174         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1175         vlv_set_power_well(dev_priv, power_well, true);
1176
1177         /* Poll for phypwrgood signal */
1178         if (intel_wait_for_register(dev_priv,
1179                                     DISPLAY_PHY_STATUS,
1180                                     PHY_POWERGOOD(phy),
1181                                     PHY_POWERGOOD(phy),
1182                                     1))
1183                 DRM_ERROR("Display PHY %d is not power up\n", phy);
1184
1185         mutex_lock(&dev_priv->sb_lock);
1186
1187         /* Enable dynamic power down */
1188         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW28);
1189         tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
1190                 DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
1191         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp);
1192
1193         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1194                 tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1);
1195                 tmp |= DPIO_DYNPWRDOWNEN_CH1;
1196                 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp);
1197         } else {
1198                 /*
1199                  * Force the non-existing CL2 off. BXT does this
1200                  * too, so maybe it saves some power even though
1201                  * CL2 doesn't exist?
1202                  */
1203                 tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30);
1204                 tmp |= DPIO_CL2_LDOFUSE_PWRENB;
1205                 vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, tmp);
1206         }
1207
1208         mutex_unlock(&dev_priv->sb_lock);
1209
1210         dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
1211         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1212
1213         DRM_DEBUG_KMS("Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1214                       phy, dev_priv->chv_phy_control);
1215
1216         assert_chv_phy_status(dev_priv);
1217 }
1218
1219 static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1220                                             struct i915_power_well *power_well)
1221 {
1222         enum dpio_phy phy;
1223
1224         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
1225                      power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
1226
1227         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1228                 phy = DPIO_PHY0;
1229                 assert_pll_disabled(dev_priv, PIPE_A);
1230                 assert_pll_disabled(dev_priv, PIPE_B);
1231         } else {
1232                 phy = DPIO_PHY1;
1233                 assert_pll_disabled(dev_priv, PIPE_C);
1234         }
1235
1236         dev_priv->chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
1237         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1238
1239         vlv_set_power_well(dev_priv, power_well, false);
1240
1241         DRM_DEBUG_KMS("Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1242                       phy, dev_priv->chv_phy_control);
1243
1244         /* PHY is fully reset now, so we can enable the PHY state asserts */
1245         dev_priv->chv_phy_assert[phy] = true;
1246
1247         assert_chv_phy_status(dev_priv);
1248 }
1249
1250 static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1251                                      enum dpio_channel ch, bool override, unsigned int mask)
1252 {
1253         enum pipe pipe = phy == DPIO_PHY0 ? PIPE_A : PIPE_C;
1254         u32 reg, val, expected, actual;
1255
1256         /*
1257          * The BIOS can leave the PHY is some weird state
1258          * where it doesn't fully power down some parts.
1259          * Disable the asserts until the PHY has been fully
1260          * reset (ie. the power well has been disabled at
1261          * least once).
1262          */
1263         if (!dev_priv->chv_phy_assert[phy])
1264                 return;
1265
1266         if (ch == DPIO_CH0)
1267                 reg = _CHV_CMN_DW0_CH0;
1268         else
1269                 reg = _CHV_CMN_DW6_CH1;
1270
1271         mutex_lock(&dev_priv->sb_lock);
1272         val = vlv_dpio_read(dev_priv, pipe, reg);
1273         mutex_unlock(&dev_priv->sb_lock);
1274
1275         /*
1276          * This assumes !override is only used when the port is disabled.
1277          * All lanes should power down even without the override when
1278          * the port is disabled.
1279          */
1280         if (!override || mask == 0xf) {
1281                 expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1282                 /*
1283                  * If CH1 common lane is not active anymore
1284                  * (eg. for pipe B DPLL) the entire channel will
1285                  * shut down, which causes the common lane registers
1286                  * to read as 0. That means we can't actually check
1287                  * the lane power down status bits, but as the entire
1288                  * register reads as 0 it's a good indication that the
1289                  * channel is indeed entirely powered down.
1290                  */
1291                 if (ch == DPIO_CH1 && val == 0)
1292                         expected = 0;
1293         } else if (mask != 0x0) {
1294                 expected = DPIO_ANYDL_POWERDOWN;
1295         } else {
1296                 expected = 0;
1297         }
1298
1299         if (ch == DPIO_CH0)
1300                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH0;
1301         else
1302                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1;
1303         actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1304
1305         WARN(actual != expected,
1306              "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
1307              !!(actual & DPIO_ALLDL_POWERDOWN), !!(actual & DPIO_ANYDL_POWERDOWN),
1308              !!(expected & DPIO_ALLDL_POWERDOWN), !!(expected & DPIO_ANYDL_POWERDOWN),
1309              reg, val);
1310 }
1311
1312 bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1313                           enum dpio_channel ch, bool override)
1314 {
1315         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1316         bool was_override;
1317
1318         mutex_lock(&power_domains->lock);
1319
1320         was_override = dev_priv->chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1321
1322         if (override == was_override)
1323                 goto out;
1324
1325         if (override)
1326                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1327         else
1328                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1329
1330         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1331
1332         DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
1333                       phy, ch, dev_priv->chv_phy_control);
1334
1335         assert_chv_phy_status(dev_priv);
1336
1337 out:
1338         mutex_unlock(&power_domains->lock);
1339
1340         return was_override;
1341 }
1342
1343 void chv_phy_powergate_lanes(struct intel_encoder *encoder,
1344                              bool override, unsigned int mask)
1345 {
1346         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1347         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1348         enum dpio_phy phy = vlv_dport_to_phy(enc_to_dig_port(&encoder->base));
1349         enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));
1350
1351         mutex_lock(&power_domains->lock);
1352
1353         dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
1354         dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
1355
1356         if (override)
1357                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1358         else
1359                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1360
1361         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1362
1363         DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
1364                       phy, ch, mask, dev_priv->chv_phy_control);
1365
1366         assert_chv_phy_status(dev_priv);
1367
1368         assert_chv_phy_powergate(dev_priv, phy, ch, override, mask);
1369
1370         mutex_unlock(&power_domains->lock);
1371 }
1372
1373 static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
1374                                         struct i915_power_well *power_well)
1375 {
1376         enum pipe pipe = PIPE_A;
1377         bool enabled;
1378         u32 state, ctrl;
1379
1380         mutex_lock(&dev_priv->pcu_lock);
1381
1382         state = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe);
1383         /*
1384          * We only ever set the power-on and power-gate states, anything
1385          * else is unexpected.
1386          */
1387         WARN_ON(state != DP_SSS_PWR_ON(pipe) && state != DP_SSS_PWR_GATE(pipe));
1388         enabled = state == DP_SSS_PWR_ON(pipe);
1389
1390         /*
1391          * A transient state at this point would mean some unexpected party
1392          * is poking at the power controls too.
1393          */
1394         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSC_MASK(pipe);
1395         WARN_ON(ctrl << 16 != state);
1396
1397         mutex_unlock(&dev_priv->pcu_lock);
1398
1399         return enabled;
1400 }
1401
1402 static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
1403                                     struct i915_power_well *power_well,
1404                                     bool enable)
1405 {
1406         enum pipe pipe = PIPE_A;
1407         u32 state;
1408         u32 ctrl;
1409
1410         state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
1411
1412         mutex_lock(&dev_priv->pcu_lock);
1413
1414 #define COND \
1415         ((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe)) == state)
1416
1417         if (COND)
1418                 goto out;
1419
1420         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
1421         ctrl &= ~DP_SSC_MASK(pipe);
1422         ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
1423         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, ctrl);
1424
1425         if (wait_for(COND, 100))
1426                 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
1427                           state,
1428                           vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ));
1429
1430 #undef COND
1431
1432 out:
1433         mutex_unlock(&dev_priv->pcu_lock);
1434 }
1435
1436 static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
1437                                        struct i915_power_well *power_well)
1438 {
1439         WARN_ON_ONCE(power_well->id != CHV_DISP_PW_PIPE_A);
1440
1441         chv_set_pipe_power_well(dev_priv, power_well, true);
1442
1443         vlv_display_power_well_init(dev_priv);
1444 }
1445
1446 static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
1447                                         struct i915_power_well *power_well)
1448 {
1449         WARN_ON_ONCE(power_well->id != CHV_DISP_PW_PIPE_A);
1450
1451         vlv_display_power_well_deinit(dev_priv);
1452
1453         chv_set_pipe_power_well(dev_priv, power_well, false);
1454 }
1455
1456 static void
1457 __intel_display_power_get_domain(struct drm_i915_private *dev_priv,
1458                                  enum intel_display_power_domain domain)
1459 {
1460         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1461         struct i915_power_well *power_well;
1462
1463         for_each_power_domain_well(dev_priv, power_well, BIT_ULL(domain))
1464                 intel_power_well_get(dev_priv, power_well);
1465
1466         power_domains->domain_use_count[domain]++;
1467 }
1468
1469 /**
1470  * intel_display_power_get - grab a power domain reference
1471  * @dev_priv: i915 device instance
1472  * @domain: power domain to reference
1473  *
1474  * This function grabs a power domain reference for @domain and ensures that the
1475  * power domain and all its parents are powered up. Therefore users should only
1476  * grab a reference to the innermost power domain they need.
1477  *
1478  * Any power domain reference obtained by this function must have a symmetric
1479  * call to intel_display_power_put() to release the reference again.
1480  */
1481 void intel_display_power_get(struct drm_i915_private *dev_priv,
1482                              enum intel_display_power_domain domain)
1483 {
1484         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1485
1486         intel_runtime_pm_get(dev_priv);
1487
1488         mutex_lock(&power_domains->lock);
1489
1490         __intel_display_power_get_domain(dev_priv, domain);
1491
1492         mutex_unlock(&power_domains->lock);
1493 }
1494
1495 /**
1496  * intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
1497  * @dev_priv: i915 device instance
1498  * @domain: power domain to reference
1499  *
1500  * This function grabs a power domain reference for @domain and ensures that the
1501  * power domain and all its parents are powered up. Therefore users should only
1502  * grab a reference to the innermost power domain they need.
1503  *
1504  * Any power domain reference obtained by this function must have a symmetric
1505  * call to intel_display_power_put() to release the reference again.
1506  */
1507 bool intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
1508                                         enum intel_display_power_domain domain)
1509 {
1510         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1511         bool is_enabled;
1512
1513         if (!intel_runtime_pm_get_if_in_use(dev_priv))
1514                 return false;
1515
1516         mutex_lock(&power_domains->lock);
1517
1518         if (__intel_display_power_is_enabled(dev_priv, domain)) {
1519                 __intel_display_power_get_domain(dev_priv, domain);
1520                 is_enabled = true;
1521         } else {
1522                 is_enabled = false;
1523         }
1524
1525         mutex_unlock(&power_domains->lock);
1526
1527         if (!is_enabled)
1528                 intel_runtime_pm_put(dev_priv);
1529
1530         return is_enabled;
1531 }
1532
1533 /**
1534  * intel_display_power_put - release a power domain reference
1535  * @dev_priv: i915 device instance
1536  * @domain: power domain to reference
1537  *
1538  * This function drops the power domain reference obtained by
1539  * intel_display_power_get() and might power down the corresponding hardware
1540  * block right away if this is the last reference.
1541  */
1542 void intel_display_power_put(struct drm_i915_private *dev_priv,
1543                              enum intel_display_power_domain domain)
1544 {
1545         struct i915_power_domains *power_domains;
1546         struct i915_power_well *power_well;
1547
1548         power_domains = &dev_priv->power_domains;
1549
1550         mutex_lock(&power_domains->lock);
1551
1552         WARN(!power_domains->domain_use_count[domain],
1553              "Use count on domain %s is already zero\n",
1554              intel_display_power_domain_str(domain));
1555         power_domains->domain_use_count[domain]--;
1556
1557         for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain))
1558                 intel_power_well_put(dev_priv, power_well);
1559
1560         mutex_unlock(&power_domains->lock);
1561
1562         intel_runtime_pm_put(dev_priv);
1563 }
1564
1565 #define I830_PIPES_POWER_DOMAINS (              \
1566         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1567         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1568         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1569         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1570         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1571         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1572         BIT_ULL(POWER_DOMAIN_INIT))
1573
1574 #define VLV_DISPLAY_POWER_DOMAINS (             \
1575         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1576         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1577         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1578         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1579         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1580         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1581         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1582         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1583         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
1584         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
1585         BIT_ULL(POWER_DOMAIN_VGA) |                     \
1586         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
1587         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1588         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1589         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
1590         BIT_ULL(POWER_DOMAIN_INIT))
1591
1592 #define VLV_DPIO_CMN_BC_POWER_DOMAINS (         \
1593         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1594         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1595         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
1596         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1597         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1598         BIT_ULL(POWER_DOMAIN_INIT))
1599
1600 #define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS (  \
1601         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1602         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1603         BIT_ULL(POWER_DOMAIN_INIT))
1604
1605 #define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS (  \
1606         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1607         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1608         BIT_ULL(POWER_DOMAIN_INIT))
1609
1610 #define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS (  \
1611         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1612         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1613         BIT_ULL(POWER_DOMAIN_INIT))
1614
1615 #define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS (  \
1616         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1617         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1618         BIT_ULL(POWER_DOMAIN_INIT))
1619
1620 #define CHV_DISPLAY_POWER_DOMAINS (             \
1621         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1622         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1623         BIT_ULL(POWER_DOMAIN_PIPE_C) |          \
1624         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1625         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1626         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
1627         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1628         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1629         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |    \
1630         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1631         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1632         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
1633         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
1634         BIT_ULL(POWER_DOMAIN_VGA) |                     \
1635         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
1636         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1637         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1638         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
1639         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
1640         BIT_ULL(POWER_DOMAIN_INIT))
1641
1642 #define CHV_DPIO_CMN_BC_POWER_DOMAINS (         \
1643         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1644         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1645         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1646         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1647         BIT_ULL(POWER_DOMAIN_INIT))
1648
1649 #define CHV_DPIO_CMN_D_POWER_DOMAINS (          \
1650         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
1651         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
1652         BIT_ULL(POWER_DOMAIN_INIT))
1653
1654 #define HSW_DISPLAY_POWER_DOMAINS (                     \
1655         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1656         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1657         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |             \
1658         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1659         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1660         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1661         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1662         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1663         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1664         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1665         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1666         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
1667         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1668         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1669         BIT_ULL(POWER_DOMAIN_INIT))
1670
1671 #define BDW_DISPLAY_POWER_DOMAINS (                     \
1672         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1673         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1674         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1675         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1676         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1677         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1678         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1679         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1680         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1681         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1682         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
1683         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1684         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1685         BIT_ULL(POWER_DOMAIN_INIT))
1686
1687 #define SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1688         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1689         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1690         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1691         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1692         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1693         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1694         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1695         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1696         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1697         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1698         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |                \
1699         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1700         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1701         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1702         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1703         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1704         BIT_ULL(POWER_DOMAIN_INIT))
1705 #define SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS (          \
1706         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
1707         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) |           \
1708         BIT_ULL(POWER_DOMAIN_INIT))
1709 #define SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
1710         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
1711         BIT_ULL(POWER_DOMAIN_INIT))
1712 #define SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
1713         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
1714         BIT_ULL(POWER_DOMAIN_INIT))
1715 #define SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS (            \
1716         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
1717         BIT_ULL(POWER_DOMAIN_INIT))
1718 #define SKL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1719         SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1720         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
1721         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1722         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1723         BIT_ULL(POWER_DOMAIN_INIT))
1724
1725 #define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1726         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1727         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1728         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1729         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1730         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1731         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1732         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1733         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1734         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1735         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1736         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1737         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1738         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1739         BIT_ULL(POWER_DOMAIN_INIT))
1740 #define BXT_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1741         BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1742         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
1743         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1744         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1745         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
1746         BIT_ULL(POWER_DOMAIN_INIT))
1747 #define BXT_DPIO_CMN_A_POWER_DOMAINS (                  \
1748         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
1749         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1750         BIT_ULL(POWER_DOMAIN_INIT))
1751 #define BXT_DPIO_CMN_BC_POWER_DOMAINS (                 \
1752         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1753         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1754         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1755         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1756         BIT_ULL(POWER_DOMAIN_INIT))
1757
1758 #define GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1759         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1760         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1761         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1762         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1763         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1764         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1765         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1766         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1767         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1768         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1769         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1770         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1771         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1772         BIT_ULL(POWER_DOMAIN_INIT))
1773 #define GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS (            \
1774         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
1775 #define GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
1776         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
1777 #define GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
1778         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
1779 #define GLK_DPIO_CMN_A_POWER_DOMAINS (                  \
1780         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
1781         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1782         BIT_ULL(POWER_DOMAIN_INIT))
1783 #define GLK_DPIO_CMN_B_POWER_DOMAINS (                  \
1784         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1785         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1786         BIT_ULL(POWER_DOMAIN_INIT))
1787 #define GLK_DPIO_CMN_C_POWER_DOMAINS (                  \
1788         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1789         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1790         BIT_ULL(POWER_DOMAIN_INIT))
1791 #define GLK_DISPLAY_AUX_A_POWER_DOMAINS (               \
1792         BIT_ULL(POWER_DOMAIN_AUX_A) |           \
1793         BIT_ULL(POWER_DOMAIN_INIT))
1794 #define GLK_DISPLAY_AUX_B_POWER_DOMAINS (               \
1795         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1796         BIT_ULL(POWER_DOMAIN_INIT))
1797 #define GLK_DISPLAY_AUX_C_POWER_DOMAINS (               \
1798         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1799         BIT_ULL(POWER_DOMAIN_INIT))
1800 #define GLK_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1801         GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1802         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
1803         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1804         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1805         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
1806         BIT_ULL(POWER_DOMAIN_INIT))
1807
1808 #define CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1809         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1810         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1811         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1812         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1813         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1814         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1815         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1816         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1817         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1818         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1819         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1820         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1821         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1822         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1823         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1824         BIT_ULL(POWER_DOMAIN_INIT))
1825 #define CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS (            \
1826         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
1827         BIT_ULL(POWER_DOMAIN_INIT))
1828 #define CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS (            \
1829         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
1830         BIT_ULL(POWER_DOMAIN_INIT))
1831 #define CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS (            \
1832         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
1833         BIT_ULL(POWER_DOMAIN_INIT))
1834 #define CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS (            \
1835         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
1836         BIT_ULL(POWER_DOMAIN_INIT))
1837 #define CNL_DISPLAY_AUX_A_POWER_DOMAINS (               \
1838         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1839         BIT_ULL(POWER_DOMAIN_INIT))
1840 #define CNL_DISPLAY_AUX_B_POWER_DOMAINS (               \
1841         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1842         BIT_ULL(POWER_DOMAIN_INIT))
1843 #define CNL_DISPLAY_AUX_C_POWER_DOMAINS (               \
1844         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1845         BIT_ULL(POWER_DOMAIN_INIT))
1846 #define CNL_DISPLAY_AUX_D_POWER_DOMAINS (               \
1847         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1848         BIT_ULL(POWER_DOMAIN_INIT))
1849 #define CNL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1850         CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1851         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1852         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1853         BIT_ULL(POWER_DOMAIN_INIT))
1854
1855 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
1856         .sync_hw = i9xx_power_well_sync_hw_noop,
1857         .enable = i9xx_always_on_power_well_noop,
1858         .disable = i9xx_always_on_power_well_noop,
1859         .is_enabled = i9xx_always_on_power_well_enabled,
1860 };
1861
1862 static const struct i915_power_well_ops chv_pipe_power_well_ops = {
1863         .sync_hw = i9xx_power_well_sync_hw_noop,
1864         .enable = chv_pipe_power_well_enable,
1865         .disable = chv_pipe_power_well_disable,
1866         .is_enabled = chv_pipe_power_well_enabled,
1867 };
1868
1869 static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
1870         .sync_hw = i9xx_power_well_sync_hw_noop,
1871         .enable = chv_dpio_cmn_power_well_enable,
1872         .disable = chv_dpio_cmn_power_well_disable,
1873         .is_enabled = vlv_power_well_enabled,
1874 };
1875
1876 static struct i915_power_well i9xx_always_on_power_well[] = {
1877         {
1878                 .name = "always-on",
1879                 .always_on = 1,
1880                 .domains = POWER_DOMAIN_MASK,
1881                 .ops = &i9xx_always_on_power_well_ops,
1882                 .id = I915_DISP_PW_ALWAYS_ON,
1883         },
1884 };
1885
1886 static const struct i915_power_well_ops i830_pipes_power_well_ops = {
1887         .sync_hw = i830_pipes_power_well_sync_hw,
1888         .enable = i830_pipes_power_well_enable,
1889         .disable = i830_pipes_power_well_disable,
1890         .is_enabled = i830_pipes_power_well_enabled,
1891 };
1892
1893 static struct i915_power_well i830_power_wells[] = {
1894         {
1895                 .name = "always-on",
1896                 .always_on = 1,
1897                 .domains = POWER_DOMAIN_MASK,
1898                 .ops = &i9xx_always_on_power_well_ops,
1899                 .id = I915_DISP_PW_ALWAYS_ON,
1900         },
1901         {
1902                 .name = "pipes",
1903                 .domains = I830_PIPES_POWER_DOMAINS,
1904                 .ops = &i830_pipes_power_well_ops,
1905                 .id = I830_DISP_PW_PIPES,
1906         },
1907 };
1908
1909 static const struct i915_power_well_ops hsw_power_well_ops = {
1910         .sync_hw = hsw_power_well_sync_hw,
1911         .enable = hsw_power_well_enable,
1912         .disable = hsw_power_well_disable,
1913         .is_enabled = hsw_power_well_enabled,
1914 };
1915
1916 static const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
1917         .sync_hw = i9xx_power_well_sync_hw_noop,
1918         .enable = gen9_dc_off_power_well_enable,
1919         .disable = gen9_dc_off_power_well_disable,
1920         .is_enabled = gen9_dc_off_power_well_enabled,
1921 };
1922
1923 static const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = {
1924         .sync_hw = i9xx_power_well_sync_hw_noop,
1925         .enable = bxt_dpio_cmn_power_well_enable,
1926         .disable = bxt_dpio_cmn_power_well_disable,
1927         .is_enabled = bxt_dpio_cmn_power_well_enabled,
1928 };
1929
1930 static struct i915_power_well hsw_power_wells[] = {
1931         {
1932                 .name = "always-on",
1933                 .always_on = 1,
1934                 .domains = POWER_DOMAIN_MASK,
1935                 .ops = &i9xx_always_on_power_well_ops,
1936                 .id = I915_DISP_PW_ALWAYS_ON,
1937         },
1938         {
1939                 .name = "display",
1940                 .domains = HSW_DISPLAY_POWER_DOMAINS,
1941                 .ops = &hsw_power_well_ops,
1942                 .id = HSW_DISP_PW_GLOBAL,
1943                 {
1944                         .hsw.has_vga = true,
1945                 },
1946         },
1947 };
1948
1949 static struct i915_power_well bdw_power_wells[] = {
1950         {
1951                 .name = "always-on",
1952                 .always_on = 1,
1953                 .domains = POWER_DOMAIN_MASK,
1954                 .ops = &i9xx_always_on_power_well_ops,
1955                 .id = I915_DISP_PW_ALWAYS_ON,
1956         },
1957         {
1958                 .name = "display",
1959                 .domains = BDW_DISPLAY_POWER_DOMAINS,
1960                 .ops = &hsw_power_well_ops,
1961                 .id = HSW_DISP_PW_GLOBAL,
1962                 {
1963                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
1964                         .hsw.has_vga = true,
1965                 },
1966         },
1967 };
1968
1969 static const struct i915_power_well_ops vlv_display_power_well_ops = {
1970         .sync_hw = i9xx_power_well_sync_hw_noop,
1971         .enable = vlv_display_power_well_enable,
1972         .disable = vlv_display_power_well_disable,
1973         .is_enabled = vlv_power_well_enabled,
1974 };
1975
1976 static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
1977         .sync_hw = i9xx_power_well_sync_hw_noop,
1978         .enable = vlv_dpio_cmn_power_well_enable,
1979         .disable = vlv_dpio_cmn_power_well_disable,
1980         .is_enabled = vlv_power_well_enabled,
1981 };
1982
1983 static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
1984         .sync_hw = i9xx_power_well_sync_hw_noop,
1985         .enable = vlv_power_well_enable,
1986         .disable = vlv_power_well_disable,
1987         .is_enabled = vlv_power_well_enabled,
1988 };
1989
1990 static struct i915_power_well vlv_power_wells[] = {
1991         {
1992                 .name = "always-on",
1993                 .always_on = 1,
1994                 .domains = POWER_DOMAIN_MASK,
1995                 .ops = &i9xx_always_on_power_well_ops,
1996                 .id = I915_DISP_PW_ALWAYS_ON,
1997         },
1998         {
1999                 .name = "display",
2000                 .domains = VLV_DISPLAY_POWER_DOMAINS,
2001                 .id = PUNIT_POWER_WELL_DISP2D,
2002                 .ops = &vlv_display_power_well_ops,
2003         },
2004         {
2005                 .name = "dpio-tx-b-01",
2006                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2007                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2008                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2009                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2010                 .ops = &vlv_dpio_power_well_ops,
2011                 .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
2012         },
2013         {
2014                 .name = "dpio-tx-b-23",
2015                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2016                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2017                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2018                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2019                 .ops = &vlv_dpio_power_well_ops,
2020                 .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
2021         },
2022         {
2023                 .name = "dpio-tx-c-01",
2024                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2025                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2026                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2027                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2028                 .ops = &vlv_dpio_power_well_ops,
2029                 .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
2030         },
2031         {
2032                 .name = "dpio-tx-c-23",
2033                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2034                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2035                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2036                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2037                 .ops = &vlv_dpio_power_well_ops,
2038                 .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
2039         },
2040         {
2041                 .name = "dpio-common",
2042                 .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
2043                 .id = PUNIT_POWER_WELL_DPIO_CMN_BC,
2044                 .ops = &vlv_dpio_cmn_power_well_ops,
2045         },
2046 };
2047
2048 static struct i915_power_well chv_power_wells[] = {
2049         {
2050                 .name = "always-on",
2051                 .always_on = 1,
2052                 .domains = POWER_DOMAIN_MASK,
2053                 .ops = &i9xx_always_on_power_well_ops,
2054                 .id = I915_DISP_PW_ALWAYS_ON,
2055         },
2056         {
2057                 .name = "display",
2058                 /*
2059                  * Pipe A power well is the new disp2d well. Pipe B and C
2060                  * power wells don't actually exist. Pipe A power well is
2061                  * required for any pipe to work.
2062                  */
2063                 .domains = CHV_DISPLAY_POWER_DOMAINS,
2064                 .id = CHV_DISP_PW_PIPE_A,
2065                 .ops = &chv_pipe_power_well_ops,
2066         },
2067         {
2068                 .name = "dpio-common-bc",
2069                 .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS,
2070                 .id = PUNIT_POWER_WELL_DPIO_CMN_BC,
2071                 .ops = &chv_dpio_cmn_power_well_ops,
2072         },
2073         {
2074                 .name = "dpio-common-d",
2075                 .domains = CHV_DPIO_CMN_D_POWER_DOMAINS,
2076                 .id = PUNIT_POWER_WELL_DPIO_CMN_D,
2077                 .ops = &chv_dpio_cmn_power_well_ops,
2078         },
2079 };
2080
2081 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
2082                                          enum i915_power_well_id power_well_id)
2083 {
2084         struct i915_power_well *power_well;
2085         bool ret;
2086
2087         power_well = lookup_power_well(dev_priv, power_well_id);
2088         ret = power_well->ops->is_enabled(dev_priv, power_well);
2089
2090         return ret;
2091 }
2092
2093 static struct i915_power_well skl_power_wells[] = {
2094         {
2095                 .name = "always-on",
2096                 .always_on = 1,
2097                 .domains = POWER_DOMAIN_MASK,
2098                 .ops = &i9xx_always_on_power_well_ops,
2099                 .id = I915_DISP_PW_ALWAYS_ON,
2100         },
2101         {
2102                 .name = "power well 1",
2103                 /* Handled by the DMC firmware */
2104                 .domains = 0,
2105                 .ops = &hsw_power_well_ops,
2106                 .id = SKL_DISP_PW_1,
2107                 {
2108                         .hsw.has_fuses = true,
2109                 },
2110         },
2111         {
2112                 .name = "MISC IO power well",
2113                 /* Handled by the DMC firmware */
2114                 .domains = 0,
2115                 .ops = &hsw_power_well_ops,
2116                 .id = SKL_DISP_PW_MISC_IO,
2117         },
2118         {
2119                 .name = "DC off",
2120                 .domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS,
2121                 .ops = &gen9_dc_off_power_well_ops,
2122                 .id = SKL_DISP_PW_DC_OFF,
2123         },
2124         {
2125                 .name = "power well 2",
2126                 .domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2127                 .ops = &hsw_power_well_ops,
2128                 .id = SKL_DISP_PW_2,
2129                 {
2130                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2131                         .hsw.has_vga = true,
2132                         .hsw.has_fuses = true,
2133                 },
2134         },
2135         {
2136                 .name = "DDI A/E IO power well",
2137                 .domains = SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS,
2138                 .ops = &hsw_power_well_ops,
2139                 .id = SKL_DISP_PW_DDI_A_E,
2140         },
2141         {
2142                 .name = "DDI B IO power well",
2143                 .domains = SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS,
2144                 .ops = &hsw_power_well_ops,
2145                 .id = SKL_DISP_PW_DDI_B,
2146         },
2147         {
2148                 .name = "DDI C IO power well",
2149                 .domains = SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS,
2150                 .ops = &hsw_power_well_ops,
2151                 .id = SKL_DISP_PW_DDI_C,
2152         },
2153         {
2154                 .name = "DDI D IO power well",
2155                 .domains = SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS,
2156                 .ops = &hsw_power_well_ops,
2157                 .id = SKL_DISP_PW_DDI_D,
2158         },
2159 };
2160
2161 static struct i915_power_well bxt_power_wells[] = {
2162         {
2163                 .name = "always-on",
2164                 .always_on = 1,
2165                 .domains = POWER_DOMAIN_MASK,
2166                 .ops = &i9xx_always_on_power_well_ops,
2167                 .id = I915_DISP_PW_ALWAYS_ON,
2168         },
2169         {
2170                 .name = "power well 1",
2171                 .domains = 0,
2172                 .ops = &hsw_power_well_ops,
2173                 .id = SKL_DISP_PW_1,
2174                 {
2175                         .hsw.has_fuses = true,
2176                 },
2177         },
2178         {
2179                 .name = "DC off",
2180                 .domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS,
2181                 .ops = &gen9_dc_off_power_well_ops,
2182                 .id = SKL_DISP_PW_DC_OFF,
2183         },
2184         {
2185                 .name = "power well 2",
2186                 .domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2187                 .ops = &hsw_power_well_ops,
2188                 .id = SKL_DISP_PW_2,
2189                 {
2190                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2191                         .hsw.has_vga = true,
2192                         .hsw.has_fuses = true,
2193                 },
2194         },
2195         {
2196                 .name = "dpio-common-a",
2197                 .domains = BXT_DPIO_CMN_A_POWER_DOMAINS,
2198                 .ops = &bxt_dpio_cmn_power_well_ops,
2199                 .id = BXT_DPIO_CMN_A,
2200                 {
2201                         .bxt.phy = DPIO_PHY1,
2202                 },
2203         },
2204         {
2205                 .name = "dpio-common-bc",
2206                 .domains = BXT_DPIO_CMN_BC_POWER_DOMAINS,
2207                 .ops = &bxt_dpio_cmn_power_well_ops,
2208                 .id = BXT_DPIO_CMN_BC,
2209                 {
2210                         .bxt.phy = DPIO_PHY0,
2211                 },
2212         },
2213 };
2214
2215 static struct i915_power_well glk_power_wells[] = {
2216         {
2217                 .name = "always-on",
2218                 .always_on = 1,
2219                 .domains = POWER_DOMAIN_MASK,
2220                 .ops = &i9xx_always_on_power_well_ops,
2221                 .id = I915_DISP_PW_ALWAYS_ON,
2222         },
2223         {
2224                 .name = "power well 1",
2225                 /* Handled by the DMC firmware */
2226                 .domains = 0,
2227                 .ops = &hsw_power_well_ops,
2228                 .id = SKL_DISP_PW_1,
2229                 {
2230                         .hsw.has_fuses = true,
2231                 },
2232         },
2233         {
2234                 .name = "DC off",
2235                 .domains = GLK_DISPLAY_DC_OFF_POWER_DOMAINS,
2236                 .ops = &gen9_dc_off_power_well_ops,
2237                 .id = SKL_DISP_PW_DC_OFF,
2238         },
2239         {
2240                 .name = "power well 2",
2241                 .domains = GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2242                 .ops = &hsw_power_well_ops,
2243                 .id = SKL_DISP_PW_2,
2244                 {
2245                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2246                         .hsw.has_vga = true,
2247                         .hsw.has_fuses = true,
2248                 },
2249         },
2250         {
2251                 .name = "dpio-common-a",
2252                 .domains = GLK_DPIO_CMN_A_POWER_DOMAINS,
2253                 .ops = &bxt_dpio_cmn_power_well_ops,
2254                 .id = BXT_DPIO_CMN_A,
2255                 {
2256                         .bxt.phy = DPIO_PHY1,
2257                 },
2258         },
2259         {
2260                 .name = "dpio-common-b",
2261                 .domains = GLK_DPIO_CMN_B_POWER_DOMAINS,
2262                 .ops = &bxt_dpio_cmn_power_well_ops,
2263                 .id = BXT_DPIO_CMN_BC,
2264                 {
2265                         .bxt.phy = DPIO_PHY0,
2266                 },
2267         },
2268         {
2269                 .name = "dpio-common-c",
2270                 .domains = GLK_DPIO_CMN_C_POWER_DOMAINS,
2271                 .ops = &bxt_dpio_cmn_power_well_ops,
2272                 .id = GLK_DPIO_CMN_C,
2273                 {
2274                         .bxt.phy = DPIO_PHY2,
2275                 },
2276         },
2277         {
2278                 .name = "AUX A",
2279                 .domains = GLK_DISPLAY_AUX_A_POWER_DOMAINS,
2280                 .ops = &hsw_power_well_ops,
2281                 .id = GLK_DISP_PW_AUX_A,
2282         },
2283         {
2284                 .name = "AUX B",
2285                 .domains = GLK_DISPLAY_AUX_B_POWER_DOMAINS,
2286                 .ops = &hsw_power_well_ops,
2287                 .id = GLK_DISP_PW_AUX_B,
2288         },
2289         {
2290                 .name = "AUX C",
2291                 .domains = GLK_DISPLAY_AUX_C_POWER_DOMAINS,
2292                 .ops = &hsw_power_well_ops,
2293                 .id = GLK_DISP_PW_AUX_C,
2294         },
2295         {
2296                 .name = "DDI A IO power well",
2297                 .domains = GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS,
2298                 .ops = &hsw_power_well_ops,
2299                 .id = GLK_DISP_PW_DDI_A,
2300         },
2301         {
2302                 .name = "DDI B IO power well",
2303                 .domains = GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS,
2304                 .ops = &hsw_power_well_ops,
2305                 .id = SKL_DISP_PW_DDI_B,
2306         },
2307         {
2308                 .name = "DDI C IO power well",
2309                 .domains = GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS,
2310                 .ops = &hsw_power_well_ops,
2311                 .id = SKL_DISP_PW_DDI_C,
2312         },
2313 };
2314
2315 static struct i915_power_well cnl_power_wells[] = {
2316         {
2317                 .name = "always-on",
2318                 .always_on = 1,
2319                 .domains = POWER_DOMAIN_MASK,
2320                 .ops = &i9xx_always_on_power_well_ops,
2321                 .id = I915_DISP_PW_ALWAYS_ON,
2322         },
2323         {
2324                 .name = "power well 1",
2325                 /* Handled by the DMC firmware */
2326                 .domains = 0,
2327                 .ops = &hsw_power_well_ops,
2328                 .id = SKL_DISP_PW_1,
2329                 {
2330                         .hsw.has_fuses = true,
2331                 },
2332         },
2333         {
2334                 .name = "AUX A",
2335                 .domains = CNL_DISPLAY_AUX_A_POWER_DOMAINS,
2336                 .ops = &hsw_power_well_ops,
2337                 .id = CNL_DISP_PW_AUX_A,
2338         },
2339         {
2340                 .name = "AUX B",
2341                 .domains = CNL_DISPLAY_AUX_B_POWER_DOMAINS,
2342                 .ops = &hsw_power_well_ops,
2343                 .id = CNL_DISP_PW_AUX_B,
2344         },
2345         {
2346                 .name = "AUX C",
2347                 .domains = CNL_DISPLAY_AUX_C_POWER_DOMAINS,
2348                 .ops = &hsw_power_well_ops,
2349                 .id = CNL_DISP_PW_AUX_C,
2350         },
2351         {
2352                 .name = "AUX D",
2353                 .domains = CNL_DISPLAY_AUX_D_POWER_DOMAINS,
2354                 .ops = &hsw_power_well_ops,
2355                 .id = CNL_DISP_PW_AUX_D,
2356         },
2357         {
2358                 .name = "DC off",
2359                 .domains = CNL_DISPLAY_DC_OFF_POWER_DOMAINS,
2360                 .ops = &gen9_dc_off_power_well_ops,
2361                 .id = SKL_DISP_PW_DC_OFF,
2362         },
2363         {
2364                 .name = "power well 2",
2365                 .domains = CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2366                 .ops = &hsw_power_well_ops,
2367                 .id = SKL_DISP_PW_2,
2368                 {
2369                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2370                         .hsw.has_vga = true,
2371                         .hsw.has_fuses = true,
2372                 },
2373         },
2374         {
2375                 .name = "DDI A IO power well",
2376                 .domains = CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS,
2377                 .ops = &hsw_power_well_ops,
2378                 .id = CNL_DISP_PW_DDI_A,
2379         },
2380         {
2381                 .name = "DDI B IO power well",
2382                 .domains = CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS,
2383                 .ops = &hsw_power_well_ops,
2384                 .id = SKL_DISP_PW_DDI_B,
2385         },
2386         {
2387                 .name = "DDI C IO power well",
2388                 .domains = CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS,
2389                 .ops = &hsw_power_well_ops,
2390                 .id = SKL_DISP_PW_DDI_C,
2391         },
2392         {
2393                 .name = "DDI D IO power well",
2394                 .domains = CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS,
2395                 .ops = &hsw_power_well_ops,
2396                 .id = SKL_DISP_PW_DDI_D,
2397         },
2398 };
2399
2400 static int
2401 sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
2402                                    int disable_power_well)
2403 {
2404         if (disable_power_well >= 0)
2405                 return !!disable_power_well;
2406
2407         return 1;
2408 }
2409
2410 static uint32_t get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
2411                                     int enable_dc)
2412 {
2413         uint32_t mask;
2414         int requested_dc;
2415         int max_dc;
2416
2417         if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
2418                 max_dc = 2;
2419                 mask = 0;
2420         } else if (IS_GEN9_LP(dev_priv)) {
2421                 max_dc = 1;
2422                 /*
2423                  * DC9 has a separate HW flow from the rest of the DC states,
2424                  * not depending on the DMC firmware. It's needed by system
2425                  * suspend/resume, so allow it unconditionally.
2426                  */
2427                 mask = DC_STATE_EN_DC9;
2428         } else {
2429                 max_dc = 0;
2430                 mask = 0;
2431         }
2432
2433         if (!i915_modparams.disable_power_well)
2434                 max_dc = 0;
2435
2436         if (enable_dc >= 0 && enable_dc <= max_dc) {
2437                 requested_dc = enable_dc;
2438         } else if (enable_dc == -1) {
2439                 requested_dc = max_dc;
2440         } else if (enable_dc > max_dc && enable_dc <= 2) {
2441                 DRM_DEBUG_KMS("Adjusting requested max DC state (%d->%d)\n",
2442                               enable_dc, max_dc);
2443                 requested_dc = max_dc;
2444         } else {
2445                 DRM_ERROR("Unexpected value for enable_dc (%d)\n", enable_dc);
2446                 requested_dc = max_dc;
2447         }
2448
2449         if (requested_dc > 1)
2450                 mask |= DC_STATE_EN_UPTO_DC6;
2451         if (requested_dc > 0)
2452                 mask |= DC_STATE_EN_UPTO_DC5;
2453
2454         DRM_DEBUG_KMS("Allowed DC state mask %02x\n", mask);
2455
2456         return mask;
2457 }
2458
2459 static void assert_power_well_ids_unique(struct drm_i915_private *dev_priv)
2460 {
2461         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2462         u64 power_well_ids;
2463         int i;
2464
2465         power_well_ids = 0;
2466         for (i = 0; i < power_domains->power_well_count; i++) {
2467                 enum i915_power_well_id id = power_domains->power_wells[i].id;
2468
2469                 WARN_ON(id >= sizeof(power_well_ids) * 8);
2470                 WARN_ON(power_well_ids & BIT_ULL(id));
2471                 power_well_ids |= BIT_ULL(id);
2472         }
2473 }
2474
2475 #define set_power_wells(power_domains, __power_wells) ({                \
2476         (power_domains)->power_wells = (__power_wells);                 \
2477         (power_domains)->power_well_count = ARRAY_SIZE(__power_wells);  \
2478 })
2479
2480 /**
2481  * intel_power_domains_init - initializes the power domain structures
2482  * @dev_priv: i915 device instance
2483  *
2484  * Initializes the power domain structures for @dev_priv depending upon the
2485  * supported platform.
2486  */
2487 int intel_power_domains_init(struct drm_i915_private *dev_priv)
2488 {
2489         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2490
2491         i915_modparams.disable_power_well =
2492                 sanitize_disable_power_well_option(dev_priv,
2493                                                    i915_modparams.disable_power_well);
2494         dev_priv->csr.allowed_dc_mask =
2495                 get_allowed_dc_mask(dev_priv, i915_modparams.enable_dc);
2496
2497         BUILD_BUG_ON(POWER_DOMAIN_NUM > 64);
2498
2499         mutex_init(&power_domains->lock);
2500
2501         /*
2502          * The enabling order will be from lower to higher indexed wells,
2503          * the disabling order is reversed.
2504          */
2505         if (IS_HASWELL(dev_priv)) {
2506                 set_power_wells(power_domains, hsw_power_wells);
2507         } else if (IS_BROADWELL(dev_priv)) {
2508                 set_power_wells(power_domains, bdw_power_wells);
2509         } else if (IS_GEN9_BC(dev_priv)) {
2510                 set_power_wells(power_domains, skl_power_wells);
2511         } else if (IS_CANNONLAKE(dev_priv)) {
2512                 set_power_wells(power_domains, cnl_power_wells);
2513         } else if (IS_BROXTON(dev_priv)) {
2514                 set_power_wells(power_domains, bxt_power_wells);
2515         } else if (IS_GEMINILAKE(dev_priv)) {
2516                 set_power_wells(power_domains, glk_power_wells);
2517         } else if (IS_CHERRYVIEW(dev_priv)) {
2518                 set_power_wells(power_domains, chv_power_wells);
2519         } else if (IS_VALLEYVIEW(dev_priv)) {
2520                 set_power_wells(power_domains, vlv_power_wells);
2521         } else if (IS_I830(dev_priv)) {
2522                 set_power_wells(power_domains, i830_power_wells);
2523         } else {
2524                 set_power_wells(power_domains, i9xx_always_on_power_well);
2525         }
2526
2527         assert_power_well_ids_unique(dev_priv);
2528
2529         return 0;
2530 }
2531
2532 /**
2533  * intel_power_domains_fini - finalizes the power domain structures
2534  * @dev_priv: i915 device instance
2535  *
2536  * Finalizes the power domain structures for @dev_priv depending upon the
2537  * supported platform. This function also disables runtime pm and ensures that
2538  * the device stays powered up so that the driver can be reloaded.
2539  */
2540 void intel_power_domains_fini(struct drm_i915_private *dev_priv)
2541 {
2542         struct device *kdev = &dev_priv->drm.pdev->dev;
2543
2544         /*
2545          * The i915.ko module is still not prepared to be loaded when
2546          * the power well is not enabled, so just enable it in case
2547          * we're going to unload/reload.
2548          * The following also reacquires the RPM reference the core passed
2549          * to the driver during loading, which is dropped in
2550          * intel_runtime_pm_enable(). We have to hand back the control of the
2551          * device to the core with this reference held.
2552          */
2553         intel_display_set_init_power(dev_priv, true);
2554
2555         /* Remove the refcount we took to keep power well support disabled. */
2556         if (!i915_modparams.disable_power_well)
2557                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
2558
2559         /*
2560          * Remove the refcount we took in intel_runtime_pm_enable() in case
2561          * the platform doesn't support runtime PM.
2562          */
2563         if (!HAS_RUNTIME_PM(dev_priv))
2564                 pm_runtime_put(kdev);
2565 }
2566
2567 static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
2568 {
2569         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2570         struct i915_power_well *power_well;
2571
2572         mutex_lock(&power_domains->lock);
2573         for_each_power_well(dev_priv, power_well) {
2574                 power_well->ops->sync_hw(dev_priv, power_well);
2575                 power_well->hw_enabled = power_well->ops->is_enabled(dev_priv,
2576                                                                      power_well);
2577         }
2578         mutex_unlock(&power_domains->lock);
2579 }
2580
2581 static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
2582 {
2583         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
2584         POSTING_READ(DBUF_CTL);
2585
2586         udelay(10);
2587
2588         if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
2589                 DRM_ERROR("DBuf power enable timeout\n");
2590 }
2591
2592 static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
2593 {
2594         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
2595         POSTING_READ(DBUF_CTL);
2596
2597         udelay(10);
2598
2599         if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
2600                 DRM_ERROR("DBuf power disable timeout!\n");
2601 }
2602
2603 static void skl_display_core_init(struct drm_i915_private *dev_priv,
2604                                    bool resume)
2605 {
2606         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2607         struct i915_power_well *well;
2608         uint32_t val;
2609
2610         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2611
2612         /* enable PCH reset handshake */
2613         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2614         I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE);
2615
2616         /* enable PG1 and Misc I/O */
2617         mutex_lock(&power_domains->lock);
2618
2619         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2620         intel_power_well_enable(dev_priv, well);
2621
2622         well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
2623         intel_power_well_enable(dev_priv, well);
2624
2625         mutex_unlock(&power_domains->lock);
2626
2627         skl_init_cdclk(dev_priv);
2628
2629         gen9_dbuf_enable(dev_priv);
2630
2631         if (resume && dev_priv->csr.dmc_payload)
2632                 intel_csr_load_program(dev_priv);
2633 }
2634
2635 static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
2636 {
2637         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2638         struct i915_power_well *well;
2639
2640         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2641
2642         gen9_dbuf_disable(dev_priv);
2643
2644         skl_uninit_cdclk(dev_priv);
2645
2646         /* The spec doesn't call for removing the reset handshake flag */
2647         /* disable PG1 and Misc I/O */
2648
2649         mutex_lock(&power_domains->lock);
2650
2651         /*
2652          * BSpec says to keep the MISC IO power well enabled here, only
2653          * remove our request for power well 1.
2654          * Note that even though the driver's request is removed power well 1
2655          * may stay enabled after this due to DMC's own request on it.
2656          */
2657         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2658         intel_power_well_disable(dev_priv, well);
2659
2660         mutex_unlock(&power_domains->lock);
2661
2662         usleep_range(10, 30);           /* 10 us delay per Bspec */
2663 }
2664
2665 void bxt_display_core_init(struct drm_i915_private *dev_priv,
2666                            bool resume)
2667 {
2668         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2669         struct i915_power_well *well;
2670         uint32_t val;
2671
2672         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2673
2674         /*
2675          * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
2676          * or else the reset will hang because there is no PCH to respond.
2677          * Move the handshake programming to initialization sequence.
2678          * Previously was left up to BIOS.
2679          */
2680         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2681         val &= ~RESET_PCH_HANDSHAKE_ENABLE;
2682         I915_WRITE(HSW_NDE_RSTWRN_OPT, val);
2683
2684         /* Enable PG1 */
2685         mutex_lock(&power_domains->lock);
2686
2687         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2688         intel_power_well_enable(dev_priv, well);
2689
2690         mutex_unlock(&power_domains->lock);
2691
2692         bxt_init_cdclk(dev_priv);
2693
2694         gen9_dbuf_enable(dev_priv);
2695
2696         if (resume && dev_priv->csr.dmc_payload)
2697                 intel_csr_load_program(dev_priv);
2698 }
2699
2700 void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
2701 {
2702         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2703         struct i915_power_well *well;
2704
2705         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2706
2707         gen9_dbuf_disable(dev_priv);
2708
2709         bxt_uninit_cdclk(dev_priv);
2710
2711         /* The spec doesn't call for removing the reset handshake flag */
2712
2713         /*
2714          * Disable PW1 (PG1).
2715          * Note that even though the driver's request is removed power well 1
2716          * may stay enabled after this due to DMC's own request on it.
2717          */
2718         mutex_lock(&power_domains->lock);
2719
2720         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2721         intel_power_well_disable(dev_priv, well);
2722
2723         mutex_unlock(&power_domains->lock);
2724
2725         usleep_range(10, 30);           /* 10 us delay per Bspec */
2726 }
2727
2728 enum {
2729         PROCMON_0_85V_DOT_0,
2730         PROCMON_0_95V_DOT_0,
2731         PROCMON_0_95V_DOT_1,
2732         PROCMON_1_05V_DOT_0,
2733         PROCMON_1_05V_DOT_1,
2734 };
2735
2736 static const struct cnl_procmon {
2737         u32 dw1, dw9, dw10;
2738 } cnl_procmon_values[] = {
2739         [PROCMON_0_85V_DOT_0] =
2740                 { .dw1 = 0x00000000, .dw9 = 0x62AB67BB, .dw10 = 0x51914F96, },
2741         [PROCMON_0_95V_DOT_0] =
2742                 { .dw1 = 0x00000000, .dw9 = 0x86E172C7, .dw10 = 0x77CA5EAB, },
2743         [PROCMON_0_95V_DOT_1] =
2744                 { .dw1 = 0x00000000, .dw9 = 0x93F87FE1, .dw10 = 0x8AE871C5, },
2745         [PROCMON_1_05V_DOT_0] =
2746                 { .dw1 = 0x00000000, .dw9 = 0x98FA82DD, .dw10 = 0x89E46DC1, },
2747         [PROCMON_1_05V_DOT_1] =
2748                 { .dw1 = 0x00440000, .dw9 = 0x9A00AB25, .dw10 = 0x8AE38FF1, },
2749 };
2750
2751 static void cnl_set_procmon_ref_values(struct drm_i915_private *dev_priv)
2752 {
2753         const struct cnl_procmon *procmon;
2754         u32 val;
2755
2756         val = I915_READ(CNL_PORT_COMP_DW3);
2757         switch (val & (PROCESS_INFO_MASK | VOLTAGE_INFO_MASK)) {
2758         default:
2759                 MISSING_CASE(val);
2760         case VOLTAGE_INFO_0_85V | PROCESS_INFO_DOT_0:
2761                 procmon = &cnl_procmon_values[PROCMON_0_85V_DOT_0];
2762                 break;
2763         case VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_0:
2764                 procmon = &cnl_procmon_values[PROCMON_0_95V_DOT_0];
2765                 break;
2766         case VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_1:
2767                 procmon = &cnl_procmon_values[PROCMON_0_95V_DOT_1];
2768                 break;
2769         case VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_0:
2770                 procmon = &cnl_procmon_values[PROCMON_1_05V_DOT_0];
2771                 break;
2772         case VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_1:
2773                 procmon = &cnl_procmon_values[PROCMON_1_05V_DOT_1];
2774                 break;
2775         }
2776
2777         val = I915_READ(CNL_PORT_COMP_DW1);
2778         val &= ~((0xff << 16) | 0xff);
2779         val |= procmon->dw1;
2780         I915_WRITE(CNL_PORT_COMP_DW1, val);
2781
2782         I915_WRITE(CNL_PORT_COMP_DW9, procmon->dw9);
2783         I915_WRITE(CNL_PORT_COMP_DW10, procmon->dw10);
2784 }
2785
2786 static void cnl_display_core_init(struct drm_i915_private *dev_priv, bool resume)
2787 {
2788         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2789         struct i915_power_well *well;
2790         u32 val;
2791
2792         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2793
2794         /* 1. Enable PCH Reset Handshake */
2795         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2796         val |= RESET_PCH_HANDSHAKE_ENABLE;
2797         I915_WRITE(HSW_NDE_RSTWRN_OPT, val);
2798
2799         /* 2. Enable Comp */
2800         val = I915_READ(CHICKEN_MISC_2);
2801         val &= ~CNL_COMP_PWR_DOWN;
2802         I915_WRITE(CHICKEN_MISC_2, val);
2803
2804         cnl_set_procmon_ref_values(dev_priv);
2805
2806         val = I915_READ(CNL_PORT_COMP_DW0);
2807         val |= COMP_INIT;
2808         I915_WRITE(CNL_PORT_COMP_DW0, val);
2809
2810         /* 3. */
2811         val = I915_READ(CNL_PORT_CL1CM_DW5);
2812         val |= CL_POWER_DOWN_ENABLE;
2813         I915_WRITE(CNL_PORT_CL1CM_DW5, val);
2814
2815         /*
2816          * 4. Enable Power Well 1 (PG1).
2817          *    The AUX IO power wells will be enabled on demand.
2818          */
2819         mutex_lock(&power_domains->lock);
2820         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2821         intel_power_well_enable(dev_priv, well);
2822         mutex_unlock(&power_domains->lock);
2823
2824         /* 5. Enable CD clock */
2825         cnl_init_cdclk(dev_priv);
2826
2827         /* 6. Enable DBUF */
2828         gen9_dbuf_enable(dev_priv);
2829
2830         if (resume && dev_priv->csr.dmc_payload)
2831                 intel_csr_load_program(dev_priv);
2832 }
2833
2834 static void cnl_display_core_uninit(struct drm_i915_private *dev_priv)
2835 {
2836         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2837         struct i915_power_well *well;
2838         u32 val;
2839
2840         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2841
2842         /* 1. Disable all display engine functions -> aready done */
2843
2844         /* 2. Disable DBUF */
2845         gen9_dbuf_disable(dev_priv);
2846
2847         /* 3. Disable CD clock */
2848         cnl_uninit_cdclk(dev_priv);
2849
2850         /*
2851          * 4. Disable Power Well 1 (PG1).
2852          *    The AUX IO power wells are toggled on demand, so they are already
2853          *    disabled at this point.
2854          */
2855         mutex_lock(&power_domains->lock);
2856         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2857         intel_power_well_disable(dev_priv, well);
2858         mutex_unlock(&power_domains->lock);
2859
2860         usleep_range(10, 30);           /* 10 us delay per Bspec */
2861
2862         /* 5. Disable Comp */
2863         val = I915_READ(CHICKEN_MISC_2);
2864         val |= CNL_COMP_PWR_DOWN;
2865         I915_WRITE(CHICKEN_MISC_2, val);
2866 }
2867
2868 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
2869 {
2870         struct i915_power_well *cmn_bc =
2871                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
2872         struct i915_power_well *cmn_d =
2873                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
2874
2875         /*
2876          * DISPLAY_PHY_CONTROL can get corrupted if read. As a
2877          * workaround never ever read DISPLAY_PHY_CONTROL, and
2878          * instead maintain a shadow copy ourselves. Use the actual
2879          * power well state and lane status to reconstruct the
2880          * expected initial value.
2881          */
2882         dev_priv->chv_phy_control =
2883                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
2884                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
2885                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
2886                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
2887                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
2888
2889         /*
2890          * If all lanes are disabled we leave the override disabled
2891          * with all power down bits cleared to match the state we
2892          * would use after disabling the port. Otherwise enable the
2893          * override and set the lane powerdown bits accding to the
2894          * current lane status.
2895          */
2896         if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
2897                 uint32_t status = I915_READ(DPLL(PIPE_A));
2898                 unsigned int mask;
2899
2900                 mask = status & DPLL_PORTB_READY_MASK;
2901                 if (mask == 0xf)
2902                         mask = 0x0;
2903                 else
2904                         dev_priv->chv_phy_control |=
2905                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
2906
2907                 dev_priv->chv_phy_control |=
2908                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
2909
2910                 mask = (status & DPLL_PORTC_READY_MASK) >> 4;
2911                 if (mask == 0xf)
2912                         mask = 0x0;
2913                 else
2914                         dev_priv->chv_phy_control |=
2915                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
2916
2917                 dev_priv->chv_phy_control |=
2918                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
2919
2920                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
2921
2922                 dev_priv->chv_phy_assert[DPIO_PHY0] = false;
2923         } else {
2924                 dev_priv->chv_phy_assert[DPIO_PHY0] = true;
2925         }
2926
2927         if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
2928                 uint32_t status = I915_READ(DPIO_PHY_STATUS);
2929                 unsigned int mask;
2930
2931                 mask = status & DPLL_PORTD_READY_MASK;
2932
2933                 if (mask == 0xf)
2934                         mask = 0x0;
2935                 else
2936                         dev_priv->chv_phy_control |=
2937                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
2938
2939                 dev_priv->chv_phy_control |=
2940                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
2941
2942                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
2943
2944                 dev_priv->chv_phy_assert[DPIO_PHY1] = false;
2945         } else {
2946                 dev_priv->chv_phy_assert[DPIO_PHY1] = true;
2947         }
2948
2949         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
2950
2951         DRM_DEBUG_KMS("Initial PHY_CONTROL=0x%08x\n",
2952                       dev_priv->chv_phy_control);
2953 }
2954
2955 static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
2956 {
2957         struct i915_power_well *cmn =
2958                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
2959         struct i915_power_well *disp2d =
2960                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DISP2D);
2961
2962         /* If the display might be already active skip this */
2963         if (cmn->ops->is_enabled(dev_priv, cmn) &&
2964             disp2d->ops->is_enabled(dev_priv, disp2d) &&
2965             I915_READ(DPIO_CTL) & DPIO_CMNRST)
2966                 return;
2967
2968         DRM_DEBUG_KMS("toggling display PHY side reset\n");
2969
2970         /* cmnlane needs DPLL registers */
2971         disp2d->ops->enable(dev_priv, disp2d);
2972
2973         /*
2974          * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
2975          * Need to assert and de-assert PHY SB reset by gating the
2976          * common lane power, then un-gating it.
2977          * Simply ungating isn't enough to reset the PHY enough to get
2978          * ports and lanes running.
2979          */
2980         cmn->ops->disable(dev_priv, cmn);
2981 }
2982
2983 /**
2984  * intel_power_domains_init_hw - initialize hardware power domain state
2985  * @dev_priv: i915 device instance
2986  * @resume: Called from resume code paths or not
2987  *
2988  * This function initializes the hardware power domain state and enables all
2989  * power wells belonging to the INIT power domain. Power wells in other
2990  * domains (and not in the INIT domain) are referenced or disabled during the
2991  * modeset state HW readout. After that the reference count of each power well
2992  * must match its HW enabled state, see intel_power_domains_verify_state().
2993  */
2994 void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume)
2995 {
2996         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2997
2998         power_domains->initializing = true;
2999
3000         if (IS_CANNONLAKE(dev_priv)) {
3001                 cnl_display_core_init(dev_priv, resume);
3002         } else if (IS_GEN9_BC(dev_priv)) {
3003                 skl_display_core_init(dev_priv, resume);
3004         } else if (IS_GEN9_LP(dev_priv)) {
3005                 bxt_display_core_init(dev_priv, resume);
3006         } else if (IS_CHERRYVIEW(dev_priv)) {
3007                 mutex_lock(&power_domains->lock);
3008                 chv_phy_control_init(dev_priv);
3009                 mutex_unlock(&power_domains->lock);
3010         } else if (IS_VALLEYVIEW(dev_priv)) {
3011                 mutex_lock(&power_domains->lock);
3012                 vlv_cmnlane_wa(dev_priv);
3013                 mutex_unlock(&power_domains->lock);
3014         }
3015
3016         /* For now, we need the power well to be always enabled. */
3017         intel_display_set_init_power(dev_priv, true);
3018         /* Disable power support if the user asked so. */
3019         if (!i915_modparams.disable_power_well)
3020                 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
3021         intel_power_domains_sync_hw(dev_priv);
3022         power_domains->initializing = false;
3023 }
3024
3025 /**
3026  * intel_power_domains_suspend - suspend power domain state
3027  * @dev_priv: i915 device instance
3028  *
3029  * This function prepares the hardware power domain state before entering
3030  * system suspend. It must be paired with intel_power_domains_init_hw().
3031  */
3032 void intel_power_domains_suspend(struct drm_i915_private *dev_priv)
3033 {
3034         /*
3035          * Even if power well support was disabled we still want to disable
3036          * power wells while we are system suspended.
3037          */
3038         if (!i915_modparams.disable_power_well)
3039                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
3040
3041         if (IS_CANNONLAKE(dev_priv))
3042                 cnl_display_core_uninit(dev_priv);
3043         else if (IS_GEN9_BC(dev_priv))
3044                 skl_display_core_uninit(dev_priv);
3045         else if (IS_GEN9_LP(dev_priv))
3046                 bxt_display_core_uninit(dev_priv);
3047 }
3048
3049 static void intel_power_domains_dump_info(struct drm_i915_private *dev_priv)
3050 {
3051         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3052         struct i915_power_well *power_well;
3053
3054         for_each_power_well(dev_priv, power_well) {
3055                 enum intel_display_power_domain domain;
3056
3057                 DRM_DEBUG_DRIVER("%-25s %d\n",
3058                                  power_well->name, power_well->count);
3059
3060                 for_each_power_domain(domain, power_well->domains)
3061                         DRM_DEBUG_DRIVER("  %-23s %d\n",
3062                                          intel_display_power_domain_str(domain),
3063                                          power_domains->domain_use_count[domain]);
3064         }
3065 }
3066
3067 /**
3068  * intel_power_domains_verify_state - verify the HW/SW state for all power wells
3069  * @dev_priv: i915 device instance
3070  *
3071  * Verify if the reference count of each power well matches its HW enabled
3072  * state and the total refcount of the domains it belongs to. This must be
3073  * called after modeset HW state sanitization, which is responsible for
3074  * acquiring reference counts for any power wells in use and disabling the
3075  * ones left on by BIOS but not required by any active output.
3076  */
3077 void intel_power_domains_verify_state(struct drm_i915_private *dev_priv)
3078 {
3079         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3080         struct i915_power_well *power_well;
3081         bool dump_domain_info;
3082
3083         mutex_lock(&power_domains->lock);
3084
3085         dump_domain_info = false;
3086         for_each_power_well(dev_priv, power_well) {
3087                 enum intel_display_power_domain domain;
3088                 int domains_count;
3089                 bool enabled;
3090
3091                 /*
3092                  * Power wells not belonging to any domain (like the MISC_IO
3093                  * and PW1 power wells) are under FW control, so ignore them,
3094                  * since their state can change asynchronously.
3095                  */
3096                 if (!power_well->domains)
3097                         continue;
3098
3099                 enabled = power_well->ops->is_enabled(dev_priv, power_well);
3100                 if ((power_well->count || power_well->always_on) != enabled)
3101                         DRM_ERROR("power well %s state mismatch (refcount %d/enabled %d)",
3102                                   power_well->name, power_well->count, enabled);
3103
3104                 domains_count = 0;
3105                 for_each_power_domain(domain, power_well->domains)
3106                         domains_count += power_domains->domain_use_count[domain];
3107
3108                 if (power_well->count != domains_count) {
3109                         DRM_ERROR("power well %s refcount/domain refcount mismatch "
3110                                   "(refcount %d/domains refcount %d)\n",
3111                                   power_well->name, power_well->count,
3112                                   domains_count);
3113                         dump_domain_info = true;
3114                 }
3115         }
3116
3117         if (dump_domain_info) {
3118                 static bool dumped;
3119
3120                 if (!dumped) {
3121                         intel_power_domains_dump_info(dev_priv);
3122                         dumped = true;
3123                 }
3124         }
3125
3126         mutex_unlock(&power_domains->lock);
3127 }
3128
3129 /**
3130  * intel_runtime_pm_get - grab a runtime pm reference
3131  * @dev_priv: i915 device instance
3132  *
3133  * This function grabs a device-level runtime pm reference (mostly used for GEM
3134  * code to ensure the GTT or GT is on) and ensures that it is powered up.
3135  *
3136  * Any runtime pm reference obtained by this function must have a symmetric
3137  * call to intel_runtime_pm_put() to release the reference again.
3138  */
3139 void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
3140 {
3141         struct pci_dev *pdev = dev_priv->drm.pdev;
3142         struct device *kdev = &pdev->dev;
3143         int ret;
3144
3145         ret = pm_runtime_get_sync(kdev);
3146         WARN_ONCE(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret);
3147
3148         atomic_inc(&dev_priv->runtime_pm.wakeref_count);
3149         assert_rpm_wakelock_held(dev_priv);
3150 }
3151
3152 /**
3153  * intel_runtime_pm_get_if_in_use - grab a runtime pm reference if device in use
3154  * @dev_priv: i915 device instance
3155  *
3156  * This function grabs a device-level runtime pm reference if the device is
3157  * already in use and ensures that it is powered up.
3158  *
3159  * Any runtime pm reference obtained by this function must have a symmetric
3160  * call to intel_runtime_pm_put() to release the reference again.
3161  */
3162 bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv)
3163 {
3164         struct pci_dev *pdev = dev_priv->drm.pdev;
3165         struct device *kdev = &pdev->dev;
3166
3167         if (IS_ENABLED(CONFIG_PM)) {
3168                 int ret = pm_runtime_get_if_in_use(kdev);
3169
3170                 /*
3171                  * In cases runtime PM is disabled by the RPM core and we get
3172                  * an -EINVAL return value we are not supposed to call this
3173                  * function, since the power state is undefined. This applies
3174                  * atm to the late/early system suspend/resume handlers.
3175                  */
3176                 WARN_ONCE(ret < 0,
3177                           "pm_runtime_get_if_in_use() failed: %d\n", ret);
3178                 if (ret <= 0)
3179                         return false;
3180         }
3181
3182         atomic_inc(&dev_priv->runtime_pm.wakeref_count);
3183         assert_rpm_wakelock_held(dev_priv);
3184
3185         return true;
3186 }
3187
3188 /**
3189  * intel_runtime_pm_get_noresume - grab a runtime pm reference
3190  * @dev_priv: i915 device instance
3191  *
3192  * This function grabs a device-level runtime pm reference (mostly used for GEM
3193  * code to ensure the GTT or GT is on).
3194  *
3195  * It will _not_ power up the device but instead only check that it's powered
3196  * on.  Therefore it is only valid to call this functions from contexts where
3197  * the device is known to be powered up and where trying to power it up would
3198  * result in hilarity and deadlocks. That pretty much means only the system
3199  * suspend/resume code where this is used to grab runtime pm references for
3200  * delayed setup down in work items.
3201  *
3202  * Any runtime pm reference obtained by this function must have a symmetric
3203  * call to intel_runtime_pm_put() to release the reference again.
3204  */
3205 void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
3206 {
3207         struct pci_dev *pdev = dev_priv->drm.pdev;
3208         struct device *kdev = &pdev->dev;
3209
3210         assert_rpm_wakelock_held(dev_priv);
3211         pm_runtime_get_noresume(kdev);
3212
3213         atomic_inc(&dev_priv->runtime_pm.wakeref_count);
3214 }
3215
3216 /**
3217  * intel_runtime_pm_put - release a runtime pm reference
3218  * @dev_priv: i915 device instance
3219  *
3220  * This function drops the device-level runtime pm reference obtained by
3221  * intel_runtime_pm_get() and might power down the corresponding
3222  * hardware block right away if this is the last reference.
3223  */
3224 void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
3225 {
3226         struct pci_dev *pdev = dev_priv->drm.pdev;
3227         struct device *kdev = &pdev->dev;
3228
3229         assert_rpm_wakelock_held(dev_priv);
3230         atomic_dec(&dev_priv->runtime_pm.wakeref_count);
3231
3232         pm_runtime_mark_last_busy(kdev);
3233         pm_runtime_put_autosuspend(kdev);
3234 }
3235
3236 /**
3237  * intel_runtime_pm_enable - enable runtime pm
3238  * @dev_priv: i915 device instance
3239  *
3240  * This function enables runtime pm at the end of the driver load sequence.
3241  *
3242  * Note that this function does currently not enable runtime pm for the
3243  * subordinate display power domains. That is only done on the first modeset
3244  * using intel_display_set_init_power().
3245  */
3246 void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
3247 {
3248         struct pci_dev *pdev = dev_priv->drm.pdev;
3249         struct device *kdev = &pdev->dev;
3250
3251         pm_runtime_set_autosuspend_delay(kdev, 10000); /* 10s */
3252         pm_runtime_mark_last_busy(kdev);
3253
3254         /*
3255          * Take a permanent reference to disable the RPM functionality and drop
3256          * it only when unloading the driver. Use the low level get/put helpers,
3257          * so the driver's own RPM reference tracking asserts also work on
3258          * platforms without RPM support.
3259          */
3260         if (!HAS_RUNTIME_PM(dev_priv)) {
3261                 int ret;
3262
3263                 pm_runtime_dont_use_autosuspend(kdev);
3264                 ret = pm_runtime_get_sync(kdev);
3265                 WARN(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret);
3266         } else {
3267                 pm_runtime_use_autosuspend(kdev);
3268         }
3269
3270         /*
3271          * The core calls the driver load handler with an RPM reference held.
3272          * We drop that here and will reacquire it during unloading in
3273          * intel_power_domains_fini().
3274          */
3275         pm_runtime_put_autosuspend(kdev);
3276 }