drivers/gpu/drm/i915/display/intel_display_power_well.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2022 Intel Corporation
   4  */
   5
   6 #include "i915_drv.h"
   7 #include "i915_irq.h"
   8 #include "i915_reg.h"
   9 #include "intel_backlight_regs.h"
  10 #include "intel_combo_phy.h"
  11 #include "intel_combo_phy_regs.h"
  12 #include "intel_crt.h"
  13 #include "intel_de.h"
  14 #include "intel_display_irq.h"
  15 #include "intel_display_power_well.h"
  16 #include "intel_display_types.h"
  17 #include "intel_dkl_phy.h"
  18 #include "intel_dkl_phy_regs.h"
  19 #include "intel_dmc.h"
  20 #include "intel_dmc_wl.h"
  21 #include "intel_dp_aux_regs.h"
  22 #include "intel_dpio_phy.h"
  23 #include "intel_dpll.h"
  24 #include "intel_hotplug.h"
  25 #include "intel_pcode.h"
  26 #include "intel_pps.h"
  27 #include "intel_tc.h"
  28 #include "intel_vga.h"
  29 #include "skl_watermark.h"
  30 #include "vlv_dpio_phy_regs.h"
  31 #include "vlv_sideband.h"
  32 #include "vlv_sideband_reg.h"
  33
  34 struct i915_power_well_regs {
  35         i915_reg_t bios;
  36         i915_reg_t driver;
  37         i915_reg_t kvmr;
  38         i915_reg_t debug;
  39 };
  40
  41 struct i915_power_well_ops {
  42         const struct i915_power_well_regs *regs;
  43         /*
  44          * Synchronize the well's hw state to match the current sw state, for
  45          * example enable/disable it based on the current refcount. Called
  46          * during driver init and resume time, possibly after first calling
  47          * the enable/disable handlers.
  48          */
  49         void (*sync_hw)(struct intel_display *display,
  50                         struct i915_power_well *power_well);
  51         /*
  52          * Enable the well and resources that depend on it (for example
  53          * interrupts located on the well). Called after the 0->1 refcount
  54          * transition.
  55          */
  56         void (*enable)(struct intel_display *display,
  57                        struct i915_power_well *power_well);
  58         /*
  59          * Disable the well and resources that depend on it. Called after
  60          * the 1->0 refcount transition.
  61          */
  62         void (*disable)(struct intel_display *display,
  63                         struct i915_power_well *power_well);
  64         /* Returns the hw enabled state. */
  65         bool (*is_enabled)(struct intel_display *display,
  66                            struct i915_power_well *power_well);
  67 };
  68
  69 static const struct i915_power_well_instance *
  70 i915_power_well_instance(const struct i915_power_well *power_well)
  71 {
  72         return &power_well->desc->instances->list[power_well->instance_idx];
  73 }
  74
  75 struct i915_power_well *
  76 lookup_power_well(struct intel_display *display,
  77                   enum i915_power_well_id power_well_id)
  78 {
  79         struct i915_power_well *power_well;
  80
  81         for_each_power_well(display, power_well)
  82                 if (i915_power_well_instance(power_well)->id == power_well_id)
  83                         return power_well;
  84
  85         /*
  86          * It's not feasible to add error checking code to the callers since
  87          * this condition really shouldn't happen and it doesn't even make sense
  88          * to abort things like display initialization sequences. Just return
  89          * the first power well and hope the WARN gets reported so we can fix
  90          * our driver.
  91          */
  92         drm_WARN(display->drm, 1,
  93                  "Power well %d not defined for this platform\n",
  94                  power_well_id);
  95         return &display->power.domains.power_wells[0];
  96 }
  97
  98 void intel_power_well_enable(struct intel_display *display,
  99                              struct i915_power_well *power_well)
 100 {
 101         drm_dbg_kms(display->drm, "enabling %s\n", intel_power_well_name(power_well));
 102         power_well->desc->ops->enable(display, power_well);
 103         power_well->hw_enabled = true;
 104 }
 105
 106 void intel_power_well_disable(struct intel_display *display,
 107                               struct i915_power_well *power_well)
 108 {
 109         drm_dbg_kms(display->drm, "disabling %s\n", intel_power_well_name(power_well));
 110         power_well->hw_enabled = false;
 111         power_well->desc->ops->disable(display, power_well);
 112 }
 113
 114 void intel_power_well_sync_hw(struct intel_display *display,
 115                               struct i915_power_well *power_well)
 116 {
 117         power_well->desc->ops->sync_hw(display, power_well);
 118         power_well->hw_enabled = power_well->desc->ops->is_enabled(display, power_well);
 119 }
 120
 121 void intel_power_well_get(struct intel_display *display,
 122                           struct i915_power_well *power_well)
 123 {
 124         if (!power_well->count++)
 125                 intel_power_well_enable(display, power_well);
 126 }
 127
 128 void intel_power_well_put(struct intel_display *display,
 129                           struct i915_power_well *power_well)
 130 {
 131         drm_WARN(display->drm, !power_well->count,
 132                  "Use count on power well %s is already zero",
 133                  i915_power_well_instance(power_well)->name);
 134
 135         if (!--power_well->count)
 136                 intel_power_well_disable(display, power_well);
 137 }
 138
 139 bool intel_power_well_is_enabled(struct intel_display *display,
 140                                  struct i915_power_well *power_well)
 141 {
 142         return power_well->desc->ops->is_enabled(display, power_well);
 143 }
 144
 145 bool intel_power_well_is_enabled_cached(struct i915_power_well *power_well)
 146 {
 147         return power_well->hw_enabled;
 148 }
 149
 150 bool intel_display_power_well_is_enabled(struct intel_display *display,
 151                                          enum i915_power_well_id power_well_id)
 152 {
 153         struct i915_power_well *power_well;
 154
 155         power_well = lookup_power_well(display, power_well_id);
 156
 157         return intel_power_well_is_enabled(display, power_well);
 158 }
 159
 160 bool intel_power_well_is_always_on(struct i915_power_well *power_well)
 161 {
 162         return power_well->desc->always_on;
 163 }
 164
 165 const char *intel_power_well_name(struct i915_power_well *power_well)
 166 {
 167         return i915_power_well_instance(power_well)->name;
 168 }
 169
 170 struct intel_power_domain_mask *intel_power_well_domains(struct i915_power_well *power_well)
 171 {
 172         return &power_well->domains;
 173 }
 174
 175 int intel_power_well_refcount(struct i915_power_well *power_well)
 176 {
 177         return power_well->count;
 178 }
 179
 180 /*
 181  * Starting with Haswell, we have a "Power Down Well" that can be turned off
 182  * when not needed anymore. We have 4 registers that can request the power well
 183  * to be enabled, and it will only be disabled if none of the registers is
 184  * requesting it to be enabled.
 185  */
 186 static void hsw_power_well_post_enable(struct intel_display *display,
 187                                        u8 irq_pipe_mask, bool has_vga)
 188 {
 189         struct drm_i915_private *dev_priv = to_i915(display->drm);
 190
 191         if (has_vga)
 192                 intel_vga_reset_io_mem(display);
 193
 194         if (irq_pipe_mask)
 195                 gen8_irq_power_well_post_enable(dev_priv, irq_pipe_mask);
 196 }
 197
 198 static void hsw_power_well_pre_disable(struct intel_display *display,
 199                                        u8 irq_pipe_mask)
 200 {
 201         struct drm_i915_private *dev_priv = to_i915(display->drm);
 202
 203         if (irq_pipe_mask)
 204                 gen8_irq_power_well_pre_disable(dev_priv, irq_pipe_mask);
 205 }
 206
 207 #define ICL_AUX_PW_TO_PHY(pw_idx)       \
 208         ((pw_idx) - ICL_PW_CTL_IDX_AUX_A + PHY_A)
 209
 210 #define ICL_AUX_PW_TO_CH(pw_idx)        \
 211         ((pw_idx) - ICL_PW_CTL_IDX_AUX_A + AUX_CH_A)
 212
 213 #define ICL_TBT_AUX_PW_TO_CH(pw_idx)    \
 214         ((pw_idx) - ICL_PW_CTL_IDX_AUX_TBT1 + AUX_CH_C)
 215
 216 static enum aux_ch icl_aux_pw_to_ch(const struct i915_power_well *power_well)
 217 {
 218         int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 219
 220         return power_well->desc->is_tc_tbt ? ICL_TBT_AUX_PW_TO_CH(pw_idx) :
 221                                              ICL_AUX_PW_TO_CH(pw_idx);
 222 }
 223
 224 static struct intel_digital_port *
 225 aux_ch_to_digital_port(struct intel_display *display,
 226                        enum aux_ch aux_ch)
 227 {
 228         struct intel_encoder *encoder;
 229
 230         for_each_intel_encoder(display->drm, encoder) {
 231                 struct intel_digital_port *dig_port;
 232
 233                 /* We'll check the MST primary port */
 234                 if (encoder->type == INTEL_OUTPUT_DP_MST)
 235                         continue;
 236
 237                 dig_port = enc_to_dig_port(encoder);
 238
 239                 if (dig_port && dig_port->aux_ch == aux_ch)
 240                         return dig_port;
 241         }
 242
 243         return NULL;
 244 }
 245
 246 static enum phy icl_aux_pw_to_phy(struct intel_display *display,
 247                                   const struct i915_power_well *power_well)
 248 {
 249         enum aux_ch aux_ch = icl_aux_pw_to_ch(power_well);
 250         struct intel_digital_port *dig_port = aux_ch_to_digital_port(display, aux_ch);
 251
 252         /*
 253          * FIXME should we care about the (VBT defined) dig_port->aux_ch
 254          * relationship or should this be purely defined by the hardware layout?
 255          * Currently if the port doesn't appear in the VBT, or if it's declared
 256          * as HDMI-only and routed to a combo PHY, the encoder either won't be
 257          * present at all or it will not have an aux_ch assigned.
 258          */
 259         return dig_port ? intel_encoder_to_phy(&dig_port->base) : PHY_NONE;
 260 }
 261
 262 static void hsw_wait_for_power_well_enable(struct intel_display *display,
 263                                            struct i915_power_well *power_well,
 264                                            bool timeout_expected)
 265 {
 266         const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 267         int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 268         int timeout = power_well->desc->enable_timeout ? : 1;
 269
 270         /*
 271          * For some power wells we're not supposed to watch the status bit for
 272          * an ack, but rather just wait a fixed amount of time and then
 273          * proceed.  This is only used on DG2.
 274          */
 275         if (display->platform.dg2 && power_well->desc->fixed_enable_delay) {
 276                 usleep_range(600, 1200);
 277                 return;
 278         }
 279
 280         /* Timeout for PW1:10 us, AUX:not specified, other PWs:20 us. */
 281         if (intel_de_wait_for_set(display, regs->driver,
 282                                   HSW_PWR_WELL_CTL_STATE(pw_idx), timeout)) {
 283                 drm_dbg_kms(display->drm, "%s power well enable timeout\n",
 284                             intel_power_well_name(power_well));
 285
 286                 drm_WARN_ON(display->drm, !timeout_expected);
 287
 288         }
 289 }
 290
 291 static u32 hsw_power_well_requesters(struct intel_display *display,
 292                                      const struct i915_power_well_regs *regs,
 293                                      int pw_idx)
 294 {
 295         u32 req_mask = HSW_PWR_WELL_CTL_REQ(pw_idx);
 296         u32 ret;
 297
 298         ret = intel_de_read(display, regs->bios) & req_mask ? 1 : 0;
 299         ret |= intel_de_read(display, regs->driver) & req_mask ? 2 : 0;
 300         if (regs->kvmr.reg)
 301                 ret |= intel_de_read(display, regs->kvmr) & req_mask ? 4 : 0;
 302         ret |= intel_de_read(display, regs->debug) & req_mask ? 8 : 0;
 303
 304         return ret;
 305 }
 306
 307 static void hsw_wait_for_power_well_disable(struct intel_display *display,
 308                                             struct i915_power_well *power_well)
 309 {
 310         const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 311         int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 312         bool disabled;
 313         u32 reqs;
 314
 315         /*
 316          * Bspec doesn't require waiting for PWs to get disabled, but still do
 317          * this for paranoia. The known cases where a PW will be forced on:
 318          * - a KVMR request on any power well via the KVMR request register
 319          * - a DMC request on PW1 and MISC_IO power wells via the BIOS and
 320          *   DEBUG request registers
 321          * Skip the wait in case any of the request bits are set and print a
 322          * diagnostic message.
 323          */
 324         wait_for((disabled = !(intel_de_read(display, regs->driver) &
 325                                HSW_PWR_WELL_CTL_STATE(pw_idx))) ||
 326                  (reqs = hsw_power_well_requesters(display, regs, pw_idx)), 1);
 327         if (disabled)
 328                 return;
 329
 330         drm_dbg_kms(display->drm,
 331                     "%s forced on (bios:%d driver:%d kvmr:%d debug:%d)\n",
 332                     intel_power_well_name(power_well),
 333                     !!(reqs & 1), !!(reqs & 2), !!(reqs & 4), !!(reqs & 8));
 334 }
 335
 336 static void gen9_wait_for_power_well_fuses(struct intel_display *display,
 337                                            enum skl_power_gate pg)
 338 {
 339         /* Timeout 5us for PG#0, for other PGs 1us */
 340         drm_WARN_ON(display->drm,
 341                     intel_de_wait_for_set(display, SKL_FUSE_STATUS,
 342                                           SKL_FUSE_PG_DIST_STATUS(pg), 1));
 343 }
 344
 345 static void hsw_power_well_enable(struct intel_display *display,
 346                                   struct i915_power_well *power_well)
 347 {
 348         const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 349         int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 350
 351         if (power_well->desc->has_fuses) {
 352                 enum skl_power_gate pg;
 353
 354                 pg = DISPLAY_VER(display) >= 11 ? ICL_PW_CTL_IDX_TO_PG(pw_idx) :
 355                                                  SKL_PW_CTL_IDX_TO_PG(pw_idx);
 356
 357                 /* Wa_16013190616:adlp */
 358                 if (display->platform.alderlake_p && pg == SKL_PG1)
 359                         intel_de_rmw(display, GEN8_CHICKEN_DCPR_1, 0, DISABLE_FLR_SRC);
 360
 361                 /*
 362                  * For PW1 we have to wait both for the PW0/PG0 fuse state
 363                  * before enabling the power well and PW1/PG1's own fuse
 364                  * state after the enabling. For all other power wells with
 365                  * fuses we only have to wait for that PW/PG's fuse state
 366                  * after the enabling.
 367                  */
 368                 if (pg == SKL_PG1)
 369                         gen9_wait_for_power_well_fuses(display, SKL_PG0);
 370         }
 371
 372         intel_de_rmw(display, regs->driver, 0, HSW_PWR_WELL_CTL_REQ(pw_idx));
 373
 374         hsw_wait_for_power_well_enable(display, power_well, false);
 375
 376         if (power_well->desc->has_fuses) {
 377                 enum skl_power_gate pg;
 378
 379                 pg = DISPLAY_VER(display) >= 11 ? ICL_PW_CTL_IDX_TO_PG(pw_idx) :
 380                                                  SKL_PW_CTL_IDX_TO_PG(pw_idx);
 381                 gen9_wait_for_power_well_fuses(display, pg);
 382         }
 383
 384         hsw_power_well_post_enable(display,
 385                                    power_well->desc->irq_pipe_mask,
 386                                    power_well->desc->has_vga);
 387 }
 388
 389 static void hsw_power_well_disable(struct intel_display *display,
 390                                    struct i915_power_well *power_well)
 391 {
 392         const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 393         int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 394
 395         hsw_power_well_pre_disable(display,
 396                                    power_well->desc->irq_pipe_mask);
 397
 398         intel_de_rmw(display, regs->driver, HSW_PWR_WELL_CTL_REQ(pw_idx), 0);
 399         hsw_wait_for_power_well_disable(display, power_well);
 400 }
 401
 402 static bool intel_aux_ch_is_edp(struct intel_display *display, enum aux_ch aux_ch)
 403 {
 404         struct intel_digital_port *dig_port = aux_ch_to_digital_port(display, aux_ch);
 405
 406         return dig_port && dig_port->base.type == INTEL_OUTPUT_EDP;
 407 }
 408
 409 static void
 410 icl_combo_phy_aux_power_well_enable(struct intel_display *display,
 411                                     struct i915_power_well *power_well)
 412 {
 413         const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 414         int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 415
 416         drm_WARN_ON(display->drm, !display->platform.icelake);
 417
 418         intel_de_rmw(display, regs->driver, 0, HSW_PWR_WELL_CTL_REQ(pw_idx));
 419
 420         /*
 421          * FIXME not sure if we should derive the PHY from the pw_idx, or
 422          * from the VBT defined AUX_CH->DDI->PHY mapping.
 423          */
 424         intel_de_rmw(display, ICL_PORT_CL_DW12(ICL_AUX_PW_TO_PHY(pw_idx)),
 425                      0, ICL_LANE_ENABLE_AUX);
 426
 427         hsw_wait_for_power_well_enable(display, power_well, false);
 428
 429         /* Display WA #1178: icl */
 430         if (pw_idx >= ICL_PW_CTL_IDX_AUX_A && pw_idx <= ICL_PW_CTL_IDX_AUX_B &&
 431             !intel_aux_ch_is_edp(display, ICL_AUX_PW_TO_CH(pw_idx)))
 432                 intel_de_rmw(display, ICL_PORT_TX_DW6_AUX(ICL_AUX_PW_TO_PHY(pw_idx)),
 433                              0, O_FUNC_OVRD_EN | O_LDO_BYPASS_CRI);
 434 }
 435
 436 static void
 437 icl_combo_phy_aux_power_well_disable(struct intel_display *display,
 438                                      struct i915_power_well *power_well)
 439 {
 440         const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 441         int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 442
 443         drm_WARN_ON(display->drm, !display->platform.icelake);
 444
 445         /*
 446          * FIXME not sure if we should derive the PHY from the pw_idx, or
 447          * from the VBT defined AUX_CH->DDI->PHY mapping.
 448          */
 449         intel_de_rmw(display, ICL_PORT_CL_DW12(ICL_AUX_PW_TO_PHY(pw_idx)),
 450                      ICL_LANE_ENABLE_AUX, 0);
 451
 452         intel_de_rmw(display, regs->driver, HSW_PWR_WELL_CTL_REQ(pw_idx), 0);
 453
 454         hsw_wait_for_power_well_disable(display, power_well);
 455 }
 456
 457 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
 458
 459 static void icl_tc_port_assert_ref_held(struct intel_display *display,
 460                                         struct i915_power_well *power_well,
 461                                         struct intel_digital_port *dig_port)
 462 {
 463         if (drm_WARN_ON(display->drm, !dig_port))
 464                 return;
 465
 466         if (DISPLAY_VER(display) == 11 && intel_tc_cold_requires_aux_pw(dig_port))
 467                 return;
 468
 469         drm_WARN_ON(display->drm, !intel_tc_port_ref_held(dig_port));
 470 }
 471
 472 #else
 473
 474 static void icl_tc_port_assert_ref_held(struct intel_display *display,
 475                                         struct i915_power_well *power_well,
 476                                         struct intel_digital_port *dig_port)
 477 {
 478 }
 479
 480 #endif
 481
 482 #define TGL_AUX_PW_TO_TC_PORT(pw_idx)   ((pw_idx) - TGL_PW_CTL_IDX_AUX_TC1)
 483
 484 static void icl_tc_cold_exit(struct intel_display *display)
 485 {
 486         struct drm_i915_private *i915 = to_i915(display->drm);
 487         int ret, tries = 0;
 488
 489         while (1) {
 490                 ret = snb_pcode_write_timeout(&i915->uncore, ICL_PCODE_EXIT_TCCOLD, 0,
 491                                               250, 1);
 492                 if (ret != -EAGAIN || ++tries == 3)
 493                         break;
 494                 msleep(1);
 495         }
 496
 497         /* Spec states that TC cold exit can take up to 1ms to complete */
 498         if (!ret)
 499                 msleep(1);
 500
 501         /* TODO: turn failure into a error as soon i915 CI updates ICL IFWI */
 502         drm_dbg_kms(&i915->drm, "TC cold block %s\n", ret ? "failed" :
 503                     "succeeded");
 504 }
 505
 506 static void
 507 icl_tc_phy_aux_power_well_enable(struct intel_display *display,
 508                                  struct i915_power_well *power_well)
 509 {
 510         struct drm_i915_private *dev_priv = to_i915(display->drm);
 511         enum aux_ch aux_ch = icl_aux_pw_to_ch(power_well);
 512         struct intel_digital_port *dig_port = aux_ch_to_digital_port(display, aux_ch);
 513         const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 514         bool is_tbt = power_well->desc->is_tc_tbt;
 515         bool timeout_expected;
 516
 517         icl_tc_port_assert_ref_held(display, power_well, dig_port);
 518
 519         intel_de_rmw(display, DP_AUX_CH_CTL(aux_ch),
 520                      DP_AUX_CH_CTL_TBT_IO, is_tbt ? DP_AUX_CH_CTL_TBT_IO : 0);
 521
 522         intel_de_rmw(display, regs->driver,
 523                      0,
 524                      HSW_PWR_WELL_CTL_REQ(i915_power_well_instance(power_well)->hsw.idx));
 525
 526         /*
 527          * An AUX timeout is expected if the TBT DP tunnel is down,
 528          * or need to enable AUX on a legacy TypeC port as part of the TC-cold
 529          * exit sequence.
 530          */
 531         timeout_expected = is_tbt || intel_tc_cold_requires_aux_pw(dig_port);
 532         if (DISPLAY_VER(display) == 11 && intel_tc_cold_requires_aux_pw(dig_port))
 533                 icl_tc_cold_exit(display);
 534
 535         hsw_wait_for_power_well_enable(display, power_well, timeout_expected);
 536
 537         if (DISPLAY_VER(display) >= 12 && !is_tbt) {
 538                 enum tc_port tc_port;
 539
 540                 tc_port = TGL_AUX_PW_TO_TC_PORT(i915_power_well_instance(power_well)->hsw.idx);
 541
 542                 if (wait_for(intel_dkl_phy_read(dev_priv, DKL_CMN_UC_DW_27(tc_port)) &
 543                              DKL_CMN_UC_DW27_UC_HEALTH, 1))
 544                         drm_warn(display->drm,
 545                                  "Timeout waiting TC uC health\n");
 546         }
 547 }
 548
 549 static void
 550 icl_aux_power_well_enable(struct intel_display *display,
 551                           struct i915_power_well *power_well)
 552 {
 553         struct drm_i915_private *dev_priv = to_i915(display->drm);
 554         enum phy phy = icl_aux_pw_to_phy(display, power_well);
 555
 556         if (intel_phy_is_tc(dev_priv, phy))
 557                 return icl_tc_phy_aux_power_well_enable(display, power_well);
 558         else if (display->platform.icelake)
 559                 return icl_combo_phy_aux_power_well_enable(display,
 560                                                            power_well);
 561         else
 562                 return hsw_power_well_enable(display, power_well);
 563 }
 564
 565 static void
 566 icl_aux_power_well_disable(struct intel_display *display,
 567                            struct i915_power_well *power_well)
 568 {
 569         struct drm_i915_private *dev_priv = to_i915(display->drm);
 570         enum phy phy = icl_aux_pw_to_phy(display, power_well);
 571
 572         if (intel_phy_is_tc(dev_priv, phy))
 573                 return hsw_power_well_disable(display, power_well);
 574         else if (display->platform.icelake)
 575                 return icl_combo_phy_aux_power_well_disable(display,
 576                                                             power_well);
 577         else
 578                 return hsw_power_well_disable(display, power_well);
 579 }
 580
 581 /*
 582  * We should only use the power well if we explicitly asked the hardware to
 583  * enable it, so check if it's enabled and also check if we've requested it to
 584  * be enabled.
 585  */
 586 static bool hsw_power_well_enabled(struct intel_display *display,
 587                                    struct i915_power_well *power_well)
 588 {
 589         const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 590         enum i915_power_well_id id = i915_power_well_instance(power_well)->id;
 591         int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 592         u32 mask = HSW_PWR_WELL_CTL_REQ(pw_idx) |
 593                    HSW_PWR_WELL_CTL_STATE(pw_idx);
 594         u32 val;
 595
 596         val = intel_de_read(display, regs->driver);
 597
 598         /*
 599          * On GEN9 big core due to a DMC bug the driver's request bits for PW1
 600          * and the MISC_IO PW will be not restored, so check instead for the
 601          * BIOS's own request bits, which are forced-on for these power wells
 602          * when exiting DC5/6.
 603          */
 604         if (DISPLAY_VER(display) == 9 && !display->platform.broxton &&
 605             (id == SKL_DISP_PW_1 || id == SKL_DISP_PW_MISC_IO))
 606                 val |= intel_de_read(display, regs->bios);
 607
 608         return (val & mask) == mask;
 609 }
 610
 611 static void assert_can_enable_dc9(struct intel_display *display)
 612 {
 613         struct drm_i915_private *dev_priv = to_i915(display->drm);
 614
 615         drm_WARN_ONCE(display->drm,
 616                       (intel_de_read(display, DC_STATE_EN) & DC_STATE_EN_DC9),
 617                       "DC9 already programmed to be enabled.\n");
 618         drm_WARN_ONCE(display->drm,
 619                       intel_de_read(display, DC_STATE_EN) &
 620                       DC_STATE_EN_UPTO_DC5,
 621                       "DC5 still not disabled to enable DC9.\n");
 622         drm_WARN_ONCE(display->drm,
 623                       intel_de_read(display, HSW_PWR_WELL_CTL2) &
 624                       HSW_PWR_WELL_CTL_REQ(SKL_PW_CTL_IDX_PW_2),
 625                       "Power well 2 on.\n");
 626         drm_WARN_ONCE(display->drm, intel_irqs_enabled(dev_priv),
 627                       "Interrupts not disabled yet.\n");
 628
 629          /*
 630           * TODO: check for the following to verify the conditions to enter DC9
 631           * state are satisfied:
 632           * 1] Check relevant display engine registers to verify if mode set
 633           * disable sequence was followed.
 634           * 2] Check if display uninitialize sequence is initialized.
 635           */
 636 }
 637
 638 static void assert_can_disable_dc9(struct intel_display *display)
 639 {
 640         struct drm_i915_private *dev_priv = to_i915(display->drm);
 641
 642         drm_WARN_ONCE(display->drm, intel_irqs_enabled(dev_priv),
 643                       "Interrupts not disabled yet.\n");
 644         drm_WARN_ONCE(display->drm,
 645                       intel_de_read(display, DC_STATE_EN) &
 646                       DC_STATE_EN_UPTO_DC5,
 647                       "DC5 still not disabled.\n");
 648
 649          /*
 650           * TODO: check for the following to verify DC9 state was indeed
 651           * entered before programming to disable it:
 652           * 1] Check relevant display engine registers to verify if mode
 653           *  set disable sequence was followed.
 654           * 2] Check if display uninitialize sequence is initialized.
 655           */
 656 }
 657
 658 static void gen9_write_dc_state(struct intel_display *display,
 659                                 u32 state)
 660 {
 661         int rewrites = 0;
 662         int rereads = 0;
 663         u32 v;
 664
 665         intel_de_write(display, DC_STATE_EN, state);
 666
 667         /* It has been observed that disabling the dc6 state sometimes
 668          * doesn't stick and dmc keeps returning old value. Make sure
 669          * the write really sticks enough times and also force rewrite until
 670          * we are confident that state is exactly what we want.
 671          */
 672         do  {
 673                 v = intel_de_read(display, DC_STATE_EN);
 674
 675                 if (v != state) {
 676                         intel_de_write(display, DC_STATE_EN, state);
 677                         rewrites++;
 678                         rereads = 0;
 679                 } else if (rereads++ > 5) {
 680                         break;
 681                 }
 682
 683         } while (rewrites < 100);
 684
 685         if (v != state)
 686                 drm_err(display->drm,
 687                         "Writing dc state to 0x%x failed, now 0x%x\n",
 688                         state, v);
 689
 690         /* Most of the times we need one retry, avoid spam */
 691         if (rewrites > 1)
 692                 drm_dbg_kms(display->drm,
 693                             "Rewrote dc state to 0x%x %d times\n",
 694                             state, rewrites);
 695 }
 696
 697 static u32 gen9_dc_mask(struct intel_display *display)
 698 {
 699         u32 mask;
 700
 701         mask = DC_STATE_EN_UPTO_DC5;
 702
 703         if (DISPLAY_VER(display) >= 12)
 704                 mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6
 705                                           | DC_STATE_EN_DC9;
 706         else if (DISPLAY_VER(display) == 11)
 707                 mask |= DC_STATE_EN_UPTO_DC6 | DC_STATE_EN_DC9;
 708         else if (display->platform.geminilake || display->platform.broxton)
 709                 mask |= DC_STATE_EN_DC9;
 710         else
 711                 mask |= DC_STATE_EN_UPTO_DC6;
 712
 713         return mask;
 714 }
 715
 716 void gen9_sanitize_dc_state(struct intel_display *display)
 717 {
 718         struct i915_power_domains *power_domains = &display->power.domains;
 719         u32 val;
 720
 721         if (!HAS_DISPLAY(display))
 722                 return;
 723
 724         val = intel_de_read(display, DC_STATE_EN) & gen9_dc_mask(display);
 725
 726         drm_dbg_kms(display->drm,
 727                     "Resetting DC state tracking from %02x to %02x\n",
 728                     power_domains->dc_state, val);
 729         power_domains->dc_state = val;
 730 }
 731
 732 /**
 733  * gen9_set_dc_state - set target display C power state
 734  * @display: display instance
 735  * @state: target DC power state
 736  * - DC_STATE_DISABLE
 737  * - DC_STATE_EN_UPTO_DC5
 738  * - DC_STATE_EN_UPTO_DC6
 739  * - DC_STATE_EN_DC9
 740  *
 741  * Signal to DMC firmware/HW the target DC power state passed in @state.
 742  * DMC/HW can turn off individual display clocks and power rails when entering
 743  * a deeper DC power state (higher in number) and turns these back when exiting
 744  * that state to a shallower power state (lower in number). The HW will decide
 745  * when to actually enter a given state on an on-demand basis, for instance
 746  * depending on the active state of display pipes. The state of display
 747  * registers backed by affected power rails are saved/restored as needed.
 748  *
 749  * Based on the above enabling a deeper DC power state is asynchronous wrt.
 750  * enabling it. Disabling a deeper power state is synchronous: for instance
 751  * setting %DC_STATE_DISABLE won't complete until all HW resources are turned
 752  * back on and register state is restored. This is guaranteed by the MMIO write
 753  * to DC_STATE_EN blocking until the state is restored.
 754  */
 755 void gen9_set_dc_state(struct intel_display *display, u32 state)
 756 {
 757         struct i915_power_domains *power_domains = &display->power.domains;
 758         u32 val;
 759         u32 mask;
 760
 761         if (!HAS_DISPLAY(display))
 762                 return;
 763
 764         if (drm_WARN_ON_ONCE(display->drm,
 765                              state & ~power_domains->allowed_dc_mask))
 766                 state &= power_domains->allowed_dc_mask;
 767
 768         val = intel_de_read(display, DC_STATE_EN);
 769         mask = gen9_dc_mask(display);
 770         drm_dbg_kms(display->drm, "Setting DC state from %02x to %02x\n",
 771                     val & mask, state);
 772
 773         /* Check if DMC is ignoring our DC state requests */
 774         if ((val & mask) != power_domains->dc_state)
 775                 drm_err(display->drm, "DC state mismatch (0x%x -> 0x%x)\n",
 776                         power_domains->dc_state, val & mask);
 777
 778         val &= ~mask;
 779         val |= state;
 780
 781         gen9_write_dc_state(display, val);
 782
 783         power_domains->dc_state = val & mask;
 784 }
 785
 786 static void tgl_enable_dc3co(struct intel_display *display)
 787 {
 788         drm_dbg_kms(display->drm, "Enabling DC3CO\n");
 789         gen9_set_dc_state(display, DC_STATE_EN_DC3CO);
 790 }
 791
 792 static void tgl_disable_dc3co(struct intel_display *display)
 793 {
 794         drm_dbg_kms(display->drm, "Disabling DC3CO\n");
 795         intel_de_rmw(display, DC_STATE_EN, DC_STATE_DC3CO_STATUS, 0);
 796         gen9_set_dc_state(display, DC_STATE_DISABLE);
 797         /*
 798          * Delay of 200us DC3CO Exit time B.Spec 49196
 799          */
 800         usleep_range(200, 210);
 801 }
 802
 803 static void assert_can_enable_dc5(struct intel_display *display)
 804 {
 805         struct drm_i915_private __maybe_unused *dev_priv = to_i915(display->drm);
 806         enum i915_power_well_id high_pg;
 807
 808         /* Power wells at this level and above must be disabled for DC5 entry */
 809         if (DISPLAY_VER(display) == 12)
 810                 high_pg = ICL_DISP_PW_3;
 811         else
 812                 high_pg = SKL_DISP_PW_2;
 813
 814         drm_WARN_ONCE(display->drm,
 815                       intel_display_power_well_is_enabled(display, high_pg),
 816                       "Power wells above platform's DC5 limit still enabled.\n");
 817
 818         drm_WARN_ONCE(display->drm,
 819                       (intel_de_read(display, DC_STATE_EN) &
 820                        DC_STATE_EN_UPTO_DC5),
 821                       "DC5 already programmed to be enabled.\n");
 822         assert_rpm_wakelock_held(&dev_priv->runtime_pm);
 823
 824         assert_dmc_loaded(display);
 825 }
 826
 827 void gen9_enable_dc5(struct intel_display *display)
 828 {
 829         assert_can_enable_dc5(display);
 830
 831         drm_dbg_kms(display->drm, "Enabling DC5\n");
 832
 833         /* Wa Display #1183: skl,kbl,cfl */
 834         if (DISPLAY_VER(display) == 9 && !display->platform.broxton)
 835                 intel_de_rmw(display, GEN8_CHICKEN_DCPR_1,
 836                              0, SKL_SELECT_ALTERNATE_DC_EXIT);
 837
 838         intel_dmc_wl_enable(display, DC_STATE_EN_UPTO_DC5);
 839
 840         gen9_set_dc_state(display, DC_STATE_EN_UPTO_DC5);
 841 }
 842
 843 static void assert_can_enable_dc6(struct intel_display *display)
 844 {
 845         drm_WARN_ONCE(display->drm,
 846                       (intel_de_read(display, UTIL_PIN_CTL) &
 847                        (UTIL_PIN_ENABLE | UTIL_PIN_MODE_MASK)) ==
 848                       (UTIL_PIN_ENABLE | UTIL_PIN_MODE_PWM),
 849                       "Utility pin enabled in PWM mode\n");
 850         drm_WARN_ONCE(display->drm,
 851                       (intel_de_read(display, DC_STATE_EN) &
 852                        DC_STATE_EN_UPTO_DC6),
 853                       "DC6 already programmed to be enabled.\n");
 854
 855         assert_dmc_loaded(display);
 856 }
 857
 858 void skl_enable_dc6(struct intel_display *display)
 859 {
 860         assert_can_enable_dc6(display);
 861
 862         drm_dbg_kms(display->drm, "Enabling DC6\n");
 863
 864         /* Wa Display #1183: skl,kbl,cfl */
 865         if (DISPLAY_VER(display) == 9 && !display->platform.broxton)
 866                 intel_de_rmw(display, GEN8_CHICKEN_DCPR_1,
 867                              0, SKL_SELECT_ALTERNATE_DC_EXIT);
 868
 869         intel_dmc_wl_enable(display, DC_STATE_EN_UPTO_DC6);
 870
 871         gen9_set_dc_state(display, DC_STATE_EN_UPTO_DC6);
 872 }
 873
 874 void bxt_enable_dc9(struct intel_display *display)
 875 {
 876         assert_can_enable_dc9(display);
 877
 878         drm_dbg_kms(display->drm, "Enabling DC9\n");
 879         /*
 880          * Power sequencer reset is needed on BXT/GLK, because the PPS registers
 881          * aren't always on, unlike with South Display Engine on PCH.
 882          */
 883         if (display->platform.broxton || display->platform.geminilake)
 884                 bxt_pps_reset_all(display);
 885         gen9_set_dc_state(display, DC_STATE_EN_DC9);
 886 }
 887
 888 void bxt_disable_dc9(struct intel_display *display)
 889 {
 890         assert_can_disable_dc9(display);
 891
 892         drm_dbg_kms(display->drm, "Disabling DC9\n");
 893
 894         gen9_set_dc_state(display, DC_STATE_DISABLE);
 895
 896         intel_pps_unlock_regs_wa(display);
 897 }
 898
 899 static void hsw_power_well_sync_hw(struct intel_display *display,
 900                                    struct i915_power_well *power_well)
 901 {
 902         const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 903         int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
 904         u32 mask = HSW_PWR_WELL_CTL_REQ(pw_idx);
 905         u32 bios_req = intel_de_read(display, regs->bios);
 906
 907         /* Take over the request bit if set by BIOS. */
 908         if (bios_req & mask) {
 909                 u32 drv_req = intel_de_read(display, regs->driver);
 910
 911                 if (!(drv_req & mask))
 912                         intel_de_write(display, regs->driver, drv_req | mask);
 913                 intel_de_write(display, regs->bios, bios_req & ~mask);
 914         }
 915 }
 916
 917 static void bxt_dpio_cmn_power_well_enable(struct intel_display *display,
 918                                            struct i915_power_well *power_well)
 919 {
 920         bxt_dpio_phy_init(display, i915_power_well_instance(power_well)->bxt.phy);
 921 }
 922
 923 static void bxt_dpio_cmn_power_well_disable(struct intel_display *display,
 924                                             struct i915_power_well *power_well)
 925 {
 926         bxt_dpio_phy_uninit(display, i915_power_well_instance(power_well)->bxt.phy);
 927 }
 928
 929 static bool bxt_dpio_cmn_power_well_enabled(struct intel_display *display,
 930                                             struct i915_power_well *power_well)
 931 {
 932         return bxt_dpio_phy_is_enabled(display, i915_power_well_instance(power_well)->bxt.phy);
 933 }
 934
 935 static void bxt_verify_dpio_phy_power_wells(struct intel_display *display)
 936 {
 937         struct i915_power_well *power_well;
 938
 939         power_well = lookup_power_well(display, BXT_DISP_PW_DPIO_CMN_A);
 940         if (intel_power_well_refcount(power_well) > 0)
 941                 bxt_dpio_phy_verify_state(display, i915_power_well_instance(power_well)->bxt.phy);
 942
 943         power_well = lookup_power_well(display, VLV_DISP_PW_DPIO_CMN_BC);
 944         if (intel_power_well_refcount(power_well) > 0)
 945                 bxt_dpio_phy_verify_state(display, i915_power_well_instance(power_well)->bxt.phy);
 946
 947         if (display->platform.geminilake) {
 948                 power_well = lookup_power_well(display,
 949                                                GLK_DISP_PW_DPIO_CMN_C);
 950                 if (intel_power_well_refcount(power_well) > 0)
 951                         bxt_dpio_phy_verify_state(display,
 952                                                   i915_power_well_instance(power_well)->bxt.phy);
 953         }
 954 }
 955
 956 static bool gen9_dc_off_power_well_enabled(struct intel_display *display,
 957                                            struct i915_power_well *power_well)
 958 {
 959         return ((intel_de_read(display, DC_STATE_EN) & DC_STATE_EN_DC3CO) == 0 &&
 960                 (intel_de_read(display, DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0);
 961 }
 962
 963 static void gen9_assert_dbuf_enabled(struct intel_display *display)
 964 {
 965         struct drm_i915_private *dev_priv = to_i915(display->drm);
 966         u8 hw_enabled_dbuf_slices = intel_enabled_dbuf_slices_mask(dev_priv);
 967         u8 enabled_dbuf_slices = display->dbuf.enabled_slices;
 968
 969         drm_WARN(display->drm,
 970                  hw_enabled_dbuf_slices != enabled_dbuf_slices,
 971                  "Unexpected DBuf power power state (0x%08x, expected 0x%08x)\n",
 972                  hw_enabled_dbuf_slices,
 973                  enabled_dbuf_slices);
 974 }
 975
 976 void gen9_disable_dc_states(struct intel_display *display)
 977 {
 978         struct drm_i915_private *dev_priv = to_i915(display->drm);
 979         struct i915_power_domains *power_domains = &display->power.domains;
 980         struct intel_cdclk_config cdclk_config = {};
 981         u32 old_state = power_domains->dc_state;
 982
 983         if (power_domains->target_dc_state == DC_STATE_EN_DC3CO) {
 984                 tgl_disable_dc3co(display);
 985                 return;
 986         }
 987
 988         if (HAS_DISPLAY(display)) {
 989                 intel_dmc_wl_get_noreg(display);
 990                 gen9_set_dc_state(display, DC_STATE_DISABLE);
 991                 intel_dmc_wl_put_noreg(display);
 992         } else {
 993                 gen9_set_dc_state(display, DC_STATE_DISABLE);
 994                 return;
 995         }
 996
 997         if (old_state == DC_STATE_EN_UPTO_DC5 ||
 998             old_state == DC_STATE_EN_UPTO_DC6)
 999                 intel_dmc_wl_disable(display);
1000
1001         intel_cdclk_get_cdclk(display, &cdclk_config);
1002         /* Can't read out voltage_level so can't use intel_cdclk_changed() */
1003         drm_WARN_ON(display->drm,
1004                     intel_cdclk_clock_changed(&display->cdclk.hw,
1005                                               &cdclk_config));
1006
1007         gen9_assert_dbuf_enabled(display);
1008
1009         if (display->platform.geminilake || display->platform.broxton)
1010                 bxt_verify_dpio_phy_power_wells(display);
1011
1012         if (DISPLAY_VER(display) >= 11)
1013                 /*
1014                  * DMC retains HW context only for port A, the other combo
1015                  * PHY's HW context for port B is lost after DC transitions,
1016                  * so we need to restore it manually.
1017                  */
1018                 intel_combo_phy_init(dev_priv);
1019 }
1020
1021 static void gen9_dc_off_power_well_enable(struct intel_display *display,
1022                                           struct i915_power_well *power_well)
1023 {
1024         gen9_disable_dc_states(display);
1025 }
1026
1027 static void gen9_dc_off_power_well_disable(struct intel_display *display,
1028                                            struct i915_power_well *power_well)
1029 {
1030         struct i915_power_domains *power_domains = &display->power.domains;
1031
1032         if (!intel_dmc_has_payload(display))
1033                 return;
1034
1035         switch (power_domains->target_dc_state) {
1036         case DC_STATE_EN_DC3CO:
1037                 tgl_enable_dc3co(display);
1038                 break;
1039         case DC_STATE_EN_UPTO_DC6:
1040                 skl_enable_dc6(display);
1041                 break;
1042         case DC_STATE_EN_UPTO_DC5:
1043                 gen9_enable_dc5(display);
1044                 break;
1045         }
1046 }
1047
1048 static void i9xx_power_well_sync_hw_noop(struct intel_display *display,
1049                                          struct i915_power_well *power_well)
1050 {
1051 }
1052
1053 static void i9xx_always_on_power_well_noop(struct intel_display *display,
1054                                            struct i915_power_well *power_well)
1055 {
1056 }
1057
1058 static bool i9xx_always_on_power_well_enabled(struct intel_display *display,
1059                                               struct i915_power_well *power_well)
1060 {
1061         return true;
1062 }
1063
1064 static void i830_pipes_power_well_enable(struct intel_display *display,
1065                                          struct i915_power_well *power_well)
1066 {
1067         if ((intel_de_read(display, TRANSCONF(display, PIPE_A)) & TRANSCONF_ENABLE) == 0)
1068                 i830_enable_pipe(display, PIPE_A);
1069         if ((intel_de_read(display, TRANSCONF(display, PIPE_B)) & TRANSCONF_ENABLE) == 0)
1070                 i830_enable_pipe(display, PIPE_B);
1071 }
1072
1073 static void i830_pipes_power_well_disable(struct intel_display *display,
1074                                           struct i915_power_well *power_well)
1075 {
1076         i830_disable_pipe(display, PIPE_B);
1077         i830_disable_pipe(display, PIPE_A);
1078 }
1079
1080 static bool i830_pipes_power_well_enabled(struct intel_display *display,
1081                                           struct i915_power_well *power_well)
1082 {
1083         return intel_de_read(display, TRANSCONF(display, PIPE_A)) & TRANSCONF_ENABLE &&
1084                 intel_de_read(display, TRANSCONF(display, PIPE_B)) & TRANSCONF_ENABLE;
1085 }
1086
1087 static void i830_pipes_power_well_sync_hw(struct intel_display *display,
1088                                           struct i915_power_well *power_well)
1089 {
1090         if (intel_power_well_refcount(power_well) > 0)
1091                 i830_pipes_power_well_enable(display, power_well);
1092         else
1093                 i830_pipes_power_well_disable(display, power_well);
1094 }
1095
1096 static void vlv_set_power_well(struct intel_display *display,
1097                                struct i915_power_well *power_well, bool enable)
1098 {
1099         struct drm_i915_private *dev_priv = to_i915(display->drm);
1100         int pw_idx = i915_power_well_instance(power_well)->vlv.idx;
1101         u32 mask;
1102         u32 state;
1103         u32 ctrl;
1104
1105         mask = PUNIT_PWRGT_MASK(pw_idx);
1106         state = enable ? PUNIT_PWRGT_PWR_ON(pw_idx) :
1107                          PUNIT_PWRGT_PWR_GATE(pw_idx);
1108
1109         vlv_punit_get(dev_priv);
1110
1111 #define COND \
1112         ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
1113
1114         if (COND)
1115                 goto out;
1116
1117         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
1118         ctrl &= ~mask;
1119         ctrl |= state;
1120         vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
1121
1122         if (wait_for(COND, 100))
1123                 drm_err(display->drm,
1124                         "timeout setting power well state %08x (%08x)\n",
1125                         state,
1126                         vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
1127
1128 #undef COND
1129
1130 out:
1131         vlv_punit_put(dev_priv);
1132 }
1133
1134 static void vlv_power_well_enable(struct intel_display *display,
1135                                   struct i915_power_well *power_well)
1136 {
1137         vlv_set_power_well(display, power_well, true);
1138 }
1139
1140 static void vlv_power_well_disable(struct intel_display *display,
1141                                    struct i915_power_well *power_well)
1142 {
1143         vlv_set_power_well(display, power_well, false);
1144 }
1145
1146 static bool vlv_power_well_enabled(struct intel_display *display,
1147                                    struct i915_power_well *power_well)
1148 {
1149         struct drm_i915_private *dev_priv = to_i915(display->drm);
1150         int pw_idx = i915_power_well_instance(power_well)->vlv.idx;
1151         bool enabled = false;
1152         u32 mask;
1153         u32 state;
1154         u32 ctrl;
1155
1156         mask = PUNIT_PWRGT_MASK(pw_idx);
1157         ctrl = PUNIT_PWRGT_PWR_ON(pw_idx);
1158
1159         vlv_punit_get(dev_priv);
1160
1161         state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
1162         /*
1163          * We only ever set the power-on and power-gate states, anything
1164          * else is unexpected.
1165          */
1166         drm_WARN_ON(display->drm, state != PUNIT_PWRGT_PWR_ON(pw_idx) &&
1167                     state != PUNIT_PWRGT_PWR_GATE(pw_idx));
1168         if (state == ctrl)
1169                 enabled = true;
1170
1171         /*
1172          * A transient state at this point would mean some unexpected party
1173          * is poking at the power controls too.
1174          */
1175         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
1176         drm_WARN_ON(display->drm, ctrl != state);
1177
1178         vlv_punit_put(dev_priv);
1179
1180         return enabled;
1181 }
1182
1183 static void vlv_init_display_clock_gating(struct intel_display *display)
1184 {
1185         /*
1186          * On driver load, a pipe may be active and driving a DSI display.
1187          * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
1188          * (and never recovering) in this case. intel_dsi_post_disable() will
1189          * clear it when we turn off the display.
1190          */
1191         intel_de_rmw(display, DSPCLK_GATE_D(display),
1192                      ~DPOUNIT_CLOCK_GATE_DISABLE, VRHUNIT_CLOCK_GATE_DISABLE);
1193
1194         /*
1195          * Disable trickle feed and enable pnd deadline calculation
1196          */
1197         intel_de_write(display, MI_ARB_VLV,
1198                        MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
1199         intel_de_write(display, CBR1_VLV, 0);
1200
1201         drm_WARN_ON(display->drm, DISPLAY_RUNTIME_INFO(display)->rawclk_freq == 0);
1202         intel_de_write(display, RAWCLK_FREQ_VLV,
1203                        DIV_ROUND_CLOSEST(DISPLAY_RUNTIME_INFO(display)->rawclk_freq,
1204                                          1000));
1205 }
1206
1207 static void vlv_display_power_well_init(struct intel_display *display)
1208 {
1209         struct drm_i915_private *dev_priv = to_i915(display->drm);
1210         struct intel_encoder *encoder;
1211         enum pipe pipe;
1212
1213         /*
1214          * Enable the CRI clock source so we can get at the
1215          * display and the reference clock for VGA
1216          * hotplug / manual detection. Supposedly DSI also
1217          * needs the ref clock up and running.
1218          *
1219          * CHV DPLL B/C have some issues if VGA mode is enabled.
1220          */
1221         for_each_pipe(display, pipe) {
1222                 u32 val = intel_de_read(display, DPLL(display, pipe));
1223
1224                 val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1225                 if (pipe != PIPE_A)
1226                         val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1227
1228                 intel_de_write(display, DPLL(display, pipe), val);
1229         }
1230
1231         vlv_init_display_clock_gating(display);
1232
1233         spin_lock_irq(&dev_priv->irq_lock);
1234         valleyview_enable_display_irqs(dev_priv);
1235         spin_unlock_irq(&dev_priv->irq_lock);
1236
1237         /*
1238          * During driver initialization/resume we can avoid restoring the
1239          * part of the HW/SW state that will be inited anyway explicitly.
1240          */
1241         if (display->power.domains.initializing)
1242                 return;
1243
1244         intel_hpd_init(dev_priv);
1245         intel_hpd_poll_disable(dev_priv);
1246
1247         /* Re-enable the ADPA, if we have one */
1248         for_each_intel_encoder(display->drm, encoder) {
1249                 if (encoder->type == INTEL_OUTPUT_ANALOG)
1250                         intel_crt_reset(&encoder->base);
1251         }
1252
1253         intel_vga_redisable_power_on(display);
1254
1255         intel_pps_unlock_regs_wa(display);
1256 }
1257
1258 static void vlv_display_power_well_deinit(struct intel_display *display)
1259 {
1260         struct drm_i915_private *dev_priv = to_i915(display->drm);
1261
1262         spin_lock_irq(&dev_priv->irq_lock);
1263         valleyview_disable_display_irqs(dev_priv);
1264         spin_unlock_irq(&dev_priv->irq_lock);
1265
1266         /* make sure we're done processing display irqs */
1267         intel_synchronize_irq(dev_priv);
1268
1269         vlv_pps_reset_all(display);
1270
1271         /* Prevent us from re-enabling polling on accident in late suspend */
1272         if (!display->drm->dev->power.is_suspended)
1273                 intel_hpd_poll_enable(dev_priv);
1274 }
1275
1276 static void vlv_display_power_well_enable(struct intel_display *display,
1277                                           struct i915_power_well *power_well)
1278 {
1279         vlv_set_power_well(display, power_well, true);
1280
1281         vlv_display_power_well_init(display);
1282 }
1283
1284 static void vlv_display_power_well_disable(struct intel_display *display,
1285                                            struct i915_power_well *power_well)
1286 {
1287         vlv_display_power_well_deinit(display);
1288
1289         vlv_set_power_well(display, power_well, false);
1290 }
1291
1292 static void vlv_dpio_cmn_power_well_enable(struct intel_display *display,
1293                                            struct i915_power_well *power_well)
1294 {
1295         /* since ref/cri clock was enabled */
1296         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1297
1298         vlv_set_power_well(display, power_well, true);
1299
1300         /*
1301          * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
1302          *  6.  De-assert cmn_reset/side_reset. Same as VLV X0.
1303          *   a. GUnit 0x2110 bit[0] set to 1 (def 0)
1304          *   b. The other bits such as sfr settings / modesel may all
1305          *      be set to 0.
1306          *
1307          * This should only be done on init and resume from S3 with
1308          * both PLLs disabled, or we risk losing DPIO and PLL
1309          * synchronization.
1310          */
1311         intel_de_rmw(display, DPIO_CTL, 0, DPIO_CMNRST);
1312 }
1313
1314 static void vlv_dpio_cmn_power_well_disable(struct intel_display *display,
1315                                             struct i915_power_well *power_well)
1316 {
1317         struct drm_i915_private *dev_priv = to_i915(display->drm);
1318         enum pipe pipe;
1319
1320         for_each_pipe(display, pipe)
1321                 assert_pll_disabled(dev_priv, pipe);
1322
1323         /* Assert common reset */
1324         intel_de_rmw(display, DPIO_CTL, DPIO_CMNRST, 0);
1325
1326         vlv_set_power_well(display, power_well, false);
1327 }
1328
1329 #define BITS_SET(val, bits) (((val) & (bits)) == (bits))
1330
1331 static void assert_chv_phy_status(struct intel_display *display)
1332 {
1333         struct i915_power_well *cmn_bc =
1334                 lookup_power_well(display, VLV_DISP_PW_DPIO_CMN_BC);
1335         struct i915_power_well *cmn_d =
1336                 lookup_power_well(display, CHV_DISP_PW_DPIO_CMN_D);
1337         u32 phy_control = display->power.chv_phy_control;
1338         u32 phy_status = 0;
1339         u32 phy_status_mask = 0xffffffff;
1340
1341         /*
1342          * The BIOS can leave the PHY is some weird state
1343          * where it doesn't fully power down some parts.
1344          * Disable the asserts until the PHY has been fully
1345          * reset (ie. the power well has been disabled at
1346          * least once).
1347          */
1348         if (!display->power.chv_phy_assert[DPIO_PHY0])
1349                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
1350                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
1351                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
1352                                      PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) |
1353                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
1354                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));
1355
1356         if (!display->power.chv_phy_assert[DPIO_PHY1])
1357                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
1358                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
1359                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));
1360
1361         if (intel_power_well_is_enabled(display, cmn_bc)) {
1362                 phy_status |= PHY_POWERGOOD(DPIO_PHY0);
1363
1364                 /* this assumes override is only used to enable lanes */
1365                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0)
1366                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0);
1367
1368                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0)
1369                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1);
1370
1371                 /* CL1 is on whenever anything is on in either channel */
1372                 if (BITS_SET(phy_control,
1373                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) |
1374                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)))
1375                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0);
1376
1377                 /*
1378                  * The DPLLB check accounts for the pipe B + port A usage
1379                  * with CL2 powered up but all the lanes in the second channel
1380                  * powered down.
1381                  */
1382                 if (BITS_SET(phy_control,
1383                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) &&
1384                     (intel_de_read(display, DPLL(display, PIPE_B)) & DPLL_VCO_ENABLE) == 0)
1385                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1);
1386
1387                 if (BITS_SET(phy_control,
1388                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0)))
1389                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0);
1390                 if (BITS_SET(phy_control,
1391                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0)))
1392                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1);
1393
1394                 if (BITS_SET(phy_control,
1395                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1)))
1396                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0);
1397                 if (BITS_SET(phy_control,
1398                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1)))
1399                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1);
1400         }
1401
1402         if (intel_power_well_is_enabled(display, cmn_d)) {
1403                 phy_status |= PHY_POWERGOOD(DPIO_PHY1);
1404
1405                 /* this assumes override is only used to enable lanes */
1406                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0)
1407                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0);
1408
1409                 if (BITS_SET(phy_control,
1410                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0)))
1411                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0);
1412
1413                 if (BITS_SET(phy_control,
1414                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0)))
1415                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0);
1416                 if (BITS_SET(phy_control,
1417                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0)))
1418                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1);
1419         }
1420
1421         phy_status &= phy_status_mask;
1422
1423         /*
1424          * The PHY may be busy with some initial calibration and whatnot,
1425          * so the power state can take a while to actually change.
1426          */
1427         if (intel_de_wait(display, DISPLAY_PHY_STATUS,
1428                           phy_status_mask, phy_status, 10))
1429                 drm_err(display->drm,
1430                         "Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
1431                         intel_de_read(display, DISPLAY_PHY_STATUS) & phy_status_mask,
1432                         phy_status, display->power.chv_phy_control);
1433 }
1434
1435 #undef BITS_SET
1436
1437 static void chv_dpio_cmn_power_well_enable(struct intel_display *display,
1438                                            struct i915_power_well *power_well)
1439 {
1440         struct drm_i915_private *dev_priv = to_i915(display->drm);
1441         enum i915_power_well_id id = i915_power_well_instance(power_well)->id;
1442         enum dpio_phy phy;
1443         u32 tmp;
1444
1445         drm_WARN_ON_ONCE(display->drm,
1446                          id != VLV_DISP_PW_DPIO_CMN_BC &&
1447                          id != CHV_DISP_PW_DPIO_CMN_D);
1448
1449         if (id == VLV_DISP_PW_DPIO_CMN_BC)
1450                 phy = DPIO_PHY0;
1451         else
1452                 phy = DPIO_PHY1;
1453
1454         /* since ref/cri clock was enabled */
1455         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1456         vlv_set_power_well(display, power_well, true);
1457
1458         /* Poll for phypwrgood signal */
1459         if (intel_de_wait_for_set(display, DISPLAY_PHY_STATUS,
1460                                   PHY_POWERGOOD(phy), 1))
1461                 drm_err(display->drm, "Display PHY %d is not power up\n",
1462                         phy);
1463
1464         vlv_dpio_get(dev_priv);
1465
1466         /* Enable dynamic power down */
1467         tmp = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW28);
1468         tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
1469                 DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
1470         vlv_dpio_write(dev_priv, phy, CHV_CMN_DW28, tmp);
1471
1472         if (id == VLV_DISP_PW_DPIO_CMN_BC) {
1473                 tmp = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW6_CH1);
1474                 tmp |= DPIO_DYNPWRDOWNEN_CH1;
1475                 vlv_dpio_write(dev_priv, phy, CHV_CMN_DW6_CH1, tmp);
1476         } else {
1477                 /*
1478                  * Force the non-existing CL2 off. BXT does this
1479                  * too, so maybe it saves some power even though
1480                  * CL2 doesn't exist?
1481                  */
1482                 tmp = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW30);
1483                 tmp |= DPIO_CL2_LDOFUSE_PWRENB;
1484                 vlv_dpio_write(dev_priv, phy, CHV_CMN_DW30, tmp);
1485         }
1486
1487         vlv_dpio_put(dev_priv);
1488
1489         display->power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
1490         intel_de_write(display, DISPLAY_PHY_CONTROL,
1491                        display->power.chv_phy_control);
1492
1493         drm_dbg_kms(display->drm,
1494                     "Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1495                     phy, display->power.chv_phy_control);
1496
1497         assert_chv_phy_status(display);
1498 }
1499
1500 static void chv_dpio_cmn_power_well_disable(struct intel_display *display,
1501                                             struct i915_power_well *power_well)
1502 {
1503         struct drm_i915_private *dev_priv = to_i915(display->drm);
1504         enum i915_power_well_id id = i915_power_well_instance(power_well)->id;
1505         enum dpio_phy phy;
1506
1507         drm_WARN_ON_ONCE(display->drm,
1508                          id != VLV_DISP_PW_DPIO_CMN_BC &&
1509                          id != CHV_DISP_PW_DPIO_CMN_D);
1510
1511         if (id == VLV_DISP_PW_DPIO_CMN_BC) {
1512                 phy = DPIO_PHY0;
1513                 assert_pll_disabled(dev_priv, PIPE_A);
1514                 assert_pll_disabled(dev_priv, PIPE_B);
1515         } else {
1516                 phy = DPIO_PHY1;
1517                 assert_pll_disabled(dev_priv, PIPE_C);
1518         }
1519
1520         display->power.chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
1521         intel_de_write(display, DISPLAY_PHY_CONTROL,
1522                        display->power.chv_phy_control);
1523
1524         vlv_set_power_well(display, power_well, false);
1525
1526         drm_dbg_kms(display->drm,
1527                     "Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1528                     phy, display->power.chv_phy_control);
1529
1530         /* PHY is fully reset now, so we can enable the PHY state asserts */
1531         display->power.chv_phy_assert[phy] = true;
1532
1533         assert_chv_phy_status(display);
1534 }
1535
1536 static void assert_chv_phy_powergate(struct intel_display *display, enum dpio_phy phy,
1537                                      enum dpio_channel ch, bool override, unsigned int mask)
1538 {
1539         struct drm_i915_private *dev_priv = to_i915(display->drm);
1540         u32 reg, val, expected, actual;
1541
1542         /*
1543          * The BIOS can leave the PHY is some weird state
1544          * where it doesn't fully power down some parts.
1545          * Disable the asserts until the PHY has been fully
1546          * reset (ie. the power well has been disabled at
1547          * least once).
1548          */
1549         if (!display->power.chv_phy_assert[phy])
1550                 return;
1551
1552         if (ch == DPIO_CH0)
1553                 reg = CHV_CMN_DW0_CH0;
1554         else
1555                 reg = CHV_CMN_DW6_CH1;
1556
1557         vlv_dpio_get(dev_priv);
1558         val = vlv_dpio_read(dev_priv, phy, reg);
1559         vlv_dpio_put(dev_priv);
1560
1561         /*
1562          * This assumes !override is only used when the port is disabled.
1563          * All lanes should power down even without the override when
1564          * the port is disabled.
1565          */
1566         if (!override || mask == 0xf) {
1567                 expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1568                 /*
1569                  * If CH1 common lane is not active anymore
1570                  * (eg. for pipe B DPLL) the entire channel will
1571                  * shut down, which causes the common lane registers
1572                  * to read as 0. That means we can't actually check
1573                  * the lane power down status bits, but as the entire
1574                  * register reads as 0 it's a good indication that the
1575                  * channel is indeed entirely powered down.
1576                  */
1577                 if (ch == DPIO_CH1 && val == 0)
1578                         expected = 0;
1579         } else if (mask != 0x0) {
1580                 expected = DPIO_ANYDL_POWERDOWN;
1581         } else {
1582                 expected = 0;
1583         }
1584
1585         if (ch == DPIO_CH0)
1586                 actual = REG_FIELD_GET(DPIO_ANYDL_POWERDOWN_CH0 |
1587                                        DPIO_ALLDL_POWERDOWN_CH0, val);
1588         else
1589                 actual = REG_FIELD_GET(DPIO_ANYDL_POWERDOWN_CH1 |
1590                                        DPIO_ALLDL_POWERDOWN_CH1, val);
1591
1592         drm_WARN(display->drm, actual != expected,
1593                  "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
1594                  !!(actual & DPIO_ALLDL_POWERDOWN),
1595                  !!(actual & DPIO_ANYDL_POWERDOWN),
1596                  !!(expected & DPIO_ALLDL_POWERDOWN),
1597                  !!(expected & DPIO_ANYDL_POWERDOWN),
1598                  reg, val);
1599 }
1600
1601 bool chv_phy_powergate_ch(struct intel_display *display, enum dpio_phy phy,
1602                           enum dpio_channel ch, bool override)
1603 {
1604         struct i915_power_domains *power_domains = &display->power.domains;
1605         bool was_override;
1606
1607         mutex_lock(&power_domains->lock);
1608
1609         was_override = display->power.chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1610
1611         if (override == was_override)
1612                 goto out;
1613
1614         if (override)
1615                 display->power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1616         else
1617                 display->power.chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1618
1619         intel_de_write(display, DISPLAY_PHY_CONTROL,
1620                        display->power.chv_phy_control);
1621
1622         drm_dbg_kms(display->drm,
1623                     "Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
1624                     phy, ch, display->power.chv_phy_control);
1625
1626         assert_chv_phy_status(display);
1627
1628 out:
1629         mutex_unlock(&power_domains->lock);
1630
1631         return was_override;
1632 }
1633
1634 void chv_phy_powergate_lanes(struct intel_encoder *encoder,
1635                              bool override, unsigned int mask)
1636 {
1637         struct intel_display *display = to_intel_display(encoder);
1638         struct i915_power_domains *power_domains = &display->power.domains;
1639         enum dpio_phy phy = vlv_dig_port_to_phy(enc_to_dig_port(encoder));
1640         enum dpio_channel ch = vlv_dig_port_to_channel(enc_to_dig_port(encoder));
1641
1642         mutex_lock(&power_domains->lock);
1643
1644         display->power.chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
1645         display->power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
1646
1647         if (override)
1648                 display->power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1649         else
1650                 display->power.chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1651
1652         intel_de_write(display, DISPLAY_PHY_CONTROL,
1653                        display->power.chv_phy_control);
1654
1655         drm_dbg_kms(display->drm,
1656                     "Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
1657                     phy, ch, mask, display->power.chv_phy_control);
1658
1659         assert_chv_phy_status(display);
1660
1661         assert_chv_phy_powergate(display, phy, ch, override, mask);
1662
1663         mutex_unlock(&power_domains->lock);
1664 }
1665
1666 static bool chv_pipe_power_well_enabled(struct intel_display *display,
1667                                         struct i915_power_well *power_well)
1668 {
1669         struct drm_i915_private *dev_priv = to_i915(display->drm);
1670         enum pipe pipe = PIPE_A;
1671         bool enabled;
1672         u32 state, ctrl;
1673
1674         vlv_punit_get(dev_priv);
1675
1676         state = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSS_MASK(pipe);
1677         /*
1678          * We only ever set the power-on and power-gate states, anything
1679          * else is unexpected.
1680          */
1681         drm_WARN_ON(display->drm, state != DP_SSS_PWR_ON(pipe) &&
1682                     state != DP_SSS_PWR_GATE(pipe));
1683         enabled = state == DP_SSS_PWR_ON(pipe);
1684
1685         /*
1686          * A transient state at this point would mean some unexpected party
1687          * is poking at the power controls too.
1688          */
1689         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSC_MASK(pipe);
1690         drm_WARN_ON(display->drm, ctrl << 16 != state);
1691
1692         vlv_punit_put(dev_priv);
1693
1694         return enabled;
1695 }
1696
1697 static void chv_set_pipe_power_well(struct intel_display *display,
1698                                     struct i915_power_well *power_well,
1699                                     bool enable)
1700 {
1701         struct drm_i915_private *dev_priv = to_i915(display->drm);
1702         enum pipe pipe = PIPE_A;
1703         u32 state;
1704         u32 ctrl;
1705
1706         state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
1707
1708         vlv_punit_get(dev_priv);
1709
1710 #define COND \
1711         ((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSS_MASK(pipe)) == state)
1712
1713         if (COND)
1714                 goto out;
1715
1716         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
1717         ctrl &= ~DP_SSC_MASK(pipe);
1718         ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
1719         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, ctrl);
1720
1721         if (wait_for(COND, 100))
1722                 drm_err(display->drm,
1723                         "timeout setting power well state %08x (%08x)\n",
1724                         state,
1725                         vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM));
1726
1727 #undef COND
1728
1729 out:
1730         vlv_punit_put(dev_priv);
1731 }
1732
1733 static void chv_pipe_power_well_sync_hw(struct intel_display *display,
1734                                         struct i915_power_well *power_well)
1735 {
1736         intel_de_write(display, DISPLAY_PHY_CONTROL,
1737                        display->power.chv_phy_control);
1738 }
1739
1740 static void chv_pipe_power_well_enable(struct intel_display *display,
1741                                        struct i915_power_well *power_well)
1742 {
1743         chv_set_pipe_power_well(display, power_well, true);
1744
1745         vlv_display_power_well_init(display);
1746 }
1747
1748 static void chv_pipe_power_well_disable(struct intel_display *display,
1749                                         struct i915_power_well *power_well)
1750 {
1751         vlv_display_power_well_deinit(display);
1752
1753         chv_set_pipe_power_well(display, power_well, false);
1754 }
1755
1756 static void
1757 tgl_tc_cold_request(struct intel_display *display, bool block)
1758 {
1759         struct drm_i915_private *i915 = to_i915(display->drm);
1760         u8 tries = 0;
1761         int ret;
1762
1763         while (1) {
1764                 u32 low_val;
1765                 u32 high_val = 0;
1766
1767                 if (block)
1768                         low_val = TGL_PCODE_EXIT_TCCOLD_DATA_L_BLOCK_REQ;
1769                 else
1770                         low_val = TGL_PCODE_EXIT_TCCOLD_DATA_L_UNBLOCK_REQ;
1771
1772                 /*
1773                  * Spec states that we should timeout the request after 200us
1774                  * but the function below will timeout after 500us
1775                  */
1776                 ret = snb_pcode_read(&i915->uncore, TGL_PCODE_TCCOLD, &low_val, &high_val);
1777                 if (ret == 0) {
1778                         if (block &&
1779                             (low_val & TGL_PCODE_EXIT_TCCOLD_DATA_L_EXIT_FAILED))
1780                                 ret = -EIO;
1781                         else
1782                                 break;
1783                 }
1784
1785                 if (++tries == 3)
1786                         break;
1787
1788                 msleep(1);
1789         }
1790
1791         if (ret)
1792                 drm_err(&i915->drm, "TC cold %sblock failed\n",
1793                         block ? "" : "un");
1794         else
1795                 drm_dbg_kms(&i915->drm, "TC cold %sblock succeeded\n",
1796                             block ? "" : "un");
1797 }
1798
1799 static void
1800 tgl_tc_cold_off_power_well_enable(struct intel_display *display,
1801                                   struct i915_power_well *power_well)
1802 {
1803         tgl_tc_cold_request(display, true);
1804 }
1805
1806 static void
1807 tgl_tc_cold_off_power_well_disable(struct intel_display *display,
1808                                    struct i915_power_well *power_well)
1809 {
1810         tgl_tc_cold_request(display, false);
1811 }
1812
1813 static void
1814 tgl_tc_cold_off_power_well_sync_hw(struct intel_display *display,
1815                                    struct i915_power_well *power_well)
1816 {
1817         if (intel_power_well_refcount(power_well) > 0)
1818                 tgl_tc_cold_off_power_well_enable(display, power_well);
1819         else
1820                 tgl_tc_cold_off_power_well_disable(display, power_well);
1821 }
1822
1823 static bool
1824 tgl_tc_cold_off_power_well_is_enabled(struct intel_display *display,
1825                                       struct i915_power_well *power_well)
1826 {
1827         /*
1828          * Not the correctly implementation but there is no way to just read it
1829          * from PCODE, so returning count to avoid state mismatch errors
1830          */
1831         return intel_power_well_refcount(power_well);
1832 }
1833
1834 static void xelpdp_aux_power_well_enable(struct intel_display *display,
1835                                          struct i915_power_well *power_well)
1836 {
1837         struct drm_i915_private *dev_priv = to_i915(display->drm);
1838         enum aux_ch aux_ch = i915_power_well_instance(power_well)->xelpdp.aux_ch;
1839         enum phy phy = icl_aux_pw_to_phy(display, power_well);
1840
1841         if (intel_phy_is_tc(dev_priv, phy))
1842                 icl_tc_port_assert_ref_held(display, power_well,
1843                                             aux_ch_to_digital_port(display, aux_ch));
1844
1845         intel_de_rmw(display, XELPDP_DP_AUX_CH_CTL(display, aux_ch),
1846                      XELPDP_DP_AUX_CH_CTL_POWER_REQUEST,
1847                      XELPDP_DP_AUX_CH_CTL_POWER_REQUEST);
1848
1849         /*
1850          * The power status flag cannot be used to determine whether aux
1851          * power wells have finished powering up.  Instead we're
1852          * expected to just wait a fixed 600us after raising the request
1853          * bit.
1854          */
1855         usleep_range(600, 1200);
1856 }
1857
1858 static void xelpdp_aux_power_well_disable(struct intel_display *display,
1859                                           struct i915_power_well *power_well)
1860 {
1861         enum aux_ch aux_ch = i915_power_well_instance(power_well)->xelpdp.aux_ch;
1862
1863         intel_de_rmw(display, XELPDP_DP_AUX_CH_CTL(display, aux_ch),
1864                      XELPDP_DP_AUX_CH_CTL_POWER_REQUEST,
1865                      0);
1866         usleep_range(10, 30);
1867 }
1868
1869 static bool xelpdp_aux_power_well_enabled(struct intel_display *display,
1870                                           struct i915_power_well *power_well)
1871 {
1872         enum aux_ch aux_ch = i915_power_well_instance(power_well)->xelpdp.aux_ch;
1873
1874         return intel_de_read(display, XELPDP_DP_AUX_CH_CTL(display, aux_ch)) &
1875                 XELPDP_DP_AUX_CH_CTL_POWER_STATUS;
1876 }
1877
1878 static void xe2lpd_pica_power_well_enable(struct intel_display *display,
1879                                           struct i915_power_well *power_well)
1880 {
1881         intel_de_write(display, XE2LPD_PICA_PW_CTL,
1882                        XE2LPD_PICA_CTL_POWER_REQUEST);
1883
1884         if (intel_de_wait_for_set(display, XE2LPD_PICA_PW_CTL,
1885                                   XE2LPD_PICA_CTL_POWER_STATUS, 1)) {
1886                 drm_dbg_kms(display->drm, "pica power well enable timeout\n");
1887
1888                 drm_WARN(display->drm, 1, "Power well PICA timeout when enabled");
1889         }
1890 }
1891
1892 static void xe2lpd_pica_power_well_disable(struct intel_display *display,
1893                                            struct i915_power_well *power_well)
1894 {
1895         intel_de_write(display, XE2LPD_PICA_PW_CTL, 0);
1896
1897         if (intel_de_wait_for_clear(display, XE2LPD_PICA_PW_CTL,
1898                                     XE2LPD_PICA_CTL_POWER_STATUS, 1)) {
1899                 drm_dbg_kms(display->drm, "pica power well disable timeout\n");
1900
1901                 drm_WARN(display->drm, 1, "Power well PICA timeout when disabled");
1902         }
1903 }
1904
1905 static bool xe2lpd_pica_power_well_enabled(struct intel_display *display,
1906                                            struct i915_power_well *power_well)
1907 {
1908         return intel_de_read(display, XE2LPD_PICA_PW_CTL) &
1909                 XE2LPD_PICA_CTL_POWER_STATUS;
1910 }
1911
1912 const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
1913         .sync_hw = i9xx_power_well_sync_hw_noop,
1914         .enable = i9xx_always_on_power_well_noop,
1915         .disable = i9xx_always_on_power_well_noop,
1916         .is_enabled = i9xx_always_on_power_well_enabled,
1917 };
1918
1919 const struct i915_power_well_ops chv_pipe_power_well_ops = {
1920         .sync_hw = chv_pipe_power_well_sync_hw,
1921         .enable = chv_pipe_power_well_enable,
1922         .disable = chv_pipe_power_well_disable,
1923         .is_enabled = chv_pipe_power_well_enabled,
1924 };
1925
1926 const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
1927         .sync_hw = i9xx_power_well_sync_hw_noop,
1928         .enable = chv_dpio_cmn_power_well_enable,
1929         .disable = chv_dpio_cmn_power_well_disable,
1930         .is_enabled = vlv_power_well_enabled,
1931 };
1932
1933 const struct i915_power_well_ops i830_pipes_power_well_ops = {
1934         .sync_hw = i830_pipes_power_well_sync_hw,
1935         .enable = i830_pipes_power_well_enable,
1936         .disable = i830_pipes_power_well_disable,
1937         .is_enabled = i830_pipes_power_well_enabled,
1938 };
1939
1940 static const struct i915_power_well_regs hsw_power_well_regs = {
1941         .bios   = HSW_PWR_WELL_CTL1,
1942         .driver = HSW_PWR_WELL_CTL2,
1943         .kvmr   = HSW_PWR_WELL_CTL3,
1944         .debug  = HSW_PWR_WELL_CTL4,
1945 };
1946
1947 const struct i915_power_well_ops hsw_power_well_ops = {
1948         .regs = &hsw_power_well_regs,
1949         .sync_hw = hsw_power_well_sync_hw,
1950         .enable = hsw_power_well_enable,
1951         .disable = hsw_power_well_disable,
1952         .is_enabled = hsw_power_well_enabled,
1953 };
1954
1955 const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
1956         .sync_hw = i9xx_power_well_sync_hw_noop,
1957         .enable = gen9_dc_off_power_well_enable,
1958         .disable = gen9_dc_off_power_well_disable,
1959         .is_enabled = gen9_dc_off_power_well_enabled,
1960 };
1961
1962 const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = {
1963         .sync_hw = i9xx_power_well_sync_hw_noop,
1964         .enable = bxt_dpio_cmn_power_well_enable,
1965         .disable = bxt_dpio_cmn_power_well_disable,
1966         .is_enabled = bxt_dpio_cmn_power_well_enabled,
1967 };
1968
1969 const struct i915_power_well_ops vlv_display_power_well_ops = {
1970         .sync_hw = i9xx_power_well_sync_hw_noop,
1971         .enable = vlv_display_power_well_enable,
1972         .disable = vlv_display_power_well_disable,
1973         .is_enabled = vlv_power_well_enabled,
1974 };
1975
1976 const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
1977         .sync_hw = i9xx_power_well_sync_hw_noop,
1978         .enable = vlv_dpio_cmn_power_well_enable,
1979         .disable = vlv_dpio_cmn_power_well_disable,
1980         .is_enabled = vlv_power_well_enabled,
1981 };
1982
1983 const struct i915_power_well_ops vlv_dpio_power_well_ops = {
1984         .sync_hw = i9xx_power_well_sync_hw_noop,
1985         .enable = vlv_power_well_enable,
1986         .disable = vlv_power_well_disable,
1987         .is_enabled = vlv_power_well_enabled,
1988 };
1989
1990 static const struct i915_power_well_regs icl_aux_power_well_regs = {
1991         .bios   = ICL_PWR_WELL_CTL_AUX1,
1992         .driver = ICL_PWR_WELL_CTL_AUX2,
1993         .debug  = ICL_PWR_WELL_CTL_AUX4,
1994 };
1995
1996 const struct i915_power_well_ops icl_aux_power_well_ops = {
1997         .regs = &icl_aux_power_well_regs,
1998         .sync_hw = hsw_power_well_sync_hw,
1999         .enable = icl_aux_power_well_enable,
2000         .disable = icl_aux_power_well_disable,
2001         .is_enabled = hsw_power_well_enabled,
2002 };
2003
2004 static const struct i915_power_well_regs icl_ddi_power_well_regs = {
2005         .bios   = ICL_PWR_WELL_CTL_DDI1,
2006         .driver = ICL_PWR_WELL_CTL_DDI2,
2007         .debug  = ICL_PWR_WELL_CTL_DDI4,
2008 };
2009
2010 const struct i915_power_well_ops icl_ddi_power_well_ops = {
2011         .regs = &icl_ddi_power_well_regs,
2012         .sync_hw = hsw_power_well_sync_hw,
2013         .enable = hsw_power_well_enable,
2014         .disable = hsw_power_well_disable,
2015         .is_enabled = hsw_power_well_enabled,
2016 };
2017
2018 const struct i915_power_well_ops tgl_tc_cold_off_ops = {
2019         .sync_hw = tgl_tc_cold_off_power_well_sync_hw,
2020         .enable = tgl_tc_cold_off_power_well_enable,
2021         .disable = tgl_tc_cold_off_power_well_disable,
2022         .is_enabled = tgl_tc_cold_off_power_well_is_enabled,
2023 };
2024
2025 const struct i915_power_well_ops xelpdp_aux_power_well_ops = {
2026         .sync_hw = i9xx_power_well_sync_hw_noop,
2027         .enable = xelpdp_aux_power_well_enable,
2028         .disable = xelpdp_aux_power_well_disable,
2029         .is_enabled = xelpdp_aux_power_well_enabled,
2030 };
2031
2032 const struct i915_power_well_ops xe2lpd_pica_power_well_ops = {
2033         .sync_hw = i9xx_power_well_sync_hw_noop,
2034         .enable = xe2lpd_pica_power_well_enable,
2035         .disable = xe2lpd_pica_power_well_disable,
2036         .is_enabled = xe2lpd_pica_power_well_enabled,
2037 };