drivers/gpu/drm/i915/intel_clock_gating.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <[email protected]>
  25  *
  26  */
  27
  28 #include "display/intel_de.h"
  29 #include "display/intel_display.h"
  30 #include "display/intel_display_trace.h"
  31 #include "display/skl_watermark.h"
  32
  33 #include "gt/intel_engine_regs.h"
  34 #include "gt/intel_gt.h"
  35 #include "gt/intel_gt_mcr.h"
  36 #include "gt/intel_gt_regs.h"
  37
  38 #include "i915_drv.h"
  39 #include "i915_reg.h"
  40 #include "intel_clock_gating.h"
  41 #include "intel_mchbar_regs.h"
  42 #include "vlv_sideband.h"
  43
  44 struct drm_i915_clock_gating_funcs {
  45         void (*init_clock_gating)(struct drm_i915_private *i915);
  46 };
  47
  48 static void gen9_init_clock_gating(struct drm_i915_private *i915)
  49 {
  50         if (HAS_LLC(i915)) {
  51                 /*
  52                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  53                  * Display WA #0390: skl,kbl
  54                  *
  55                  * Must match Sampler, Pixel Back End, and Media. See
  56                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  57                  */
  58                 intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
  59         }
  60
  61         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  62         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
  63
  64         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  65         intel_uncore_rmw(&i915->uncore, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
  66
  67         /*
  68          * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
  69          * Display WA #0859: skl,bxt,kbl,glk,cfl
  70          */
  71         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
  72 }
  73
  74 static void bxt_init_clock_gating(struct drm_i915_private *i915)
  75 {
  76         gen9_init_clock_gating(i915);
  77
  78         /* WaDisableSDEUnitClockGating:bxt */
  79         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
  80
  81         /*
  82          * FIXME:
  83          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
  84          */
  85         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
  86
  87         /*
  88          * Wa: Backlight PWM may stop in the asserted state, causing backlight
  89          * to stay fully on.
  90          */
  91         intel_uncore_write(&i915->uncore, GEN9_CLKGATE_DIS_0,
  92                            intel_uncore_read(&i915->uncore, GEN9_CLKGATE_DIS_0) |
  93                            PWM1_GATING_DIS | PWM2_GATING_DIS);
  94
  95         /*
  96          * Lower the display internal timeout.
  97          * This is needed to avoid any hard hangs when DSI port PLL
  98          * is off and a MMIO access is attempted by any privilege
  99          * application, using batch buffers or any other means.
 100          */
 101         intel_uncore_write(&i915->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
 102
 103         /*
 104          * WaFbcTurnOffFbcWatermark:bxt
 105          * Display WA #0562: bxt
 106          */
 107         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 108
 109         /*
 110          * WaFbcHighMemBwCorruptionAvoidance:bxt
 111          * Display WA #0883: bxt
 112          */
 113         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
 114 }
 115
 116 static void glk_init_clock_gating(struct drm_i915_private *i915)
 117 {
 118         gen9_init_clock_gating(i915);
 119
 120         /*
 121          * WaDisablePWMClockGating:glk
 122          * Backlight PWM may stop in the asserted state, causing backlight
 123          * to stay fully on.
 124          */
 125         intel_uncore_write(&i915->uncore, GEN9_CLKGATE_DIS_0,
 126                            intel_uncore_read(&i915->uncore, GEN9_CLKGATE_DIS_0) |
 127                            PWM1_GATING_DIS | PWM2_GATING_DIS);
 128 }
 129
 130 static void ibx_init_clock_gating(struct drm_i915_private *i915)
 131 {
 132         /*
 133          * On Ibex Peak and Cougar Point, we need to disable clock
 134          * gating for the panel power sequencer or it will fail to
 135          * start up when no ports are active.
 136          */
 137         intel_uncore_write(&i915->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
 138 }
 139
 140 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
 141 {
 142         enum pipe pipe;
 143
 144         for_each_pipe(dev_priv, pipe) {
 145                 intel_uncore_rmw(&dev_priv->uncore, DSPCNTR(pipe), 0, DISP_TRICKLE_FEED_DISABLE);
 146
 147                 intel_uncore_rmw(&dev_priv->uncore, DSPSURF(pipe), 0, 0);
 148                 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
 149         }
 150 }
 151
 152 static void ilk_init_clock_gating(struct drm_i915_private *i915)
 153 {
 154         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 155
 156         /*
 157          * Required for FBC
 158          * WaFbcDisableDpfcClockGating:ilk
 159          */
 160         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
 161                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
 162                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
 163
 164         intel_uncore_write(&i915->uncore, PCH_3DCGDIS0,
 165                            MARIUNIT_CLOCK_GATE_DISABLE |
 166                            SVSMUNIT_CLOCK_GATE_DISABLE);
 167         intel_uncore_write(&i915->uncore, PCH_3DCGDIS1,
 168                            VFMUNIT_CLOCK_GATE_DISABLE);
 169
 170         /*
 171          * According to the spec the following bits should be set in
 172          * order to enable memory self-refresh
 173          * The bit 22/21 of 0x42004
 174          * The bit 5 of 0x42020
 175          * The bit 15 of 0x45000
 176          */
 177         intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN2,
 178                            (intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN2) |
 179                             ILK_DPARB_GATE | ILK_VSDPFD_FULL));
 180         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
 181         intel_uncore_write(&i915->uncore, DISP_ARB_CTL,
 182                            (intel_uncore_read(&i915->uncore, DISP_ARB_CTL) |
 183                             DISP_FBC_WM_DIS));
 184
 185         /*
 186          * Based on the document from hardware guys the following bits
 187          * should be set unconditionally in order to enable FBC.
 188          * The bit 22 of 0x42000
 189          * The bit 22 of 0x42004
 190          * The bit 7,8,9 of 0x42020.
 191          */
 192         if (IS_IRONLAKE_M(i915)) {
 193                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
 194                 intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
 195                 intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
 196         }
 197
 198         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
 199
 200         intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
 201
 202         g4x_disable_trickle_feed(i915);
 203
 204         ibx_init_clock_gating(i915);
 205 }
 206
 207 static void cpt_init_clock_gating(struct drm_i915_private *i915)
 208 {
 209         enum pipe pipe;
 210         u32 val;
 211
 212         /*
 213          * On Ibex Peak and Cougar Point, we need to disable clock
 214          * gating for the panel power sequencer or it will fail to
 215          * start up when no ports are active.
 216          */
 217         intel_uncore_write(&i915->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
 218                            PCH_DPLUNIT_CLOCK_GATE_DISABLE |
 219                            PCH_CPUNIT_CLOCK_GATE_DISABLE);
 220         intel_uncore_rmw(&i915->uncore, SOUTH_CHICKEN2, 0, DPLS_EDP_PPS_FIX_DIS);
 221         /* The below fixes the weird display corruption, a few pixels shifted
 222          * downward, on (only) LVDS of some HP laptops with IVY.
 223          */
 224         for_each_pipe(i915, pipe) {
 225                 val = intel_uncore_read(&i915->uncore, TRANS_CHICKEN2(pipe));
 226                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
 227                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
 228                 if (i915->display.vbt.fdi_rx_polarity_inverted)
 229                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
 230                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
 231                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
 232                 intel_uncore_write(&i915->uncore, TRANS_CHICKEN2(pipe), val);
 233         }
 234         /* WADP0ClockGatingDisable */
 235         for_each_pipe(i915, pipe) {
 236                 intel_uncore_write(&i915->uncore, TRANS_CHICKEN1(pipe),
 237                                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
 238         }
 239 }
 240
 241 static void gen6_check_mch_setup(struct drm_i915_private *i915)
 242 {
 243         u32 tmp;
 244
 245         tmp = intel_uncore_read(&i915->uncore, MCH_SSKPD);
 246         if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
 247                 drm_dbg_kms(&i915->drm,
 248                             "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
 249                             tmp);
 250 }
 251
 252 static void gen6_init_clock_gating(struct drm_i915_private *i915)
 253 {
 254         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 255
 256         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
 257
 258         intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
 259
 260         intel_uncore_write(&i915->uncore, GEN6_UCGCTL1,
 261                            intel_uncore_read(&i915->uncore, GEN6_UCGCTL1) |
 262                            GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
 263                            GEN6_CSUNIT_CLOCK_GATE_DISABLE);
 264
 265         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
 266          * gating disable must be set.  Failure to set it results in
 267          * flickering pixels due to Z write ordering failures after
 268          * some amount of runtime in the Mesa "fire" demo, and Unigine
 269          * Sanctuary and Tropics, and apparently anything else with
 270          * alpha test or pixel discard.
 271          *
 272          * According to the spec, bit 11 (RCCUNIT) must also be set,
 273          * but we didn't debug actual testcases to find it out.
 274          *
 275          * WaDisableRCCUnitClockGating:snb
 276          * WaDisableRCPBUnitClockGating:snb
 277          */
 278         intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
 279                            GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
 280                            GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
 281
 282         /*
 283          * According to the spec the following bits should be
 284          * set in order to enable memory self-refresh and fbc:
 285          * The bit21 and bit22 of 0x42000
 286          * The bit21 and bit22 of 0x42004
 287          * The bit5 and bit7 of 0x42020
 288          * The bit14 of 0x70180
 289          * The bit14 of 0x71180
 290          *
 291          * WaFbcAsynchFlipDisableFbcQueue:snb
 292          */
 293         intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN1,
 294                            intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN1) |
 295                            ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
 296         intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN2,
 297                            intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN2) |
 298                            ILK_DPARB_GATE | ILK_VSDPFD_FULL);
 299         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D,
 300                            intel_uncore_read(&i915->uncore, ILK_DSPCLK_GATE_D) |
 301                            ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
 302                            ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
 303
 304         g4x_disable_trickle_feed(i915);
 305
 306         cpt_init_clock_gating(i915);
 307
 308         gen6_check_mch_setup(i915);
 309 }
 310
 311 static void lpt_init_clock_gating(struct drm_i915_private *i915)
 312 {
 313         /*
 314          * TODO: this bit should only be enabled when really needed, then
 315          * disabled when not needed anymore in order to save power.
 316          */
 317         if (HAS_PCH_LPT_LP(i915))
 318                 intel_uncore_rmw(&i915->uncore, SOUTH_DSPCLK_GATE_D,
 319                                  0, PCH_LP_PARTITION_LEVEL_DISABLE);
 320
 321         /* WADPOClockGatingDisable:hsw */
 322         intel_uncore_rmw(&i915->uncore, TRANS_CHICKEN1(PIPE_A),
 323                          0, TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
 324 }
 325
 326 static void gen8_set_l3sqc_credits(struct drm_i915_private *i915,
 327                                    int general_prio_credits,
 328                                    int high_prio_credits)
 329 {
 330         u32 misccpctl;
 331         u32 val;
 332
 333         /* WaTempDisableDOPClkGating:bdw */
 334         misccpctl = intel_uncore_rmw(&i915->uncore, GEN7_MISCCPCTL,
 335                                      GEN7_DOP_CLOCK_GATE_ENABLE, 0);
 336
 337         val = intel_gt_mcr_read_any(to_gt(i915), GEN8_L3SQCREG1);
 338         val &= ~L3_PRIO_CREDITS_MASK;
 339         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
 340         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
 341         intel_gt_mcr_multicast_write(to_gt(i915), GEN8_L3SQCREG1, val);
 342
 343         /*
 344          * Wait at least 100 clocks before re-enabling clock gating.
 345          * See the definition of L3SQCREG1 in BSpec.
 346          */
 347         intel_gt_mcr_read_any(to_gt(i915), GEN8_L3SQCREG1);
 348         udelay(1);
 349         intel_uncore_write(&i915->uncore, GEN7_MISCCPCTL, misccpctl);
 350 }
 351
 352 static void xehpsdv_init_clock_gating(struct drm_i915_private *i915)
 353 {
 354         /* Wa_22010146351:xehpsdv */
 355         if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
 356                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
 357 }
 358
 359 static void dg2_init_clock_gating(struct drm_i915_private *i915)
 360 {
 361         /* Wa_22010954014:dg2 */
 362         intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
 363                          SGSI_SIDECLK_DIS);
 364 }
 365
 366 static void pvc_init_clock_gating(struct drm_i915_private *i915)
 367 {
 368         /* Wa_14012385139:pvc */
 369         if (IS_PVC_BD_STEP(i915, STEP_A0, STEP_B0))
 370                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
 371
 372         /* Wa_22010954014:pvc */
 373         if (IS_PVC_BD_STEP(i915, STEP_A0, STEP_B0))
 374                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);
 375 }
 376
 377 static void cnp_init_clock_gating(struct drm_i915_private *i915)
 378 {
 379         if (!HAS_PCH_CNP(i915))
 380                 return;
 381
 382         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
 383         intel_uncore_rmw(&i915->uncore, SOUTH_DSPCLK_GATE_D, 0, CNP_PWM_CGE_GATING_DISABLE);
 384 }
 385
 386 static void cfl_init_clock_gating(struct drm_i915_private *i915)
 387 {
 388         cnp_init_clock_gating(i915);
 389         gen9_init_clock_gating(i915);
 390
 391         /* WAC6entrylatency:cfl */
 392         intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 393
 394         /*
 395          * WaFbcTurnOffFbcWatermark:cfl
 396          * Display WA #0562: cfl
 397          */
 398         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 399
 400         /*
 401          * WaFbcNukeOnHostModify:cfl
 402          * Display WA #0873: cfl
 403          */
 404         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 405                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
 406 }
 407
 408 static void kbl_init_clock_gating(struct drm_i915_private *i915)
 409 {
 410         gen9_init_clock_gating(i915);
 411
 412         /* WAC6entrylatency:kbl */
 413         intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 414
 415         /* WaDisableSDEUnitClockGating:kbl */
 416         if (IS_KABYLAKE(i915) && IS_GRAPHICS_STEP(i915, 0, STEP_C0))
 417                 intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6,
 418                                  0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 419
 420         /* WaDisableGamClockGating:kbl */
 421         if (IS_KABYLAKE(i915) && IS_GRAPHICS_STEP(i915, 0, STEP_C0))
 422                 intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1,
 423                                  0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
 424
 425         /*
 426          * WaFbcTurnOffFbcWatermark:kbl
 427          * Display WA #0562: kbl
 428          */
 429         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 430
 431         /*
 432          * WaFbcNukeOnHostModify:kbl
 433          * Display WA #0873: kbl
 434          */
 435         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 436                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
 437 }
 438
 439 static void skl_init_clock_gating(struct drm_i915_private *i915)
 440 {
 441         gen9_init_clock_gating(i915);
 442
 443         /* WaDisableDopClockGating:skl */
 444         intel_uncore_rmw(&i915->uncore, GEN7_MISCCPCTL,
 445                          GEN7_DOP_CLOCK_GATE_ENABLE, 0);
 446
 447         /* WAC6entrylatency:skl */
 448         intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 449
 450         /*
 451          * WaFbcTurnOffFbcWatermark:skl
 452          * Display WA #0562: skl
 453          */
 454         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 455
 456         /*
 457          * WaFbcNukeOnHostModify:skl
 458          * Display WA #0873: skl
 459          */
 460         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 461                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
 462
 463         /*
 464          * WaFbcHighMemBwCorruptionAvoidance:skl
 465          * Display WA #0883: skl
 466          */
 467         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
 468 }
 469
 470 static void bdw_init_clock_gating(struct drm_i915_private *i915)
 471 {
 472         enum pipe pipe;
 473
 474         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
 475         intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
 476
 477         /* WaSwitchSolVfFArbitrationPriority:bdw */
 478         intel_uncore_rmw(&i915->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
 479
 480         /* WaPsrDPAMaskVBlankInSRD:bdw */
 481         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, HSW_MASK_VBL_TO_PIPE_IN_SRD);
 482
 483         for_each_pipe(i915, pipe) {
 484                 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
 485                 intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(pipe),
 486                                  0, BDW_UNMASK_VBL_TO_REGS_IN_SRD);
 487         }
 488
 489         /* WaVSRefCountFullforceMissDisable:bdw */
 490         /* WaDSRefCountFullforceMissDisable:bdw */
 491         intel_uncore_rmw(&i915->uncore, GEN7_FF_THREAD_MODE,
 492                          GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
 493
 494         intel_uncore_write(&i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
 495                            _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
 496
 497         /* WaDisableSDEUnitClockGating:bdw */
 498         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 499
 500         /* WaProgramL3SqcReg1Default:bdw */
 501         gen8_set_l3sqc_credits(i915, 30, 2);
 502
 503         /* WaKVMNotificationOnConfigChange:bdw */
 504         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR2_1,
 505                          0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
 506
 507         lpt_init_clock_gating(i915);
 508
 509         /* WaDisableDopClockGating:bdw
 510          *
 511          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
 512          * clock gating.
 513          */
 514         intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1, 0, GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
 515 }
 516
 517 static void hsw_init_clock_gating(struct drm_i915_private *i915)
 518 {
 519         enum pipe pipe;
 520
 521         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
 522         intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
 523
 524         /* WaPsrDPAMaskVBlankInSRD:hsw */
 525         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, HSW_MASK_VBL_TO_PIPE_IN_SRD);
 526
 527         for_each_pipe(i915, pipe) {
 528                 /* WaPsrDPRSUnmaskVBlankInSRD:hsw */
 529                 intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(pipe),
 530                                  0, HSW_UNMASK_VBL_TO_REGS_IN_SRD);
 531         }
 532
 533         /* This is required by WaCatErrorRejectionIssue:hsw */
 534         intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 535                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 536
 537         /* WaSwitchSolVfFArbitrationPriority:hsw */
 538         intel_uncore_rmw(&i915->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
 539
 540         lpt_init_clock_gating(i915);
 541 }
 542
 543 static void ivb_init_clock_gating(struct drm_i915_private *i915)
 544 {
 545         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
 546
 547         /* WaFbcAsynchFlipDisableFbcQueue:ivb */
 548         intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
 549
 550         /* WaDisableBackToBackFlipFix:ivb */
 551         intel_uncore_write(&i915->uncore, IVB_CHICKEN3,
 552                            CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
 553                            CHICKEN3_DGMG_DONE_FIX_DISABLE);
 554
 555         if (IS_IVB_GT1(i915))
 556                 intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
 557                                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 558         else {
 559                 /* must write both registers */
 560                 intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
 561                                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 562                 intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2_GT2,
 563                                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 564         }
 565
 566         /*
 567          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
 568          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
 569          */
 570         intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
 571                            GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
 572
 573         /* This is required by WaCatErrorRejectionIssue:ivb */
 574         intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 575                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 576
 577         g4x_disable_trickle_feed(i915);
 578
 579         intel_uncore_rmw(&i915->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
 580                          GEN6_MBC_SNPCR_MED);
 581
 582         if (!HAS_PCH_NOP(i915))
 583                 cpt_init_clock_gating(i915);
 584
 585         gen6_check_mch_setup(i915);
 586 }
 587
 588 static void vlv_init_clock_gating(struct drm_i915_private *i915)
 589 {
 590         /* WaDisableBackToBackFlipFix:vlv */
 591         intel_uncore_write(&i915->uncore, IVB_CHICKEN3,
 592                            CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
 593                            CHICKEN3_DGMG_DONE_FIX_DISABLE);
 594
 595         /* WaDisableDopClockGating:vlv */
 596         intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
 597                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 598
 599         /* This is required by WaCatErrorRejectionIssue:vlv */
 600         intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 601                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 602
 603         /*
 604          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
 605          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
 606          */
 607         intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
 608                            GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
 609
 610         /* WaDisableL3Bank2xClockGate:vlv
 611          * Disabling L3 clock gating- MMIO 940c[25] = 1
 612          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
 613         intel_uncore_rmw(&i915->uncore, GEN7_UCGCTL4, 0, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
 614
 615         /*
 616          * WaDisableVLVClockGating_VBIIssue:vlv
 617          * Disable clock gating on th GCFG unit to prevent a delay
 618          * in the reporting of vblank events.
 619          */
 620         intel_uncore_write(&i915->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
 621 }
 622
 623 static void chv_init_clock_gating(struct drm_i915_private *i915)
 624 {
 625         /* WaVSRefCountFullforceMissDisable:chv */
 626         /* WaDSRefCountFullforceMissDisable:chv */
 627         intel_uncore_rmw(&i915->uncore, GEN7_FF_THREAD_MODE,
 628                          GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
 629
 630         /* WaDisableSemaphoreAndSyncFlipWait:chv */
 631         intel_uncore_write(&i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
 632                            _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
 633
 634         /* WaDisableCSUnitClockGating:chv */
 635         intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
 636
 637         /* WaDisableSDEUnitClockGating:chv */
 638         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 639
 640         /*
 641          * WaProgramL3SqcReg1Default:chv
 642          * See gfxspecs/Related Documents/Performance Guide/
 643          * LSQC Setting Recommendations.
 644          */
 645         gen8_set_l3sqc_credits(i915, 38, 2);
 646 }
 647
 648 static void g4x_init_clock_gating(struct drm_i915_private *i915)
 649 {
 650         u32 dspclk_gate;
 651
 652         intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, 0);
 653         intel_uncore_write(&i915->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
 654                            GS_UNIT_CLOCK_GATE_DISABLE |
 655                            CL_UNIT_CLOCK_GATE_DISABLE);
 656         intel_uncore_write(&i915->uncore, RAMCLK_GATE_D, 0);
 657         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
 658                 OVRUNIT_CLOCK_GATE_DISABLE |
 659                 OVCUNIT_CLOCK_GATE_DISABLE;
 660         if (IS_GM45(i915))
 661                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
 662         intel_uncore_write(&i915->uncore, DSPCLK_GATE_D(i915), dspclk_gate);
 663
 664         g4x_disable_trickle_feed(i915);
 665 }
 666
 667 static void i965gm_init_clock_gating(struct drm_i915_private *i915)
 668 {
 669         struct intel_uncore *uncore = &i915->uncore;
 670
 671         intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
 672         intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
 673         intel_uncore_write(uncore, DSPCLK_GATE_D(i915), 0);
 674         intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
 675         intel_uncore_write16(uncore, DEUC, 0);
 676         intel_uncore_write(uncore,
 677                            MI_ARB_STATE,
 678                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 679 }
 680
 681 static void i965g_init_clock_gating(struct drm_i915_private *i915)
 682 {
 683         intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
 684                            I965_RCC_CLOCK_GATE_DISABLE |
 685                            I965_RCPB_CLOCK_GATE_DISABLE |
 686                            I965_ISC_CLOCK_GATE_DISABLE |
 687                            I965_FBC_CLOCK_GATE_DISABLE);
 688         intel_uncore_write(&i915->uncore, RENCLK_GATE_D2, 0);
 689         intel_uncore_write(&i915->uncore, MI_ARB_STATE,
 690                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 691 }
 692
 693 static void gen3_init_clock_gating(struct drm_i915_private *i915)
 694 {
 695         u32 dstate = intel_uncore_read(&i915->uncore, D_STATE);
 696
 697         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
 698                 DSTATE_DOT_CLOCK_GATING;
 699         intel_uncore_write(&i915->uncore, D_STATE, dstate);
 700
 701         if (IS_PINEVIEW(i915))
 702                 intel_uncore_write(&i915->uncore, ECOSKPD(RENDER_RING_BASE),
 703                                    _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
 704
 705         /* IIR "flip pending" means done if this bit is set */
 706         intel_uncore_write(&i915->uncore, ECOSKPD(RENDER_RING_BASE),
 707                            _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
 708
 709         /* interrupts should cause a wake up from C3 */
 710         intel_uncore_write(&i915->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
 711
 712         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
 713         intel_uncore_write(&i915->uncore, MI_ARB_STATE,
 714                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
 715
 716         intel_uncore_write(&i915->uncore, MI_ARB_STATE,
 717                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 718 }
 719
 720 static void i85x_init_clock_gating(struct drm_i915_private *i915)
 721 {
 722         intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
 723
 724         /* interrupts should cause a wake up from C3 */
 725         intel_uncore_write(&i915->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
 726                            _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
 727
 728         intel_uncore_write(&i915->uncore, MEM_MODE,
 729                            _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
 730
 731         /*
 732          * Have FBC ignore 3D activity since we use software
 733          * render tracking, and otherwise a pure 3D workload
 734          * (even if it just renders a single frame and then does
 735          * abosultely nothing) would not allow FBC to recompress
 736          * until a 2D blit occurs.
 737          */
 738         intel_uncore_write(&i915->uncore, SCPD0,
 739                            _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
 740 }
 741
 742 static void i830_init_clock_gating(struct drm_i915_private *i915)
 743 {
 744         intel_uncore_write(&i915->uncore, MEM_MODE,
 745                            _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
 746                            _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
 747 }
 748
 749 void intel_clock_gating_init(struct drm_i915_private *i915)
 750 {
 751         i915->clock_gating_funcs->init_clock_gating(i915);
 752 }
 753
 754 static void nop_init_clock_gating(struct drm_i915_private *i915)
 755 {
 756         drm_dbg_kms(&i915->drm,
 757                     "No clock gating settings or workarounds applied.\n");
 758 }
 759
 760 #define CG_FUNCS(platform)                                              \
 761 static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
 762         .init_clock_gating = platform##_init_clock_gating,              \
 763 }
 764
 765 CG_FUNCS(pvc);
 766 CG_FUNCS(dg2);
 767 CG_FUNCS(xehpsdv);
 768 CG_FUNCS(cfl);
 769 CG_FUNCS(skl);
 770 CG_FUNCS(kbl);
 771 CG_FUNCS(bxt);
 772 CG_FUNCS(glk);
 773 CG_FUNCS(bdw);
 774 CG_FUNCS(chv);
 775 CG_FUNCS(hsw);
 776 CG_FUNCS(ivb);
 777 CG_FUNCS(vlv);
 778 CG_FUNCS(gen6);
 779 CG_FUNCS(ilk);
 780 CG_FUNCS(g4x);
 781 CG_FUNCS(i965gm);
 782 CG_FUNCS(i965g);
 783 CG_FUNCS(gen3);
 784 CG_FUNCS(i85x);
 785 CG_FUNCS(i830);
 786 CG_FUNCS(nop);
 787 #undef CG_FUNCS
 788
 789 /**
 790  * intel_clock_gating_hooks_init - setup the clock gating hooks
 791  * @i915: device private
 792  *
 793  * Setup the hooks that configure which clocks of a given platform can be
 794  * gated and also apply various GT and display specific workarounds for these
 795  * platforms. Note that some GT specific workarounds are applied separately
 796  * when GPU contexts or batchbuffers start their execution.
 797  */
 798 void intel_clock_gating_hooks_init(struct drm_i915_private *i915)
 799 {
 800         if (IS_PONTEVECCHIO(i915))
 801                 i915->clock_gating_funcs = &pvc_clock_gating_funcs;
 802         else if (IS_DG2(i915))
 803                 i915->clock_gating_funcs = &dg2_clock_gating_funcs;
 804         else if (IS_XEHPSDV(i915))
 805                 i915->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
 806         else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
 807                 i915->clock_gating_funcs = &cfl_clock_gating_funcs;
 808         else if (IS_SKYLAKE(i915))
 809                 i915->clock_gating_funcs = &skl_clock_gating_funcs;
 810         else if (IS_KABYLAKE(i915))
 811                 i915->clock_gating_funcs = &kbl_clock_gating_funcs;
 812         else if (IS_BROXTON(i915))
 813                 i915->clock_gating_funcs = &bxt_clock_gating_funcs;
 814         else if (IS_GEMINILAKE(i915))
 815                 i915->clock_gating_funcs = &glk_clock_gating_funcs;
 816         else if (IS_BROADWELL(i915))
 817                 i915->clock_gating_funcs = &bdw_clock_gating_funcs;
 818         else if (IS_CHERRYVIEW(i915))
 819                 i915->clock_gating_funcs = &chv_clock_gating_funcs;
 820         else if (IS_HASWELL(i915))
 821                 i915->clock_gating_funcs = &hsw_clock_gating_funcs;
 822         else if (IS_IVYBRIDGE(i915))
 823                 i915->clock_gating_funcs = &ivb_clock_gating_funcs;
 824         else if (IS_VALLEYVIEW(i915))
 825                 i915->clock_gating_funcs = &vlv_clock_gating_funcs;
 826         else if (GRAPHICS_VER(i915) == 6)
 827                 i915->clock_gating_funcs = &gen6_clock_gating_funcs;
 828         else if (GRAPHICS_VER(i915) == 5)
 829                 i915->clock_gating_funcs = &ilk_clock_gating_funcs;
 830         else if (IS_G4X(i915))
 831                 i915->clock_gating_funcs = &g4x_clock_gating_funcs;
 832         else if (IS_I965GM(i915))
 833                 i915->clock_gating_funcs = &i965gm_clock_gating_funcs;
 834         else if (IS_I965G(i915))
 835                 i915->clock_gating_funcs = &i965g_clock_gating_funcs;
 836         else if (GRAPHICS_VER(i915) == 3)
 837                 i915->clock_gating_funcs = &gen3_clock_gating_funcs;
 838         else if (IS_I85X(i915) || IS_I865G(i915))
 839                 i915->clock_gating_funcs = &i85x_clock_gating_funcs;
 840         else if (GRAPHICS_VER(i915) == 2)
 841                 i915->clock_gating_funcs = &i830_clock_gating_funcs;
 842         else
 843                 i915->clock_gating_funcs = &nop_clock_gating_funcs;
 844 }