drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <[email protected]>
  25  *
  26  */
  27
  28 #include "display/intel_de.h"
  29 #include "display/intel_display.h"
  30 #include "display/intel_display_trace.h"
  31 #include "display/skl_watermark.h"
  32
  33 #include "gt/intel_engine_regs.h"
  34 #include "gt/intel_gt.h"
  35 #include "gt/intel_gt_mcr.h"
  36 #include "gt/intel_gt_regs.h"
  37
  38 #include "i915_drv.h"
  39 #include "intel_mchbar_regs.h"
  40 #include "intel_pm.h"
  41 #include "vlv_sideband.h"
  42
  43 struct drm_i915_clock_gating_funcs {
  44         void (*init_clock_gating)(struct drm_i915_private *i915);
  45 };
  46
  47 /* used in computing the new watermarks state */
  48 struct intel_wm_config {
  49         unsigned int num_pipes_active;
  50         bool sprites_enabled;
  51         bool sprites_scaled;
  52 };
  53
  54 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
  55 {
  56         if (HAS_LLC(dev_priv)) {
  57                 /*
  58                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  59                  * Display WA #0390: skl,kbl
  60                  *
  61                  * Must match Sampler, Pixel Back End, and Media. See
  62                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  63                  */
  64                 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
  65         }
  66
  67         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  68         intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
  69
  70         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  71         intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
  72
  73         /*
  74          * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
  75          * Display WA #0859: skl,bxt,kbl,glk,cfl
  76          */
  77         intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
  78 }
  79
  80 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
  81 {
  82         gen9_init_clock_gating(dev_priv);
  83
  84         /* WaDisableSDEUnitClockGating:bxt */
  85         intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
  86
  87         /*
  88          * FIXME:
  89          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
  90          */
  91         intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
  92
  93         /*
  94          * Wa: Backlight PWM may stop in the asserted state, causing backlight
  95          * to stay fully on.
  96          */
  97         intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
  98                    PWM1_GATING_DIS | PWM2_GATING_DIS);
  99
 100         /*
 101          * Lower the display internal timeout.
 102          * This is needed to avoid any hard hangs when DSI port PLL
 103          * is off and a MMIO access is attempted by any privilege
 104          * application, using batch buffers or any other means.
 105          */
 106         intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
 107
 108         /*
 109          * WaFbcTurnOffFbcWatermark:bxt
 110          * Display WA #0562: bxt
 111          */
 112         intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 113
 114         /*
 115          * WaFbcHighMemBwCorruptionAvoidance:bxt
 116          * Display WA #0883: bxt
 117          */
 118         intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
 119 }
 120
 121 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
 122 {
 123         gen9_init_clock_gating(dev_priv);
 124
 125         /*
 126          * WaDisablePWMClockGating:glk
 127          * Backlight PWM may stop in the asserted state, causing backlight
 128          * to stay fully on.
 129          */
 130         intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
 131                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 132 }
 133
 134 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
 135 {
 136         u32 tmp;
 137
 138         tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
 139
 140         switch (tmp & CLKCFG_FSB_MASK) {
 141         case CLKCFG_FSB_533:
 142                 dev_priv->fsb_freq = 533; /* 133*4 */
 143                 break;
 144         case CLKCFG_FSB_800:
 145                 dev_priv->fsb_freq = 800; /* 200*4 */
 146                 break;
 147         case CLKCFG_FSB_667:
 148                 dev_priv->fsb_freq =  667; /* 167*4 */
 149                 break;
 150         case CLKCFG_FSB_400:
 151                 dev_priv->fsb_freq = 400; /* 100*4 */
 152                 break;
 153         }
 154
 155         switch (tmp & CLKCFG_MEM_MASK) {
 156         case CLKCFG_MEM_533:
 157                 dev_priv->mem_freq = 533;
 158                 break;
 159         case CLKCFG_MEM_667:
 160                 dev_priv->mem_freq = 667;
 161                 break;
 162         case CLKCFG_MEM_800:
 163                 dev_priv->mem_freq = 800;
 164                 break;
 165         }
 166
 167         /* detect pineview DDR3 setting */
 168         tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
 169         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 170 }
 171
 172 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
 173 {
 174         u16 ddrpll, csipll;
 175
 176         ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
 177         csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
 178
 179         switch (ddrpll & 0xff) {
 180         case 0xc:
 181                 dev_priv->mem_freq = 800;
 182                 break;
 183         case 0x10:
 184                 dev_priv->mem_freq = 1066;
 185                 break;
 186         case 0x14:
 187                 dev_priv->mem_freq = 1333;
 188                 break;
 189         case 0x18:
 190                 dev_priv->mem_freq = 1600;
 191                 break;
 192         default:
 193                 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
 194                         ddrpll & 0xff);
 195                 dev_priv->mem_freq = 0;
 196                 break;
 197         }
 198
 199         switch (csipll & 0x3ff) {
 200         case 0x00c:
 201                 dev_priv->fsb_freq = 3200;
 202                 break;
 203         case 0x00e:
 204                 dev_priv->fsb_freq = 3733;
 205                 break;
 206         case 0x010:
 207                 dev_priv->fsb_freq = 4266;
 208                 break;
 209         case 0x012:
 210                 dev_priv->fsb_freq = 4800;
 211                 break;
 212         case 0x014:
 213                 dev_priv->fsb_freq = 5333;
 214                 break;
 215         case 0x016:
 216                 dev_priv->fsb_freq = 5866;
 217                 break;
 218         case 0x018:
 219                 dev_priv->fsb_freq = 6400;
 220                 break;
 221         default:
 222                 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
 223                         csipll & 0x3ff);
 224                 dev_priv->fsb_freq = 0;
 225                 break;
 226         }
 227 }
 228
 229 static const struct cxsr_latency cxsr_latency_table[] = {
 230         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 231         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 232         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 233         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 234         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 235
 236         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 237         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 238         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 239         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 240         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 241
 242         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 243         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 244         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 245         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 246         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 247
 248         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 249         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 250         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 251         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 252         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 253
 254         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 255         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 256         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 257         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 258         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 259
 260         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 261         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 262         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 263         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 264         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 265 };
 266
 267 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
 268                                                          bool is_ddr3,
 269                                                          int fsb,
 270                                                          int mem)
 271 {
 272         const struct cxsr_latency *latency;
 273         int i;
 274
 275         if (fsb == 0 || mem == 0)
 276                 return NULL;
 277
 278         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 279                 latency = &cxsr_latency_table[i];
 280                 if (is_desktop == latency->is_desktop &&
 281                     is_ddr3 == latency->is_ddr3 &&
 282                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 283                         return latency;
 284         }
 285
 286         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 287
 288         return NULL;
 289 }
 290
 291 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 292 {
 293         u32 val;
 294
 295         vlv_punit_get(dev_priv);
 296
 297         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 298         if (enable)
 299                 val &= ~FORCE_DDR_HIGH_FREQ;
 300         else
 301                 val |= FORCE_DDR_HIGH_FREQ;
 302         val &= ~FORCE_DDR_LOW_FREQ;
 303         val |= FORCE_DDR_FREQ_REQ_ACK;
 304         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 305
 306         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 307                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 308                 drm_err(&dev_priv->drm,
 309                         "timed out waiting for Punit DDR DVFS request\n");
 310
 311         vlv_punit_put(dev_priv);
 312 }
 313
 314 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 315 {
 316         u32 val;
 317
 318         vlv_punit_get(dev_priv);
 319
 320         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 321         if (enable)
 322                 val |= DSP_MAXFIFO_PM5_ENABLE;
 323         else
 324                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 325         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 326
 327         vlv_punit_put(dev_priv);
 328 }
 329
 330 #define FW_WM(value, plane) \
 331         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 332
 333 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 334 {
 335         bool was_enabled;
 336         u32 val;
 337
 338         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 339                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 340                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 341                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
 342         } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
 343                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
 344                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 345                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
 346         } else if (IS_PINEVIEW(dev_priv)) {
 347                 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
 348                 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 349                 if (enable)
 350                         val |= PINEVIEW_SELF_REFRESH_EN;
 351                 else
 352                         val &= ~PINEVIEW_SELF_REFRESH_EN;
 353                 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
 354                 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
 355         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 356                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
 357                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 358                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 359                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
 360                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
 361         } else if (IS_I915GM(dev_priv)) {
 362                 /*
 363                  * FIXME can't find a bit like this for 915G, and
 364                  * and yet it does have the related watermark in
 365                  * FW_BLC_SELF. What's going on?
 366                  */
 367                 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
 368                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 369                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 370                 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
 371                 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
 372         } else {
 373                 return false;
 374         }
 375
 376         trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
 377
 378         drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
 379                     str_enabled_disabled(enable),
 380                     str_enabled_disabled(was_enabled));
 381
 382         return was_enabled;
 383 }
 384
 385 /**
 386  * intel_set_memory_cxsr - Configure CxSR state
 387  * @dev_priv: i915 device
 388  * @enable: Allow vs. disallow CxSR
 389  *
 390  * Allow or disallow the system to enter a special CxSR
 391  * (C-state self refresh) state. What typically happens in CxSR mode
 392  * is that several display FIFOs may get combined into a single larger
 393  * FIFO for a particular plane (so called max FIFO mode) to allow the
 394  * system to defer memory fetches longer, and the memory will enter
 395  * self refresh.
 396  *
 397  * Note that enabling CxSR does not guarantee that the system enter
 398  * this special mode, nor does it guarantee that the system stays
 399  * in that mode once entered. So this just allows/disallows the system
 400  * to autonomously utilize the CxSR mode. Other factors such as core
 401  * C-states will affect when/if the system actually enters/exits the
 402  * CxSR mode.
 403  *
 404  * Note that on VLV/CHV this actually only controls the max FIFO mode,
 405  * and the system is free to enter/exit memory self refresh at any time
 406  * even when the use of CxSR has been disallowed.
 407  *
 408  * While the system is actually in the CxSR/max FIFO mode, some plane
 409  * control registers will not get latched on vblank. Thus in order to
 410  * guarantee the system will respond to changes in the plane registers
 411  * we must always disallow CxSR prior to making changes to those registers.
 412  * Unfortunately the system will re-evaluate the CxSR conditions at
 413  * frame start which happens after vblank start (which is when the plane
 414  * registers would get latched), so we can't proceed with the plane update
 415  * during the same frame where we disallowed CxSR.
 416  *
 417  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 418  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 419  * the hardware w.r.t. HPLL SR when writing to plane registers.
 420  * Disallowing just CxSR is sufficient.
 421  */
 422 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 423 {
 424         bool ret;
 425
 426         mutex_lock(&dev_priv->display.wm.wm_mutex);
 427         ret = _intel_set_memory_cxsr(dev_priv, enable);
 428         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 429                 dev_priv->display.wm.vlv.cxsr = enable;
 430         else if (IS_G4X(dev_priv))
 431                 dev_priv->display.wm.g4x.cxsr = enable;
 432         mutex_unlock(&dev_priv->display.wm.wm_mutex);
 433
 434         return ret;
 435 }
 436
 437 /*
 438  * Latency for FIFO fetches is dependent on several factors:
 439  *   - memory configuration (speed, channels)
 440  *   - chipset
 441  *   - current MCH state
 442  * It can be fairly high in some situations, so here we assume a fairly
 443  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 444  * set this value too high, the FIFO will fetch frequently to stay full)
 445  * and power consumption (set it too low to save power and we might see
 446  * FIFO underruns and display "flicker").
 447  *
 448  * A value of 5us seems to be a good balance; safe for very low end
 449  * platforms but not overly aggressive on lower latency configs.
 450  */
 451 static const int pessimal_latency_ns = 5000;
 452
 453 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 454         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 455
 456 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 457 {
 458         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 459         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 460         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 461         enum pipe pipe = crtc->pipe;
 462         int sprite0_start, sprite1_start;
 463         u32 dsparb, dsparb2, dsparb3;
 464
 465         switch (pipe) {
 466         case PIPE_A:
 467                 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 468                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 469                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 470                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 471                 break;
 472         case PIPE_B:
 473                 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 474                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 475                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 476                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 477                 break;
 478         case PIPE_C:
 479                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 480                 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
 481                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 482                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 483                 break;
 484         default:
 485                 MISSING_CASE(pipe);
 486                 return;
 487         }
 488
 489         fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 490         fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 491         fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 492         fifo_state->plane[PLANE_CURSOR] = 63;
 493 }
 494
 495 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
 496                               enum i9xx_plane_id i9xx_plane)
 497 {
 498         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 499         int size;
 500
 501         size = dsparb & 0x7f;
 502         if (i9xx_plane == PLANE_B)
 503                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 504
 505         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 506                     dsparb, plane_name(i9xx_plane), size);
 507
 508         return size;
 509 }
 510
 511 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
 512                               enum i9xx_plane_id i9xx_plane)
 513 {
 514         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 515         int size;
 516
 517         size = dsparb & 0x1ff;
 518         if (i9xx_plane == PLANE_B)
 519                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 520         size >>= 1; /* Convert to cachelines */
 521
 522         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 523                     dsparb, plane_name(i9xx_plane), size);
 524
 525         return size;
 526 }
 527
 528 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
 529                               enum i9xx_plane_id i9xx_plane)
 530 {
 531         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 532         int size;
 533
 534         size = dsparb & 0x7f;
 535         size >>= 2; /* Convert to cachelines */
 536
 537         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 538                     dsparb, plane_name(i9xx_plane), size);
 539
 540         return size;
 541 }
 542
 543 /* Pineview has different values for various configs */
 544 static const struct intel_watermark_params pnv_display_wm = {
 545         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 546         .max_wm = PINEVIEW_MAX_WM,
 547         .default_wm = PINEVIEW_DFT_WM,
 548         .guard_size = PINEVIEW_GUARD_WM,
 549         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 550 };
 551
 552 static const struct intel_watermark_params pnv_display_hplloff_wm = {
 553         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 554         .max_wm = PINEVIEW_MAX_WM,
 555         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 556         .guard_size = PINEVIEW_GUARD_WM,
 557         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 558 };
 559
 560 static const struct intel_watermark_params pnv_cursor_wm = {
 561         .fifo_size = PINEVIEW_CURSOR_FIFO,
 562         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 563         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 564         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 565         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 566 };
 567
 568 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
 569         .fifo_size = PINEVIEW_CURSOR_FIFO,
 570         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 571         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 572         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 573         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 574 };
 575
 576 static const struct intel_watermark_params i965_cursor_wm_info = {
 577         .fifo_size = I965_CURSOR_FIFO,
 578         .max_wm = I965_CURSOR_MAX_WM,
 579         .default_wm = I965_CURSOR_DFT_WM,
 580         .guard_size = 2,
 581         .cacheline_size = I915_FIFO_LINE_SIZE,
 582 };
 583
 584 static const struct intel_watermark_params i945_wm_info = {
 585         .fifo_size = I945_FIFO_SIZE,
 586         .max_wm = I915_MAX_WM,
 587         .default_wm = 1,
 588         .guard_size = 2,
 589         .cacheline_size = I915_FIFO_LINE_SIZE,
 590 };
 591
 592 static const struct intel_watermark_params i915_wm_info = {
 593         .fifo_size = I915_FIFO_SIZE,
 594         .max_wm = I915_MAX_WM,
 595         .default_wm = 1,
 596         .guard_size = 2,
 597         .cacheline_size = I915_FIFO_LINE_SIZE,
 598 };
 599
 600 static const struct intel_watermark_params i830_a_wm_info = {
 601         .fifo_size = I855GM_FIFO_SIZE,
 602         .max_wm = I915_MAX_WM,
 603         .default_wm = 1,
 604         .guard_size = 2,
 605         .cacheline_size = I830_FIFO_LINE_SIZE,
 606 };
 607
 608 static const struct intel_watermark_params i830_bc_wm_info = {
 609         .fifo_size = I855GM_FIFO_SIZE,
 610         .max_wm = I915_MAX_WM/2,
 611         .default_wm = 1,
 612         .guard_size = 2,
 613         .cacheline_size = I830_FIFO_LINE_SIZE,
 614 };
 615
 616 static const struct intel_watermark_params i845_wm_info = {
 617         .fifo_size = I830_FIFO_SIZE,
 618         .max_wm = I915_MAX_WM,
 619         .default_wm = 1,
 620         .guard_size = 2,
 621         .cacheline_size = I830_FIFO_LINE_SIZE,
 622 };
 623
 624 /**
 625  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 626  * @pixel_rate: Pipe pixel rate in kHz
 627  * @cpp: Plane bytes per pixel
 628  * @latency: Memory wakeup latency in 0.1us units
 629  *
 630  * Compute the watermark using the method 1 or "small buffer"
 631  * formula. The caller may additonally add extra cachelines
 632  * to account for TLB misses and clock crossings.
 633  *
 634  * This method is concerned with the short term drain rate
 635  * of the FIFO, ie. it does not account for blanking periods
 636  * which would effectively reduce the average drain rate across
 637  * a longer period. The name "small" refers to the fact the
 638  * FIFO is relatively small compared to the amount of data
 639  * fetched.
 640  *
 641  * The FIFO level vs. time graph might look something like:
 642  *
 643  *   |\   |\
 644  *   | \  | \
 645  * __---__---__ (- plane active, _ blanking)
 646  * -> time
 647  *
 648  * or perhaps like this:
 649  *
 650  *   |\|\  |\|\
 651  * __----__----__ (- plane active, _ blanking)
 652  * -> time
 653  *
 654  * Returns:
 655  * The watermark in bytes
 656  */
 657 static unsigned int intel_wm_method1(unsigned int pixel_rate,
 658                                      unsigned int cpp,
 659                                      unsigned int latency)
 660 {
 661         u64 ret;
 662
 663         ret = mul_u32_u32(pixel_rate, cpp * latency);
 664         ret = DIV_ROUND_UP_ULL(ret, 10000);
 665
 666         return ret;
 667 }
 668
 669 /**
 670  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 671  * @pixel_rate: Pipe pixel rate in kHz
 672  * @htotal: Pipe horizontal total
 673  * @width: Plane width in pixels
 674  * @cpp: Plane bytes per pixel
 675  * @latency: Memory wakeup latency in 0.1us units
 676  *
 677  * Compute the watermark using the method 2 or "large buffer"
 678  * formula. The caller may additonally add extra cachelines
 679  * to account for TLB misses and clock crossings.
 680  *
 681  * This method is concerned with the long term drain rate
 682  * of the FIFO, ie. it does account for blanking periods
 683  * which effectively reduce the average drain rate across
 684  * a longer period. The name "large" refers to the fact the
 685  * FIFO is relatively large compared to the amount of data
 686  * fetched.
 687  *
 688  * The FIFO level vs. time graph might look something like:
 689  *
 690  *    |\___       |\___
 691  *    |    \___   |    \___
 692  *    |        \  |        \
 693  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 694  * -> time
 695  *
 696  * Returns:
 697  * The watermark in bytes
 698  */
 699 static unsigned int intel_wm_method2(unsigned int pixel_rate,
 700                                      unsigned int htotal,
 701                                      unsigned int width,
 702                                      unsigned int cpp,
 703                                      unsigned int latency)
 704 {
 705         unsigned int ret;
 706
 707         /*
 708          * FIXME remove once all users are computing
 709          * watermarks in the correct place.
 710          */
 711         if (WARN_ON_ONCE(htotal == 0))
 712                 htotal = 1;
 713
 714         ret = (latency * pixel_rate) / (htotal * 10000);
 715         ret = (ret + 1) * width * cpp;
 716
 717         return ret;
 718 }
 719
 720 /**
 721  * intel_calculate_wm - calculate watermark level
 722  * @pixel_rate: pixel clock
 723  * @wm: chip FIFO params
 724  * @fifo_size: size of the FIFO buffer
 725  * @cpp: bytes per pixel
 726  * @latency_ns: memory latency for the platform
 727  *
 728  * Calculate the watermark level (the level at which the display plane will
 729  * start fetching from memory again).  Each chip has a different display
 730  * FIFO size and allocation, so the caller needs to figure that out and pass
 731  * in the correct intel_watermark_params structure.
 732  *
 733  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 734  * on the pixel size.  When it reaches the watermark level, it'll start
 735  * fetching FIFO line sized based chunks from memory until the FIFO fills
 736  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 737  * will occur, and a display engine hang could result.
 738  */
 739 static unsigned int intel_calculate_wm(int pixel_rate,
 740                                        const struct intel_watermark_params *wm,
 741                                        int fifo_size, int cpp,
 742                                        unsigned int latency_ns)
 743 {
 744         int entries, wm_size;
 745
 746         /*
 747          * Note: we need to make sure we don't overflow for various clock &
 748          * latency values.
 749          * clocks go from a few thousand to several hundred thousand.
 750          * latency is usually a few thousand
 751          */
 752         entries = intel_wm_method1(pixel_rate, cpp,
 753                                    latency_ns / 100);
 754         entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 755                 wm->guard_size;
 756         DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
 757
 758         wm_size = fifo_size - entries;
 759         DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
 760
 761         /* Don't promote wm_size to unsigned... */
 762         if (wm_size > wm->max_wm)
 763                 wm_size = wm->max_wm;
 764         if (wm_size <= 0)
 765                 wm_size = wm->default_wm;
 766
 767         /*
 768          * Bspec seems to indicate that the value shouldn't be lower than
 769          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 770          * Lets go for 8 which is the burst size since certain platforms
 771          * already use a hardcoded 8 (which is what the spec says should be
 772          * done).
 773          */
 774         if (wm_size <= 8)
 775                 wm_size = 8;
 776
 777         return wm_size;
 778 }
 779
 780 static bool is_disabling(int old, int new, int threshold)
 781 {
 782         return old >= threshold && new < threshold;
 783 }
 784
 785 static bool is_enabling(int old, int new, int threshold)
 786 {
 787         return old < threshold && new >= threshold;
 788 }
 789
 790 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
 791 {
 792         return dev_priv->display.wm.max_level + 1;
 793 }
 794
 795 bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
 796                             const struct intel_plane_state *plane_state)
 797 {
 798         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 799
 800         /* FIXME check the 'enable' instead */
 801         if (!crtc_state->hw.active)
 802                 return false;
 803
 804         /*
 805          * Treat cursor with fb as always visible since cursor updates
 806          * can happen faster than the vrefresh rate, and the current
 807          * watermark code doesn't handle that correctly. Cursor updates
 808          * which set/clear the fb or change the cursor size are going
 809          * to get throttled by intel_legacy_cursor_update() to work
 810          * around this problem with the watermark code.
 811          */
 812         if (plane->id == PLANE_CURSOR)
 813                 return plane_state->hw.fb != NULL;
 814         else
 815                 return plane_state->uapi.visible;
 816 }
 817
 818 static bool intel_crtc_active(struct intel_crtc *crtc)
 819 {
 820         /* Be paranoid as we can arrive here with only partial
 821          * state retrieved from the hardware during setup.
 822          *
 823          * We can ditch the adjusted_mode.crtc_clock check as soon
 824          * as Haswell has gained clock readout/fastboot support.
 825          *
 826          * We can ditch the crtc->primary->state->fb check as soon as we can
 827          * properly reconstruct framebuffers.
 828          *
 829          * FIXME: The intel_crtc->active here should be switched to
 830          * crtc->state->active once we have proper CRTC states wired up
 831          * for atomic.
 832          */
 833         return crtc && crtc->active && crtc->base.primary->state->fb &&
 834                 crtc->config->hw.adjusted_mode.crtc_clock;
 835 }
 836
 837 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 838 {
 839         struct intel_crtc *crtc, *enabled = NULL;
 840
 841         for_each_intel_crtc(&dev_priv->drm, crtc) {
 842                 if (intel_crtc_active(crtc)) {
 843                         if (enabled)
 844                                 return NULL;
 845                         enabled = crtc;
 846                 }
 847         }
 848
 849         return enabled;
 850 }
 851
 852 static void pnv_update_wm(struct drm_i915_private *dev_priv)
 853 {
 854         struct intel_crtc *crtc;
 855         const struct cxsr_latency *latency;
 856         u32 reg;
 857         unsigned int wm;
 858
 859         latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
 860                                          dev_priv->is_ddr3,
 861                                          dev_priv->fsb_freq,
 862                                          dev_priv->mem_freq);
 863         if (!latency) {
 864                 drm_dbg_kms(&dev_priv->drm,
 865                             "Unknown FSB/MEM found, disable CxSR\n");
 866                 intel_set_memory_cxsr(dev_priv, false);
 867                 return;
 868         }
 869
 870         crtc = single_enabled_crtc(dev_priv);
 871         if (crtc) {
 872                 const struct drm_framebuffer *fb =
 873                         crtc->base.primary->state->fb;
 874                 int pixel_rate = crtc->config->pixel_rate;
 875                 int cpp = fb->format->cpp[0];
 876
 877                 /* Display SR */
 878                 wm = intel_calculate_wm(pixel_rate, &pnv_display_wm,
 879                                         pnv_display_wm.fifo_size,
 880                                         cpp, latency->display_sr);
 881                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
 882                 reg &= ~DSPFW_SR_MASK;
 883                 reg |= FW_WM(wm, SR);
 884                 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
 885                 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
 886
 887                 /* cursor SR */
 888                 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_wm,
 889                                         pnv_display_wm.fifo_size,
 890                                         4, latency->cursor_sr);
 891                 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_CURSOR_SR_MASK,
 892                                  FW_WM(wm, CURSOR_SR));
 893
 894                 /* Display HPLL off SR */
 895                 wm = intel_calculate_wm(pixel_rate, &pnv_display_hplloff_wm,
 896                                         pnv_display_hplloff_wm.fifo_size,
 897                                         cpp, latency->display_hpll_disable);
 898                 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR));
 899
 900                 /* cursor HPLL off SR */
 901                 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_hplloff_wm,
 902                                         pnv_display_hplloff_wm.fifo_size,
 903                                         4, latency->cursor_hpll_disable);
 904                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
 905                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 906                 reg |= FW_WM(wm, HPLL_CURSOR);
 907                 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
 908                 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
 909
 910                 intel_set_memory_cxsr(dev_priv, true);
 911         } else {
 912                 intel_set_memory_cxsr(dev_priv, false);
 913         }
 914 }
 915
 916 /*
 917  * Documentation says:
 918  * "If the line size is small, the TLB fetches can get in the way of the
 919  *  data fetches, causing some lag in the pixel data return which is not
 920  *  accounted for in the above formulas. The following adjustment only
 921  *  needs to be applied if eight whole lines fit in the buffer at once.
 922  *  The WM is adjusted upwards by the difference between the FIFO size
 923  *  and the size of 8 whole lines. This adjustment is always performed
 924  *  in the actual pixel depth regardless of whether FBC is enabled or not."
 925  */
 926 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 927 {
 928         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 929
 930         return max(0, tlb_miss);
 931 }
 932
 933 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 934                                 const struct g4x_wm_values *wm)
 935 {
 936         enum pipe pipe;
 937
 938         for_each_pipe(dev_priv, pipe)
 939                 trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
 940
 941         intel_uncore_write(&dev_priv->uncore, DSPFW1,
 942                    FW_WM(wm->sr.plane, SR) |
 943                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 944                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 945                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 946         intel_uncore_write(&dev_priv->uncore, DSPFW2,
 947                    (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 948                    FW_WM(wm->sr.fbc, FBC_SR) |
 949                    FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
 950                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
 951                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 952                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 953         intel_uncore_write(&dev_priv->uncore, DSPFW3,
 954                    (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
 955                    FW_WM(wm->sr.cursor, CURSOR_SR) |
 956                    FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
 957                    FW_WM(wm->hpll.plane, HPLL_SR));
 958
 959         intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
 960 }
 961
 962 #define FW_WM_VLV(value, plane) \
 963         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
 964
 965 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
 966                                 const struct vlv_wm_values *wm)
 967 {
 968         enum pipe pipe;
 969
 970         for_each_pipe(dev_priv, pipe) {
 971                 trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
 972
 973                 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
 974                            (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
 975                            (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
 976                            (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
 977                            (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
 978         }
 979
 980         /*
 981          * Zero the (unused) WM1 watermarks, and also clear all the
 982          * high order bits so that there are no out of bounds values
 983          * present in the registers during the reprogramming.
 984          */
 985         intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
 986         intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
 987         intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
 988         intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
 989         intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
 990
 991         intel_uncore_write(&dev_priv->uncore, DSPFW1,
 992                    FW_WM(wm->sr.plane, SR) |
 993                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 994                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 995                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 996         intel_uncore_write(&dev_priv->uncore, DSPFW2,
 997                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
 998                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 999                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1000         intel_uncore_write(&dev_priv->uncore, DSPFW3,
1001                    FW_WM(wm->sr.cursor, CURSOR_SR));
1002
1003         if (IS_CHERRYVIEW(dev_priv)) {
1004                 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1005                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1006                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1007                 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1008                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1009                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1010                 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1011                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1012                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1013                 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1014                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1015                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1016                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1017                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1018                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1019                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1020                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1021                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1022                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1023                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1024         } else {
1025                 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1026                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1027                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1028                 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1029                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1030                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1031                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1032                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1033                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1034                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1035                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1036         }
1037
1038         intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1039 }
1040
1041 #undef FW_WM_VLV
1042
1043 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1044 {
1045         /* all latencies in usec */
1046         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1047         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1048         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1049
1050         dev_priv->display.wm.max_level = G4X_WM_LEVEL_HPLL;
1051 }
1052
1053 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1054 {
1055         /*
1056          * DSPCNTR[13] supposedly controls whether the
1057          * primary plane can use the FIFO space otherwise
1058          * reserved for the sprite plane. It's not 100% clear
1059          * what the actual FIFO size is, but it looks like we
1060          * can happily set both primary and sprite watermarks
1061          * up to 127 cachelines. So that would seem to mean
1062          * that either DSPCNTR[13] doesn't do anything, or that
1063          * the total FIFO is >= 256 cachelines in size. Either
1064          * way, we don't seem to have to worry about this
1065          * repartitioning as the maximum watermark value the
1066          * register can hold for each plane is lower than the
1067          * minimum FIFO size.
1068          */
1069         switch (plane_id) {
1070         case PLANE_CURSOR:
1071                 return 63;
1072         case PLANE_PRIMARY:
1073                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1074         case PLANE_SPRITE0:
1075                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1076         default:
1077                 MISSING_CASE(plane_id);
1078                 return 0;
1079         }
1080 }
1081
1082 static int g4x_fbc_fifo_size(int level)
1083 {
1084         switch (level) {
1085         case G4X_WM_LEVEL_SR:
1086                 return 7;
1087         case G4X_WM_LEVEL_HPLL:
1088                 return 15;
1089         default:
1090                 MISSING_CASE(level);
1091                 return 0;
1092         }
1093 }
1094
1095 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1096                           const struct intel_plane_state *plane_state,
1097                           int level)
1098 {
1099         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1100         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1101         const struct drm_display_mode *pipe_mode =
1102                 &crtc_state->hw.pipe_mode;
1103         unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;
1104         unsigned int pixel_rate, htotal, cpp, width, wm;
1105
1106         if (latency == 0)
1107                 return USHRT_MAX;
1108
1109         if (!intel_wm_plane_visible(crtc_state, plane_state))
1110                 return 0;
1111
1112         cpp = plane_state->hw.fb->format->cpp[0];
1113
1114         /*
1115          * WaUse32BppForSRWM:ctg,elk
1116          *
1117          * The spec fails to list this restriction for the
1118          * HPLL watermark, which seems a little strange.
1119          * Let's use 32bpp for the HPLL watermark as well.
1120          */
1121         if (plane->id == PLANE_PRIMARY &&
1122             level != G4X_WM_LEVEL_NORMAL)
1123                 cpp = max(cpp, 4u);
1124
1125         pixel_rate = crtc_state->pixel_rate;
1126         htotal = pipe_mode->crtc_htotal;
1127         width = drm_rect_width(&plane_state->uapi.src) >> 16;
1128
1129         if (plane->id == PLANE_CURSOR) {
1130                 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1131         } else if (plane->id == PLANE_PRIMARY &&
1132                    level == G4X_WM_LEVEL_NORMAL) {
1133                 wm = intel_wm_method1(pixel_rate, cpp, latency);
1134         } else {
1135                 unsigned int small, large;
1136
1137                 small = intel_wm_method1(pixel_rate, cpp, latency);
1138                 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1139
1140                 wm = min(small, large);
1141         }
1142
1143         wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1144                               width, cpp);
1145
1146         wm = DIV_ROUND_UP(wm, 64) + 2;
1147
1148         return min_t(unsigned int, wm, USHRT_MAX);
1149 }
1150
1151 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1152                                  int level, enum plane_id plane_id, u16 value)
1153 {
1154         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1155         bool dirty = false;
1156
1157         for (; level < intel_wm_num_levels(dev_priv); level++) {
1158                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1159
1160                 dirty |= raw->plane[plane_id] != value;
1161                 raw->plane[plane_id] = value;
1162         }
1163
1164         return dirty;
1165 }
1166
1167 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1168                                int level, u16 value)
1169 {
1170         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1171         bool dirty = false;
1172
1173         /* NORMAL level doesn't have an FBC watermark */
1174         level = max(level, G4X_WM_LEVEL_SR);
1175
1176         for (; level < intel_wm_num_levels(dev_priv); level++) {
1177                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1178
1179                 dirty |= raw->fbc != value;
1180                 raw->fbc = value;
1181         }
1182
1183         return dirty;
1184 }
1185
1186 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1187                               const struct intel_plane_state *plane_state,
1188                               u32 pri_val);
1189
1190 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1191                                      const struct intel_plane_state *plane_state)
1192 {
1193         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1194         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1195         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1196         enum plane_id plane_id = plane->id;
1197         bool dirty = false;
1198         int level;
1199
1200         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1201                 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1202                 if (plane_id == PLANE_PRIMARY)
1203                         dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1204                 goto out;
1205         }
1206
1207         for (level = 0; level < num_levels; level++) {
1208                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1209                 int wm, max_wm;
1210
1211                 wm = g4x_compute_wm(crtc_state, plane_state, level);
1212                 max_wm = g4x_plane_fifo_size(plane_id, level);
1213
1214                 if (wm > max_wm)
1215                         break;
1216
1217                 dirty |= raw->plane[plane_id] != wm;
1218                 raw->plane[plane_id] = wm;
1219
1220                 if (plane_id != PLANE_PRIMARY ||
1221                     level == G4X_WM_LEVEL_NORMAL)
1222                         continue;
1223
1224                 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1225                                         raw->plane[plane_id]);
1226                 max_wm = g4x_fbc_fifo_size(level);
1227
1228                 /*
1229                  * FBC wm is not mandatory as we
1230                  * can always just disable its use.
1231                  */
1232                 if (wm > max_wm)
1233                         wm = USHRT_MAX;
1234
1235                 dirty |= raw->fbc != wm;
1236                 raw->fbc = wm;
1237         }
1238
1239         /* mark watermarks as invalid */
1240         dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1241
1242         if (plane_id == PLANE_PRIMARY)
1243                 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1244
1245  out:
1246         if (dirty) {
1247                 drm_dbg_kms(&dev_priv->drm,
1248                             "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1249                             plane->base.name,
1250                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1251                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1252                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1253
1254                 if (plane_id == PLANE_PRIMARY)
1255                         drm_dbg_kms(&dev_priv->drm,
1256                                     "FBC watermarks: SR=%d, HPLL=%d\n",
1257                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1258                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1259         }
1260
1261         return dirty;
1262 }
1263
1264 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1265                                       enum plane_id plane_id, int level)
1266 {
1267         const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1268
1269         return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1270 }
1271
1272 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1273                                      int level)
1274 {
1275         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1276
1277         if (level > dev_priv->display.wm.max_level)
1278                 return false;
1279
1280         return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1281                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1282                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1283 }
1284
1285 /* mark all levels starting from 'level' as invalid */
1286 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1287                                struct g4x_wm_state *wm_state, int level)
1288 {
1289         if (level <= G4X_WM_LEVEL_NORMAL) {
1290                 enum plane_id plane_id;
1291
1292                 for_each_plane_id_on_crtc(crtc, plane_id)
1293                         wm_state->wm.plane[plane_id] = USHRT_MAX;
1294         }
1295
1296         if (level <= G4X_WM_LEVEL_SR) {
1297                 wm_state->cxsr = false;
1298                 wm_state->sr.cursor = USHRT_MAX;
1299                 wm_state->sr.plane = USHRT_MAX;
1300                 wm_state->sr.fbc = USHRT_MAX;
1301         }
1302
1303         if (level <= G4X_WM_LEVEL_HPLL) {
1304                 wm_state->hpll_en = false;
1305                 wm_state->hpll.cursor = USHRT_MAX;
1306                 wm_state->hpll.plane = USHRT_MAX;
1307                 wm_state->hpll.fbc = USHRT_MAX;
1308         }
1309 }
1310
1311 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1312                                int level)
1313 {
1314         if (level < G4X_WM_LEVEL_SR)
1315                 return false;
1316
1317         if (level >= G4X_WM_LEVEL_SR &&
1318             wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1319                 return false;
1320
1321         if (level >= G4X_WM_LEVEL_HPLL &&
1322             wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1323                 return false;
1324
1325         return true;
1326 }
1327
1328 static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1329 {
1330         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1331         struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1332         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1333         const struct g4x_pipe_wm *raw;
1334         enum plane_id plane_id;
1335         int level;
1336
1337         level = G4X_WM_LEVEL_NORMAL;
1338         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1339                 goto out;
1340
1341         raw = &crtc_state->wm.g4x.raw[level];
1342         for_each_plane_id_on_crtc(crtc, plane_id)
1343                 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1344
1345         level = G4X_WM_LEVEL_SR;
1346         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1347                 goto out;
1348
1349         raw = &crtc_state->wm.g4x.raw[level];
1350         wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1351         wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1352         wm_state->sr.fbc = raw->fbc;
1353
1354         wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1355
1356         level = G4X_WM_LEVEL_HPLL;
1357         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1358                 goto out;
1359
1360         raw = &crtc_state->wm.g4x.raw[level];
1361         wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1362         wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1363         wm_state->hpll.fbc = raw->fbc;
1364
1365         wm_state->hpll_en = wm_state->cxsr;
1366
1367         level++;
1368
1369  out:
1370         if (level == G4X_WM_LEVEL_NORMAL)
1371                 return -EINVAL;
1372
1373         /* invalidate the higher levels */
1374         g4x_invalidate_wms(crtc, wm_state, level);
1375
1376         /*
1377          * Determine if the FBC watermark(s) can be used. IF
1378          * this isn't the case we prefer to disable the FBC
1379          * watermark(s) rather than disable the SR/HPLL
1380          * level(s) entirely. 'level-1' is the highest valid
1381          * level here.
1382          */
1383         wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1384
1385         return 0;
1386 }
1387
1388 static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1389                                struct intel_crtc *crtc)
1390 {
1391         struct intel_crtc_state *crtc_state =
1392                 intel_atomic_get_new_crtc_state(state, crtc);
1393         const struct intel_plane_state *old_plane_state;
1394         const struct intel_plane_state *new_plane_state;
1395         struct intel_plane *plane;
1396         unsigned int dirty = 0;
1397         int i;
1398
1399         for_each_oldnew_intel_plane_in_state(state, plane,
1400                                              old_plane_state,
1401                                              new_plane_state, i) {
1402                 if (new_plane_state->hw.crtc != &crtc->base &&
1403                     old_plane_state->hw.crtc != &crtc->base)
1404                         continue;
1405
1406                 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1407                         dirty |= BIT(plane->id);
1408         }
1409
1410         if (!dirty)
1411                 return 0;
1412
1413         return _g4x_compute_pipe_wm(crtc_state);
1414 }
1415
1416 static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1417                                        struct intel_crtc *crtc)
1418 {
1419         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1420         struct intel_crtc_state *new_crtc_state =
1421                 intel_atomic_get_new_crtc_state(state, crtc);
1422         const struct intel_crtc_state *old_crtc_state =
1423                 intel_atomic_get_old_crtc_state(state, crtc);
1424         struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1425         const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1426         const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1427         enum plane_id plane_id;
1428
1429         if (!new_crtc_state->hw.active ||
1430             intel_crtc_needs_modeset(new_crtc_state)) {
1431                 *intermediate = *optimal;
1432
1433                 intermediate->cxsr = false;
1434                 intermediate->hpll_en = false;
1435                 goto out;
1436         }
1437
1438         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1439                 !new_crtc_state->disable_cxsr;
1440         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1441                 !new_crtc_state->disable_cxsr;
1442         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1443
1444         for_each_plane_id_on_crtc(crtc, plane_id) {
1445                 intermediate->wm.plane[plane_id] =
1446                         max(optimal->wm.plane[plane_id],
1447                             active->wm.plane[plane_id]);
1448
1449                 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1450                             g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1451         }
1452
1453         intermediate->sr.plane = max(optimal->sr.plane,
1454                                      active->sr.plane);
1455         intermediate->sr.cursor = max(optimal->sr.cursor,
1456                                       active->sr.cursor);
1457         intermediate->sr.fbc = max(optimal->sr.fbc,
1458                                    active->sr.fbc);
1459
1460         intermediate->hpll.plane = max(optimal->hpll.plane,
1461                                        active->hpll.plane);
1462         intermediate->hpll.cursor = max(optimal->hpll.cursor,
1463                                         active->hpll.cursor);
1464         intermediate->hpll.fbc = max(optimal->hpll.fbc,
1465                                      active->hpll.fbc);
1466
1467         drm_WARN_ON(&dev_priv->drm,
1468                     (intermediate->sr.plane >
1469                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1470                      intermediate->sr.cursor >
1471                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1472                     intermediate->cxsr);
1473         drm_WARN_ON(&dev_priv->drm,
1474                     (intermediate->sr.plane >
1475                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1476                      intermediate->sr.cursor >
1477                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1478                     intermediate->hpll_en);
1479
1480         drm_WARN_ON(&dev_priv->drm,
1481                     intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1482                     intermediate->fbc_en && intermediate->cxsr);
1483         drm_WARN_ON(&dev_priv->drm,
1484                     intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1485                     intermediate->fbc_en && intermediate->hpll_en);
1486
1487 out:
1488         /*
1489          * If our intermediate WM are identical to the final WM, then we can
1490          * omit the post-vblank programming; only update if it's different.
1491          */
1492         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1493                 new_crtc_state->wm.need_postvbl_update = true;
1494
1495         return 0;
1496 }
1497
1498 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1499                          struct g4x_wm_values *wm)
1500 {
1501         struct intel_crtc *crtc;
1502         int num_active_pipes = 0;
1503
1504         wm->cxsr = true;
1505         wm->hpll_en = true;
1506         wm->fbc_en = true;
1507
1508         for_each_intel_crtc(&dev_priv->drm, crtc) {
1509                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1510
1511                 if (!crtc->active)
1512                         continue;
1513
1514                 if (!wm_state->cxsr)
1515                         wm->cxsr = false;
1516                 if (!wm_state->hpll_en)
1517                         wm->hpll_en = false;
1518                 if (!wm_state->fbc_en)
1519                         wm->fbc_en = false;
1520
1521                 num_active_pipes++;
1522         }
1523
1524         if (num_active_pipes != 1) {
1525                 wm->cxsr = false;
1526                 wm->hpll_en = false;
1527                 wm->fbc_en = false;
1528         }
1529
1530         for_each_intel_crtc(&dev_priv->drm, crtc) {
1531                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1532                 enum pipe pipe = crtc->pipe;
1533
1534                 wm->pipe[pipe] = wm_state->wm;
1535                 if (crtc->active && wm->cxsr)
1536                         wm->sr = wm_state->sr;
1537                 if (crtc->active && wm->hpll_en)
1538                         wm->hpll = wm_state->hpll;
1539         }
1540 }
1541
1542 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1543 {
1544         struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;
1545         struct g4x_wm_values new_wm = {};
1546
1547         g4x_merge_wm(dev_priv, &new_wm);
1548
1549         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1550                 return;
1551
1552         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1553                 _intel_set_memory_cxsr(dev_priv, false);
1554
1555         g4x_write_wm_values(dev_priv, &new_wm);
1556
1557         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1558                 _intel_set_memory_cxsr(dev_priv, true);
1559
1560         *old_wm = new_wm;
1561 }
1562
1563 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1564                                    struct intel_crtc *crtc)
1565 {
1566         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1567         const struct intel_crtc_state *crtc_state =
1568                 intel_atomic_get_new_crtc_state(state, crtc);
1569
1570         mutex_lock(&dev_priv->display.wm.wm_mutex);
1571         crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1572         g4x_program_watermarks(dev_priv);
1573         mutex_unlock(&dev_priv->display.wm.wm_mutex);
1574 }
1575
1576 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1577                                     struct intel_crtc *crtc)
1578 {
1579         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1580         const struct intel_crtc_state *crtc_state =
1581                 intel_atomic_get_new_crtc_state(state, crtc);
1582
1583         if (!crtc_state->wm.need_postvbl_update)
1584                 return;
1585
1586         mutex_lock(&dev_priv->display.wm.wm_mutex);
1587         crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1588         g4x_program_watermarks(dev_priv);
1589         mutex_unlock(&dev_priv->display.wm.wm_mutex);
1590 }
1591
1592 /* latency must be in 0.1us units. */
1593 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1594                                    unsigned int htotal,
1595                                    unsigned int width,
1596                                    unsigned int cpp,
1597                                    unsigned int latency)
1598 {
1599         unsigned int ret;
1600
1601         ret = intel_wm_method2(pixel_rate, htotal,
1602                                width, cpp, latency);
1603         ret = DIV_ROUND_UP(ret, 64);
1604
1605         return ret;
1606 }
1607
1608 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1609 {
1610         /* all latencies in usec */
1611         dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1612
1613         dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM2;
1614
1615         if (IS_CHERRYVIEW(dev_priv)) {
1616                 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1617                 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1618
1619                 dev_priv->display.wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1620         }
1621 }
1622
1623 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1624                                 const struct intel_plane_state *plane_state,
1625                                 int level)
1626 {
1627         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1628         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1629         const struct drm_display_mode *pipe_mode =
1630                 &crtc_state->hw.pipe_mode;
1631         unsigned int pixel_rate, htotal, cpp, width, wm;
1632
1633         if (dev_priv->display.wm.pri_latency[level] == 0)
1634                 return USHRT_MAX;
1635
1636         if (!intel_wm_plane_visible(crtc_state, plane_state))
1637                 return 0;
1638
1639         cpp = plane_state->hw.fb->format->cpp[0];
1640         pixel_rate = crtc_state->pixel_rate;
1641         htotal = pipe_mode->crtc_htotal;
1642         width = drm_rect_width(&plane_state->uapi.src) >> 16;
1643
1644         if (plane->id == PLANE_CURSOR) {
1645                 /*
1646                  * FIXME the formula gives values that are
1647                  * too big for the cursor FIFO, and hence we
1648                  * would never be able to use cursors. For
1649                  * now just hardcode the watermark.
1650                  */
1651                 wm = 63;
1652         } else {
1653                 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1654                                     dev_priv->display.wm.pri_latency[level] * 10);
1655         }
1656
1657         return min_t(unsigned int, wm, USHRT_MAX);
1658 }
1659
1660 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1661 {
1662         return (active_planes & (BIT(PLANE_SPRITE0) |
1663                                  BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1664 }
1665
1666 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1667 {
1668         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1669         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1670         const struct g4x_pipe_wm *raw =
1671                 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1672         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1673         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1674         int num_active_planes = hweight8(active_planes);
1675         const int fifo_size = 511;
1676         int fifo_extra, fifo_left = fifo_size;
1677         int sprite0_fifo_extra = 0;
1678         unsigned int total_rate;
1679         enum plane_id plane_id;
1680
1681         /*
1682          * When enabling sprite0 after sprite1 has already been enabled
1683          * we tend to get an underrun unless sprite0 already has some
1684          * FIFO space allcoated. Hence we always allocate at least one
1685          * cacheline for sprite0 whenever sprite1 is enabled.
1686          *
1687          * All other plane enable sequences appear immune to this problem.
1688          */
1689         if (vlv_need_sprite0_fifo_workaround(active_planes))
1690                 sprite0_fifo_extra = 1;
1691
1692         total_rate = raw->plane[PLANE_PRIMARY] +
1693                 raw->plane[PLANE_SPRITE0] +
1694                 raw->plane[PLANE_SPRITE1] +
1695                 sprite0_fifo_extra;
1696
1697         if (total_rate > fifo_size)
1698                 return -EINVAL;
1699
1700         if (total_rate == 0)
1701                 total_rate = 1;
1702
1703         for_each_plane_id_on_crtc(crtc, plane_id) {
1704                 unsigned int rate;
1705
1706                 if ((active_planes & BIT(plane_id)) == 0) {
1707                         fifo_state->plane[plane_id] = 0;
1708                         continue;
1709                 }
1710
1711                 rate = raw->plane[plane_id];
1712                 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1713                 fifo_left -= fifo_state->plane[plane_id];
1714         }
1715
1716         fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1717         fifo_left -= sprite0_fifo_extra;
1718
1719         fifo_state->plane[PLANE_CURSOR] = 63;
1720
1721         fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1722
1723         /* spread the remainder evenly */
1724         for_each_plane_id_on_crtc(crtc, plane_id) {
1725                 int plane_extra;
1726
1727                 if (fifo_left == 0)
1728                         break;
1729
1730                 if ((active_planes & BIT(plane_id)) == 0)
1731                         continue;
1732
1733                 plane_extra = min(fifo_extra, fifo_left);
1734                 fifo_state->plane[plane_id] += plane_extra;
1735                 fifo_left -= plane_extra;
1736         }
1737
1738         drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1739
1740         /* give it all to the first plane if none are active */
1741         if (active_planes == 0) {
1742                 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1743                 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1744         }
1745
1746         return 0;
1747 }
1748
1749 /* mark all levels starting from 'level' as invalid */
1750 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1751                                struct vlv_wm_state *wm_state, int level)
1752 {
1753         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1754
1755         for (; level < intel_wm_num_levels(dev_priv); level++) {
1756                 enum plane_id plane_id;
1757
1758                 for_each_plane_id_on_crtc(crtc, plane_id)
1759                         wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1760
1761                 wm_state->sr[level].cursor = USHRT_MAX;
1762                 wm_state->sr[level].plane = USHRT_MAX;
1763         }
1764 }
1765
1766 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1767 {
1768         if (wm > fifo_size)
1769                 return USHRT_MAX;
1770         else
1771                 return fifo_size - wm;
1772 }
1773
1774 /*
1775  * Starting from 'level' set all higher
1776  * levels to 'value' in the "raw" watermarks.
1777  */
1778 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1779                                  int level, enum plane_id plane_id, u16 value)
1780 {
1781         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1782         int num_levels = intel_wm_num_levels(dev_priv);
1783         bool dirty = false;
1784
1785         for (; level < num_levels; level++) {
1786                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1787
1788                 dirty |= raw->plane[plane_id] != value;
1789                 raw->plane[plane_id] = value;
1790         }
1791
1792         return dirty;
1793 }
1794
1795 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1796                                      const struct intel_plane_state *plane_state)
1797 {
1798         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1799         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1800         enum plane_id plane_id = plane->id;
1801         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1802         int level;
1803         bool dirty = false;
1804
1805         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1806                 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1807                 goto out;
1808         }
1809
1810         for (level = 0; level < num_levels; level++) {
1811                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1812                 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1813                 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1814
1815                 if (wm > max_wm)
1816                         break;
1817
1818                 dirty |= raw->plane[plane_id] != wm;
1819                 raw->plane[plane_id] = wm;
1820         }
1821
1822         /* mark all higher levels as invalid */
1823         dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1824
1825 out:
1826         if (dirty)
1827                 drm_dbg_kms(&dev_priv->drm,
1828                             "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1829                             plane->base.name,
1830                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1831                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1832                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1833
1834         return dirty;
1835 }
1836
1837 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1838                                       enum plane_id plane_id, int level)
1839 {
1840         const struct g4x_pipe_wm *raw =
1841                 &crtc_state->wm.vlv.raw[level];
1842         const struct vlv_fifo_state *fifo_state =
1843                 &crtc_state->wm.vlv.fifo_state;
1844
1845         return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1846 }
1847
1848 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1849 {
1850         return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1851                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1852                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1853                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1854 }
1855
1856 static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1857 {
1858         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1859         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1860         struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1861         const struct vlv_fifo_state *fifo_state =
1862                 &crtc_state->wm.vlv.fifo_state;
1863         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1864         int num_active_planes = hweight8(active_planes);
1865         enum plane_id plane_id;
1866         int level;
1867
1868         /* initially allow all levels */
1869         wm_state->num_levels = intel_wm_num_levels(dev_priv);
1870         /*
1871          * Note that enabling cxsr with no primary/sprite planes
1872          * enabled can wedge the pipe. Hence we only allow cxsr
1873          * with exactly one enabled primary/sprite plane.
1874          */
1875         wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1876
1877         for (level = 0; level < wm_state->num_levels; level++) {
1878                 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1879                 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1880
1881                 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1882                         break;
1883
1884                 for_each_plane_id_on_crtc(crtc, plane_id) {
1885                         wm_state->wm[level].plane[plane_id] =
1886                                 vlv_invert_wm_value(raw->plane[plane_id],
1887                                                     fifo_state->plane[plane_id]);
1888                 }
1889
1890                 wm_state->sr[level].plane =
1891                         vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1892                                                  raw->plane[PLANE_SPRITE0],
1893                                                  raw->plane[PLANE_SPRITE1]),
1894                                             sr_fifo_size);
1895
1896                 wm_state->sr[level].cursor =
1897                         vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1898                                             63);
1899         }
1900
1901         if (level == 0)
1902                 return -EINVAL;
1903
1904         /* limit to only levels we can actually handle */
1905         wm_state->num_levels = level;
1906
1907         /* invalidate the higher levels */
1908         vlv_invalidate_wms(crtc, wm_state, level);
1909
1910         return 0;
1911 }
1912
1913 static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1914                                struct intel_crtc *crtc)
1915 {
1916         struct intel_crtc_state *crtc_state =
1917                 intel_atomic_get_new_crtc_state(state, crtc);
1918         const struct intel_plane_state *old_plane_state;
1919         const struct intel_plane_state *new_plane_state;
1920         struct intel_plane *plane;
1921         unsigned int dirty = 0;
1922         int i;
1923
1924         for_each_oldnew_intel_plane_in_state(state, plane,
1925                                              old_plane_state,
1926                                              new_plane_state, i) {
1927                 if (new_plane_state->hw.crtc != &crtc->base &&
1928                     old_plane_state->hw.crtc != &crtc->base)
1929                         continue;
1930
1931                 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1932                         dirty |= BIT(plane->id);
1933         }
1934
1935         /*
1936          * DSPARB registers may have been reset due to the
1937          * power well being turned off. Make sure we restore
1938          * them to a consistent state even if no primary/sprite
1939          * planes are initially active. We also force a FIFO
1940          * recomputation so that we are sure to sanitize the
1941          * FIFO setting we took over from the BIOS even if there
1942          * are no active planes on the crtc.
1943          */
1944         if (intel_crtc_needs_modeset(crtc_state))
1945                 dirty = ~0;
1946
1947         if (!dirty)
1948                 return 0;
1949
1950         /* cursor changes don't warrant a FIFO recompute */
1951         if (dirty & ~BIT(PLANE_CURSOR)) {
1952                 const struct intel_crtc_state *old_crtc_state =
1953                         intel_atomic_get_old_crtc_state(state, crtc);
1954                 const struct vlv_fifo_state *old_fifo_state =
1955                         &old_crtc_state->wm.vlv.fifo_state;
1956                 const struct vlv_fifo_state *new_fifo_state =
1957                         &crtc_state->wm.vlv.fifo_state;
1958                 int ret;
1959
1960                 ret = vlv_compute_fifo(crtc_state);
1961                 if (ret)
1962                         return ret;
1963
1964                 if (intel_crtc_needs_modeset(crtc_state) ||
1965                     memcmp(old_fifo_state, new_fifo_state,
1966                            sizeof(*new_fifo_state)) != 0)
1967                         crtc_state->fifo_changed = true;
1968         }
1969
1970         return _vlv_compute_pipe_wm(crtc_state);
1971 }
1972
1973 #define VLV_FIFO(plane, value) \
1974         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1975
1976 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1977                                    struct intel_crtc *crtc)
1978 {
1979         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1980         struct intel_uncore *uncore = &dev_priv->uncore;
1981         const struct intel_crtc_state *crtc_state =
1982                 intel_atomic_get_new_crtc_state(state, crtc);
1983         const struct vlv_fifo_state *fifo_state =
1984                 &crtc_state->wm.vlv.fifo_state;
1985         int sprite0_start, sprite1_start, fifo_size;
1986         u32 dsparb, dsparb2, dsparb3;
1987
1988         if (!crtc_state->fifo_changed)
1989                 return;
1990
1991         sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1992         sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1993         fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1994
1995         drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1996         drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
1997
1998         trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1999
2000         /*
2001          * uncore.lock serves a double purpose here. It allows us to
2002          * use the less expensive I915_{READ,WRITE}_FW() functions, and
2003          * it protects the DSPARB registers from getting clobbered by
2004          * parallel updates from multiple pipes.
2005          *
2006          * intel_pipe_update_start() has already disabled interrupts
2007          * for us, so a plain spin_lock() is sufficient here.
2008          */
2009         spin_lock(&uncore->lock);
2010
2011         switch (crtc->pipe) {
2012         case PIPE_A:
2013                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2014                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2015
2016                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2017                             VLV_FIFO(SPRITEB, 0xff));
2018                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2019                            VLV_FIFO(SPRITEB, sprite1_start));
2020
2021                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2022                              VLV_FIFO(SPRITEB_HI, 0x1));
2023                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2024                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2025
2026                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2027                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2028                 break;
2029         case PIPE_B:
2030                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2031                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2032
2033                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2034                             VLV_FIFO(SPRITED, 0xff));
2035                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2036                            VLV_FIFO(SPRITED, sprite1_start));
2037
2038                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2039                              VLV_FIFO(SPRITED_HI, 0xff));
2040                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2041                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2042
2043                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2044                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2045                 break;
2046         case PIPE_C:
2047                 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2048                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2049
2050                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2051                              VLV_FIFO(SPRITEF, 0xff));
2052                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2053                             VLV_FIFO(SPRITEF, sprite1_start));
2054
2055                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2056                              VLV_FIFO(SPRITEF_HI, 0xff));
2057                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2058                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2059
2060                 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2061                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2062                 break;
2063         default:
2064                 break;
2065         }
2066
2067         intel_uncore_posting_read_fw(uncore, DSPARB);
2068
2069         spin_unlock(&uncore->lock);
2070 }
2071
2072 #undef VLV_FIFO
2073
2074 static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
2075                                        struct intel_crtc *crtc)
2076 {
2077         struct intel_crtc_state *new_crtc_state =
2078                 intel_atomic_get_new_crtc_state(state, crtc);
2079         const struct intel_crtc_state *old_crtc_state =
2080                 intel_atomic_get_old_crtc_state(state, crtc);
2081         struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2082         const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2083         const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2084         int level;
2085
2086         if (!new_crtc_state->hw.active ||
2087             intel_crtc_needs_modeset(new_crtc_state)) {
2088                 *intermediate = *optimal;
2089
2090                 intermediate->cxsr = false;
2091                 goto out;
2092         }
2093
2094         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2095         intermediate->cxsr = optimal->cxsr && active->cxsr &&
2096                 !new_crtc_state->disable_cxsr;
2097
2098         for (level = 0; level < intermediate->num_levels; level++) {
2099                 enum plane_id plane_id;
2100
2101                 for_each_plane_id_on_crtc(crtc, plane_id) {
2102                         intermediate->wm[level].plane[plane_id] =
2103                                 min(optimal->wm[level].plane[plane_id],
2104                                     active->wm[level].plane[plane_id]);
2105                 }
2106
2107                 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2108                                                     active->sr[level].plane);
2109                 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2110                                                      active->sr[level].cursor);
2111         }
2112
2113         vlv_invalidate_wms(crtc, intermediate, level);
2114
2115 out:
2116         /*
2117          * If our intermediate WM are identical to the final WM, then we can
2118          * omit the post-vblank programming; only update if it's different.
2119          */
2120         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2121                 new_crtc_state->wm.need_postvbl_update = true;
2122
2123         return 0;
2124 }
2125
2126 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2127                          struct vlv_wm_values *wm)
2128 {
2129         struct intel_crtc *crtc;
2130         int num_active_pipes = 0;
2131
2132         wm->level = dev_priv->display.wm.max_level;
2133         wm->cxsr = true;
2134
2135         for_each_intel_crtc(&dev_priv->drm, crtc) {
2136                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2137
2138                 if (!crtc->active)
2139                         continue;
2140
2141                 if (!wm_state->cxsr)
2142                         wm->cxsr = false;
2143
2144                 num_active_pipes++;
2145                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2146         }
2147
2148         if (num_active_pipes != 1)
2149                 wm->cxsr = false;
2150
2151         if (num_active_pipes > 1)
2152                 wm->level = VLV_WM_LEVEL_PM2;
2153
2154         for_each_intel_crtc(&dev_priv->drm, crtc) {
2155                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2156                 enum pipe pipe = crtc->pipe;
2157
2158                 wm->pipe[pipe] = wm_state->wm[wm->level];
2159                 if (crtc->active && wm->cxsr)
2160                         wm->sr = wm_state->sr[wm->level];
2161
2162                 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2163                 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2164                 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2165                 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2166         }
2167 }
2168
2169 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2170 {
2171         struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;
2172         struct vlv_wm_values new_wm = {};
2173
2174         vlv_merge_wm(dev_priv, &new_wm);
2175
2176         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2177                 return;
2178
2179         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2180                 chv_set_memory_dvfs(dev_priv, false);
2181
2182         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2183                 chv_set_memory_pm5(dev_priv, false);
2184
2185         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2186                 _intel_set_memory_cxsr(dev_priv, false);
2187
2188         vlv_write_wm_values(dev_priv, &new_wm);
2189
2190         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2191                 _intel_set_memory_cxsr(dev_priv, true);
2192
2193         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2194                 chv_set_memory_pm5(dev_priv, true);
2195
2196         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2197                 chv_set_memory_dvfs(dev_priv, true);
2198
2199         *old_wm = new_wm;
2200 }
2201
2202 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2203                                    struct intel_crtc *crtc)
2204 {
2205         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2206         const struct intel_crtc_state *crtc_state =
2207                 intel_atomic_get_new_crtc_state(state, crtc);
2208
2209         mutex_lock(&dev_priv->display.wm.wm_mutex);
2210         crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2211         vlv_program_watermarks(dev_priv);
2212         mutex_unlock(&dev_priv->display.wm.wm_mutex);
2213 }
2214
2215 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2216                                     struct intel_crtc *crtc)
2217 {
2218         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2219         const struct intel_crtc_state *crtc_state =
2220                 intel_atomic_get_new_crtc_state(state, crtc);
2221
2222         if (!crtc_state->wm.need_postvbl_update)
2223                 return;
2224
2225         mutex_lock(&dev_priv->display.wm.wm_mutex);
2226         crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2227         vlv_program_watermarks(dev_priv);
2228         mutex_unlock(&dev_priv->display.wm.wm_mutex);
2229 }
2230
2231 static void i965_update_wm(struct drm_i915_private *dev_priv)
2232 {
2233         struct intel_crtc *crtc;
2234         int srwm = 1;
2235         int cursor_sr = 16;
2236         bool cxsr_enabled;
2237
2238         /* Calc sr entries for one plane configs */
2239         crtc = single_enabled_crtc(dev_priv);
2240         if (crtc) {
2241                 /* self-refresh has much higher latency */
2242                 static const int sr_latency_ns = 12000;
2243                 const struct drm_display_mode *pipe_mode =
2244                         &crtc->config->hw.pipe_mode;
2245                 const struct drm_framebuffer *fb =
2246                         crtc->base.primary->state->fb;
2247                 int pixel_rate = crtc->config->pixel_rate;
2248                 int htotal = pipe_mode->crtc_htotal;
2249                 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2250                 int cpp = fb->format->cpp[0];
2251                 int entries;
2252
2253                 entries = intel_wm_method2(pixel_rate, htotal,
2254                                            width, cpp, sr_latency_ns / 100);
2255                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2256                 srwm = I965_FIFO_SIZE - entries;
2257                 if (srwm < 0)
2258                         srwm = 1;
2259                 srwm &= 0x1ff;
2260                 drm_dbg_kms(&dev_priv->drm,
2261                             "self-refresh entries: %d, wm: %d\n",
2262                             entries, srwm);
2263
2264                 entries = intel_wm_method2(pixel_rate, htotal,
2265                                            crtc->base.cursor->state->crtc_w, 4,
2266                                            sr_latency_ns / 100);
2267                 entries = DIV_ROUND_UP(entries,
2268                                        i965_cursor_wm_info.cacheline_size) +
2269                         i965_cursor_wm_info.guard_size;
2270
2271                 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2272                 if (cursor_sr > i965_cursor_wm_info.max_wm)
2273                         cursor_sr = i965_cursor_wm_info.max_wm;
2274
2275                 drm_dbg_kms(&dev_priv->drm,
2276                             "self-refresh watermark: display plane %d "
2277                             "cursor %d\n", srwm, cursor_sr);
2278
2279                 cxsr_enabled = true;
2280         } else {
2281                 cxsr_enabled = false;
2282                 /* Turn off self refresh if both pipes are enabled */
2283                 intel_set_memory_cxsr(dev_priv, false);
2284         }
2285
2286         drm_dbg_kms(&dev_priv->drm,
2287                     "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2288                     srwm);
2289
2290         /* 965 has limitations... */
2291         intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2292                    FW_WM(8, CURSORB) |
2293                    FW_WM(8, PLANEB) |
2294                    FW_WM(8, PLANEA));
2295         intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2296                    FW_WM(8, PLANEC_OLD));
2297         /* update cursor SR watermark */
2298         intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2299
2300         if (cxsr_enabled)
2301                 intel_set_memory_cxsr(dev_priv, true);
2302 }
2303
2304 #undef FW_WM
2305
2306 static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2307                                                enum i9xx_plane_id i9xx_plane)
2308 {
2309         struct intel_plane *plane;
2310
2311         for_each_intel_plane(&i915->drm, plane) {
2312                 if (plane->id == PLANE_PRIMARY &&
2313                     plane->i9xx_plane == i9xx_plane)
2314                         return intel_crtc_for_pipe(i915, plane->pipe);
2315         }
2316
2317         return NULL;
2318 }
2319
2320 static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2321 {
2322         const struct intel_watermark_params *wm_info;
2323         u32 fwater_lo;
2324         u32 fwater_hi;
2325         int cwm, srwm = 1;
2326         int fifo_size;
2327         int planea_wm, planeb_wm;
2328         struct intel_crtc *crtc;
2329
2330         if (IS_I945GM(dev_priv))
2331                 wm_info = &i945_wm_info;
2332         else if (DISPLAY_VER(dev_priv) != 2)
2333                 wm_info = &i915_wm_info;
2334         else
2335                 wm_info = &i830_a_wm_info;
2336
2337         if (DISPLAY_VER(dev_priv) == 2)
2338                 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2339         else
2340                 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2341         crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2342         if (intel_crtc_active(crtc)) {
2343                 const struct drm_framebuffer *fb =
2344                         crtc->base.primary->state->fb;
2345                 int cpp;
2346
2347                 if (DISPLAY_VER(dev_priv) == 2)
2348                         cpp = 4;
2349                 else
2350                         cpp = fb->format->cpp[0];
2351
2352                 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2353                                                wm_info, fifo_size, cpp,
2354                                                pessimal_latency_ns);
2355         } else {
2356                 planea_wm = fifo_size - wm_info->guard_size;
2357                 if (planea_wm > (long)wm_info->max_wm)
2358                         planea_wm = wm_info->max_wm;
2359         }
2360
2361         if (DISPLAY_VER(dev_priv) == 2)
2362                 wm_info = &i830_bc_wm_info;
2363
2364         if (DISPLAY_VER(dev_priv) == 2)
2365                 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2366         else
2367                 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2368         crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2369         if (intel_crtc_active(crtc)) {
2370                 const struct drm_framebuffer *fb =
2371                         crtc->base.primary->state->fb;
2372                 int cpp;
2373
2374                 if (DISPLAY_VER(dev_priv) == 2)
2375                         cpp = 4;
2376                 else
2377                         cpp = fb->format->cpp[0];
2378
2379                 planeb_wm = intel_calculate_wm(crtc->config->pixel_rate,
2380                                                wm_info, fifo_size, cpp,
2381                                                pessimal_latency_ns);
2382         } else {
2383                 planeb_wm = fifo_size - wm_info->guard_size;
2384                 if (planeb_wm > (long)wm_info->max_wm)
2385                         planeb_wm = wm_info->max_wm;
2386         }
2387
2388         drm_dbg_kms(&dev_priv->drm,
2389                     "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2390
2391         crtc = single_enabled_crtc(dev_priv);
2392         if (IS_I915GM(dev_priv) && crtc) {
2393                 struct drm_i915_gem_object *obj;
2394
2395                 obj = intel_fb_obj(crtc->base.primary->state->fb);
2396
2397                 /* self-refresh seems busted with untiled */
2398                 if (!i915_gem_object_is_tiled(obj))
2399                         crtc = NULL;
2400         }
2401
2402         /*
2403          * Overlay gets an aggressive default since video jitter is bad.
2404          */
2405         cwm = 2;
2406
2407         /* Play safe and disable self-refresh before adjusting watermarks. */
2408         intel_set_memory_cxsr(dev_priv, false);
2409
2410         /* Calc sr entries for one plane configs */
2411         if (HAS_FW_BLC(dev_priv) && crtc) {
2412                 /* self-refresh has much higher latency */
2413                 static const int sr_latency_ns = 6000;
2414                 const struct drm_display_mode *pipe_mode =
2415                         &crtc->config->hw.pipe_mode;
2416                 const struct drm_framebuffer *fb =
2417                         crtc->base.primary->state->fb;
2418                 int pixel_rate = crtc->config->pixel_rate;
2419                 int htotal = pipe_mode->crtc_htotal;
2420                 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2421                 int cpp;
2422                 int entries;
2423
2424                 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2425                         cpp = 4;
2426                 else
2427                         cpp = fb->format->cpp[0];
2428
2429                 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2430                                            sr_latency_ns / 100);
2431                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2432                 drm_dbg_kms(&dev_priv->drm,
2433                             "self-refresh entries: %d\n", entries);
2434                 srwm = wm_info->fifo_size - entries;
2435                 if (srwm < 0)
2436                         srwm = 1;
2437
2438                 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2439                         intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2440                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2441                 else
2442                         intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2443         }
2444
2445         drm_dbg_kms(&dev_priv->drm,
2446                     "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2447                      planea_wm, planeb_wm, cwm, srwm);
2448
2449         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2450         fwater_hi = (cwm & 0x1f);
2451
2452         /* Set request length to 8 cachelines per fetch */
2453         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2454         fwater_hi = fwater_hi | (1 << 8);
2455
2456         intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2457         intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2458
2459         if (crtc)
2460                 intel_set_memory_cxsr(dev_priv, true);
2461 }
2462
2463 static void i845_update_wm(struct drm_i915_private *dev_priv)
2464 {
2465         struct intel_crtc *crtc;
2466         u32 fwater_lo;
2467         int planea_wm;
2468
2469         crtc = single_enabled_crtc(dev_priv);
2470         if (crtc == NULL)
2471                 return;
2472
2473         planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2474                                        &i845_wm_info,
2475                                        i845_get_fifo_size(dev_priv, PLANE_A),
2476                                        4, pessimal_latency_ns);
2477         fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2478         fwater_lo |= (3<<8) | planea_wm;
2479
2480         drm_dbg_kms(&dev_priv->drm,
2481                     "Setting FIFO watermarks - A: %d\n", planea_wm);
2482
2483         intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2484 }
2485
2486 /* latency must be in 0.1us units. */
2487 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2488                                    unsigned int cpp,
2489                                    unsigned int latency)
2490 {
2491         unsigned int ret;
2492
2493         ret = intel_wm_method1(pixel_rate, cpp, latency);
2494         ret = DIV_ROUND_UP(ret, 64) + 2;
2495
2496         return ret;
2497 }
2498
2499 /* latency must be in 0.1us units. */
2500 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2501                                    unsigned int htotal,
2502                                    unsigned int width,
2503                                    unsigned int cpp,
2504                                    unsigned int latency)
2505 {
2506         unsigned int ret;
2507
2508         ret = intel_wm_method2(pixel_rate, htotal,
2509                                width, cpp, latency);
2510         ret = DIV_ROUND_UP(ret, 64) + 2;
2511
2512         return ret;
2513 }
2514
2515 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2516 {
2517         /*
2518          * Neither of these should be possible since this function shouldn't be
2519          * called if the CRTC is off or the plane is invisible.  But let's be
2520          * extra paranoid to avoid a potential divide-by-zero if we screw up
2521          * elsewhere in the driver.
2522          */
2523         if (WARN_ON(!cpp))
2524                 return 0;
2525         if (WARN_ON(!horiz_pixels))
2526                 return 0;
2527
2528         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2529 }
2530
2531 struct ilk_wm_maximums {
2532         u16 pri;
2533         u16 spr;
2534         u16 cur;
2535         u16 fbc;
2536 };
2537
2538 /*
2539  * For both WM_PIPE and WM_LP.
2540  * mem_value must be in 0.1us units.
2541  */
2542 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2543                               const struct intel_plane_state *plane_state,
2544                               u32 mem_value, bool is_lp)
2545 {
2546         u32 method1, method2;
2547         int cpp;
2548
2549         if (mem_value == 0)
2550                 return U32_MAX;
2551
2552         if (!intel_wm_plane_visible(crtc_state, plane_state))
2553                 return 0;
2554
2555         cpp = plane_state->hw.fb->format->cpp[0];
2556
2557         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2558
2559         if (!is_lp)
2560                 return method1;
2561
2562         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2563                                  crtc_state->hw.pipe_mode.crtc_htotal,
2564                                  drm_rect_width(&plane_state->uapi.src) >> 16,
2565                                  cpp, mem_value);
2566
2567         return min(method1, method2);
2568 }
2569
2570 /*
2571  * For both WM_PIPE and WM_LP.
2572  * mem_value must be in 0.1us units.
2573  */
2574 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2575                               const struct intel_plane_state *plane_state,
2576                               u32 mem_value)
2577 {
2578         u32 method1, method2;
2579         int cpp;
2580
2581         if (mem_value == 0)
2582                 return U32_MAX;
2583
2584         if (!intel_wm_plane_visible(crtc_state, plane_state))
2585                 return 0;
2586
2587         cpp = plane_state->hw.fb->format->cpp[0];
2588
2589         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2590         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2591                                  crtc_state->hw.pipe_mode.crtc_htotal,
2592                                  drm_rect_width(&plane_state->uapi.src) >> 16,
2593                                  cpp, mem_value);
2594         return min(method1, method2);
2595 }
2596
2597 /*
2598  * For both WM_PIPE and WM_LP.
2599  * mem_value must be in 0.1us units.
2600  */
2601 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2602                               const struct intel_plane_state *plane_state,
2603                               u32 mem_value)
2604 {
2605         int cpp;
2606
2607         if (mem_value == 0)
2608                 return U32_MAX;
2609
2610         if (!intel_wm_plane_visible(crtc_state, plane_state))
2611                 return 0;
2612
2613         cpp = plane_state->hw.fb->format->cpp[0];
2614
2615         return ilk_wm_method2(crtc_state->pixel_rate,
2616                               crtc_state->hw.pipe_mode.crtc_htotal,
2617                               drm_rect_width(&plane_state->uapi.src) >> 16,
2618                               cpp, mem_value);
2619 }
2620
2621 /* Only for WM_LP. */
2622 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2623                               const struct intel_plane_state *plane_state,
2624                               u32 pri_val)
2625 {
2626         int cpp;
2627
2628         if (!intel_wm_plane_visible(crtc_state, plane_state))
2629                 return 0;
2630
2631         cpp = plane_state->hw.fb->format->cpp[0];
2632
2633         return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2634                           cpp);
2635 }
2636
2637 static unsigned int
2638 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2639 {
2640         if (DISPLAY_VER(dev_priv) >= 8)
2641                 return 3072;
2642         else if (DISPLAY_VER(dev_priv) >= 7)
2643                 return 768;
2644         else
2645                 return 512;
2646 }
2647
2648 static unsigned int
2649 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2650                      int level, bool is_sprite)
2651 {
2652         if (DISPLAY_VER(dev_priv) >= 8)
2653                 /* BDW primary/sprite plane watermarks */
2654                 return level == 0 ? 255 : 2047;
2655         else if (DISPLAY_VER(dev_priv) >= 7)
2656                 /* IVB/HSW primary/sprite plane watermarks */
2657                 return level == 0 ? 127 : 1023;
2658         else if (!is_sprite)
2659                 /* ILK/SNB primary plane watermarks */
2660                 return level == 0 ? 127 : 511;
2661         else
2662                 /* ILK/SNB sprite plane watermarks */
2663                 return level == 0 ? 63 : 255;
2664 }
2665
2666 static unsigned int
2667 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2668 {
2669         if (DISPLAY_VER(dev_priv) >= 7)
2670                 return level == 0 ? 63 : 255;
2671         else
2672                 return level == 0 ? 31 : 63;
2673 }
2674
2675 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2676 {
2677         if (DISPLAY_VER(dev_priv) >= 8)
2678                 return 31;
2679         else
2680                 return 15;
2681 }
2682
2683 /* Calculate the maximum primary/sprite plane watermark */
2684 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2685                                      int level,
2686                                      const struct intel_wm_config *config,
2687                                      enum intel_ddb_partitioning ddb_partitioning,
2688                                      bool is_sprite)
2689 {
2690         unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2691
2692         /* if sprites aren't enabled, sprites get nothing */
2693         if (is_sprite && !config->sprites_enabled)
2694                 return 0;
2695
2696         /* HSW allows LP1+ watermarks even with multiple pipes */
2697         if (level == 0 || config->num_pipes_active > 1) {
2698                 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2699
2700                 /*
2701                  * For some reason the non self refresh
2702                  * FIFO size is only half of the self
2703                  * refresh FIFO size on ILK/SNB.
2704                  */
2705                 if (DISPLAY_VER(dev_priv) <= 6)
2706                         fifo_size /= 2;
2707         }
2708
2709         if (config->sprites_enabled) {
2710                 /* level 0 is always calculated with 1:1 split */
2711                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2712                         if (is_sprite)
2713                                 fifo_size *= 5;
2714                         fifo_size /= 6;
2715                 } else {
2716                         fifo_size /= 2;
2717                 }
2718         }
2719
2720         /* clamp to max that the registers can hold */
2721         return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2722 }
2723
2724 /* Calculate the maximum cursor plane watermark */
2725 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2726                                       int level,
2727                                       const struct intel_wm_config *config)
2728 {
2729         /* HSW LP1+ watermarks w/ multiple pipes */
2730         if (level > 0 && config->num_pipes_active > 1)
2731                 return 64;
2732
2733         /* otherwise just report max that registers can hold */
2734         return ilk_cursor_wm_reg_max(dev_priv, level);
2735 }
2736
2737 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2738                                     int level,
2739                                     const struct intel_wm_config *config,
2740                                     enum intel_ddb_partitioning ddb_partitioning,
2741                                     struct ilk_wm_maximums *max)
2742 {
2743         max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2744         max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2745         max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2746         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2747 }
2748
2749 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2750                                         int level,
2751                                         struct ilk_wm_maximums *max)
2752 {
2753         max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2754         max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2755         max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2756         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2757 }
2758
2759 static bool ilk_validate_wm_level(int level,
2760                                   const struct ilk_wm_maximums *max,
2761                                   struct intel_wm_level *result)
2762 {
2763         bool ret;
2764
2765         /* already determined to be invalid? */
2766         if (!result->enable)
2767                 return false;
2768
2769         result->enable = result->pri_val <= max->pri &&
2770                          result->spr_val <= max->spr &&
2771                          result->cur_val <= max->cur;
2772
2773         ret = result->enable;
2774
2775         /*
2776          * HACK until we can pre-compute everything,
2777          * and thus fail gracefully if LP0 watermarks
2778          * are exceeded...
2779          */
2780         if (level == 0 && !result->enable) {
2781                 if (result->pri_val > max->pri)
2782                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2783                                       level, result->pri_val, max->pri);
2784                 if (result->spr_val > max->spr)
2785                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2786                                       level, result->spr_val, max->spr);
2787                 if (result->cur_val > max->cur)
2788                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2789                                       level, result->cur_val, max->cur);
2790
2791                 result->pri_val = min_t(u32, result->pri_val, max->pri);
2792                 result->spr_val = min_t(u32, result->spr_val, max->spr);
2793                 result->cur_val = min_t(u32, result->cur_val, max->cur);
2794                 result->enable = true;
2795         }
2796
2797         return ret;
2798 }
2799
2800 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2801                                  const struct intel_crtc *crtc,
2802                                  int level,
2803                                  struct intel_crtc_state *crtc_state,
2804                                  const struct intel_plane_state *pristate,
2805                                  const struct intel_plane_state *sprstate,
2806                                  const struct intel_plane_state *curstate,
2807                                  struct intel_wm_level *result)
2808 {
2809         u16 pri_latency = dev_priv->display.wm.pri_latency[level];
2810         u16 spr_latency = dev_priv->display.wm.spr_latency[level];
2811         u16 cur_latency = dev_priv->display.wm.cur_latency[level];
2812
2813         /* WM1+ latency values stored in 0.5us units */
2814         if (level > 0) {
2815                 pri_latency *= 5;
2816                 spr_latency *= 5;
2817                 cur_latency *= 5;
2818         }
2819
2820         if (pristate) {
2821                 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2822                                                      pri_latency, level);
2823                 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2824         }
2825
2826         if (sprstate)
2827                 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2828
2829         if (curstate)
2830                 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2831
2832         result->enable = true;
2833 }
2834
2835 static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2836 {
2837         u64 sskpd;
2838
2839         sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);
2840
2841         wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2842         if (wm[0] == 0)
2843                 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2844         wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2845         wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2846         wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2847         wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2848 }
2849
2850 static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2851 {
2852         u32 sskpd;
2853
2854         sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);
2855
2856         wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2857         wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2858         wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2859         wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2860 }
2861
2862 static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2863 {
2864         u32 mltr;
2865
2866         mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);
2867
2868         /* ILK primary LP0 latency is 700 ns */
2869         wm[0] = 7;
2870         wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2871         wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2872 }
2873
2874 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2875                                        u16 wm[5])
2876 {
2877         /* ILK sprite LP0 latency is 1300 ns */
2878         if (DISPLAY_VER(dev_priv) == 5)
2879                 wm[0] = 13;
2880 }
2881
2882 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2883                                        u16 wm[5])
2884 {
2885         /* ILK cursor LP0 latency is 1300 ns */
2886         if (DISPLAY_VER(dev_priv) == 5)
2887                 wm[0] = 13;
2888 }
2889
2890 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2891 {
2892         /* how many WM levels are we expecting */
2893         if (HAS_HW_SAGV_WM(dev_priv))
2894                 return 5;
2895         else if (DISPLAY_VER(dev_priv) >= 9)
2896                 return 7;
2897         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2898                 return 4;
2899         else if (DISPLAY_VER(dev_priv) >= 6)
2900                 return 3;
2901         else
2902                 return 2;
2903 }
2904
2905 void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2906                             const char *name, const u16 wm[])
2907 {
2908         int level, max_level = ilk_wm_max_level(dev_priv);
2909
2910         for (level = 0; level <= max_level; level++) {
2911                 unsigned int latency = wm[level];
2912
2913                 if (latency == 0) {
2914                         drm_dbg_kms(&dev_priv->drm,
2915                                     "%s WM%d latency not provided\n",
2916                                     name, level);
2917                         continue;
2918                 }
2919
2920                 /*
2921                  * - latencies are in us on gen9.
2922                  * - before then, WM1+ latency values are in 0.5us units
2923                  */
2924                 if (DISPLAY_VER(dev_priv) >= 9)
2925                         latency *= 10;
2926                 else if (level > 0)
2927                         latency *= 5;
2928
2929                 drm_dbg_kms(&dev_priv->drm,
2930                             "%s WM%d latency %u (%u.%u usec)\n", name, level,
2931                             wm[level], latency / 10, latency % 10);
2932         }
2933 }
2934
2935 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2936                                     u16 wm[5], u16 min)
2937 {
2938         int level, max_level = ilk_wm_max_level(dev_priv);
2939
2940         if (wm[0] >= min)
2941                 return false;
2942
2943         wm[0] = max(wm[0], min);
2944         for (level = 1; level <= max_level; level++)
2945                 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2946
2947         return true;
2948 }
2949
2950 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2951 {
2952         bool changed;
2953
2954         /*
2955          * The BIOS provided WM memory latency values are often
2956          * inadequate for high resolution displays. Adjust them.
2957          */
2958         changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);
2959         changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);
2960         changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);
2961
2962         if (!changed)
2963                 return;
2964
2965         drm_dbg_kms(&dev_priv->drm,
2966                     "WM latency values increased to avoid potential underruns\n");
2967         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2968         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2969         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2970 }
2971
2972 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
2973 {
2974         /*
2975          * On some SNB machines (Thinkpad X220 Tablet at least)
2976          * LP3 usage can cause vblank interrupts to be lost.
2977          * The DEIIR bit will go high but it looks like the CPU
2978          * never gets interrupted.
2979          *
2980          * It's not clear whether other interrupt source could
2981          * be affected or if this is somehow limited to vblank
2982          * interrupts only. To play it safe we disable LP3
2983          * watermarks entirely.
2984          */
2985         if (dev_priv->display.wm.pri_latency[3] == 0 &&
2986             dev_priv->display.wm.spr_latency[3] == 0 &&
2987             dev_priv->display.wm.cur_latency[3] == 0)
2988                 return;
2989
2990         dev_priv->display.wm.pri_latency[3] = 0;
2991         dev_priv->display.wm.spr_latency[3] = 0;
2992         dev_priv->display.wm.cur_latency[3] = 0;
2993
2994         drm_dbg_kms(&dev_priv->drm,
2995                     "LP3 watermarks disabled due to potential for lost interrupts\n");
2996         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2997         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2998         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2999 }
3000
3001 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3002 {
3003         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
3004                 hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3005         else if (DISPLAY_VER(dev_priv) >= 6)
3006                 snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3007         else
3008                 ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3009
3010         memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,
3011                sizeof(dev_priv->display.wm.pri_latency));
3012         memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,
3013                sizeof(dev_priv->display.wm.pri_latency));
3014
3015         intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);
3016         intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);
3017
3018         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
3019         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
3020         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
3021
3022         if (DISPLAY_VER(dev_priv) == 6) {
3023                 snb_wm_latency_quirk(dev_priv);
3024                 snb_wm_lp3_irq_quirk(dev_priv);
3025         }
3026 }
3027
3028 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3029                                  struct intel_pipe_wm *pipe_wm)
3030 {
3031         /* LP0 watermark maximums depend on this pipe alone */
3032         const struct intel_wm_config config = {
3033                 .num_pipes_active = 1,
3034                 .sprites_enabled = pipe_wm->sprites_enabled,
3035                 .sprites_scaled = pipe_wm->sprites_scaled,
3036         };
3037         struct ilk_wm_maximums max;
3038
3039         /* LP0 watermarks always use 1/2 DDB partitioning */
3040         ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3041
3042         /* At least LP0 must be valid */
3043         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3044                 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3045                 return false;
3046         }
3047
3048         return true;
3049 }
3050
3051 /* Compute new watermarks for the pipe */
3052 static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
3053                                struct intel_crtc *crtc)
3054 {
3055         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3056         struct intel_crtc_state *crtc_state =
3057                 intel_atomic_get_new_crtc_state(state, crtc);
3058         struct intel_pipe_wm *pipe_wm;
3059         struct intel_plane *plane;
3060         const struct intel_plane_state *plane_state;
3061         const struct intel_plane_state *pristate = NULL;
3062         const struct intel_plane_state *sprstate = NULL;
3063         const struct intel_plane_state *curstate = NULL;
3064         int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3065         struct ilk_wm_maximums max;
3066
3067         pipe_wm = &crtc_state->wm.ilk.optimal;
3068
3069         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3070                 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3071                         pristate = plane_state;
3072                 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3073                         sprstate = plane_state;
3074                 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3075                         curstate = plane_state;
3076         }
3077
3078         pipe_wm->pipe_enabled = crtc_state->hw.active;
3079         pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
3080         pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
3081
3082         usable_level = max_level;
3083
3084         /* ILK/SNB: LP2+ watermarks only w/o sprites */
3085         if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3086                 usable_level = 1;
3087
3088         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3089         if (pipe_wm->sprites_scaled)
3090                 usable_level = 0;
3091
3092         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3093         ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3094                              pristate, sprstate, curstate, &pipe_wm->wm[0]);
3095
3096         if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3097                 return -EINVAL;
3098
3099         ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3100
3101         for (level = 1; level <= usable_level; level++) {
3102                 struct intel_wm_level *wm = &pipe_wm->wm[level];
3103
3104                 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3105                                      pristate, sprstate, curstate, wm);
3106
3107                 /*
3108                  * Disable any watermark level that exceeds the
3109                  * register maximums since such watermarks are
3110                  * always invalid.
3111                  */
3112                 if (!ilk_validate_wm_level(level, &max, wm)) {
3113                         memset(wm, 0, sizeof(*wm));
3114                         break;
3115                 }
3116         }
3117
3118         return 0;
3119 }
3120
3121 /*
3122  * Build a set of 'intermediate' watermark values that satisfy both the old
3123  * state and the new state.  These can be programmed to the hardware
3124  * immediately.
3125  */
3126 static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
3127                                        struct intel_crtc *crtc)
3128 {
3129         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3130         struct intel_crtc_state *new_crtc_state =
3131                 intel_atomic_get_new_crtc_state(state, crtc);
3132         const struct intel_crtc_state *old_crtc_state =
3133                 intel_atomic_get_old_crtc_state(state, crtc);
3134         struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
3135         const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
3136         int level, max_level = ilk_wm_max_level(dev_priv);
3137
3138         /*
3139          * Start with the final, target watermarks, then combine with the
3140          * currently active watermarks to get values that are safe both before
3141          * and after the vblank.
3142          */
3143         *a = new_crtc_state->wm.ilk.optimal;
3144         if (!new_crtc_state->hw.active ||
3145             intel_crtc_needs_modeset(new_crtc_state) ||
3146             state->skip_intermediate_wm)
3147                 return 0;
3148
3149         a->pipe_enabled |= b->pipe_enabled;
3150         a->sprites_enabled |= b->sprites_enabled;
3151         a->sprites_scaled |= b->sprites_scaled;
3152
3153         for (level = 0; level <= max_level; level++) {
3154                 struct intel_wm_level *a_wm = &a->wm[level];
3155                 const struct intel_wm_level *b_wm = &b->wm[level];
3156
3157                 a_wm->enable &= b_wm->enable;
3158                 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3159                 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3160                 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3161                 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3162         }
3163
3164         /*
3165          * We need to make sure that these merged watermark values are
3166          * actually a valid configuration themselves.  If they're not,
3167          * there's no safe way to transition from the old state to
3168          * the new state, so we need to fail the atomic transaction.
3169          */
3170         if (!ilk_validate_pipe_wm(dev_priv, a))
3171                 return -EINVAL;
3172
3173         /*
3174          * If our intermediate WM are identical to the final WM, then we can
3175          * omit the post-vblank programming; only update if it's different.
3176          */
3177         if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
3178                 new_crtc_state->wm.need_postvbl_update = true;
3179
3180         return 0;
3181 }
3182
3183 /*
3184  * Merge the watermarks from all active pipes for a specific level.
3185  */
3186 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3187                                int level,
3188                                struct intel_wm_level *ret_wm)
3189 {
3190         const struct intel_crtc *crtc;
3191
3192         ret_wm->enable = true;
3193
3194         for_each_intel_crtc(&dev_priv->drm, crtc) {
3195                 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3196                 const struct intel_wm_level *wm = &active->wm[level];
3197
3198                 if (!active->pipe_enabled)
3199                         continue;
3200
3201                 /*
3202                  * The watermark values may have been used in the past,
3203                  * so we must maintain them in the registers for some
3204                  * time even if the level is now disabled.
3205                  */
3206                 if (!wm->enable)
3207                         ret_wm->enable = false;
3208
3209                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3210                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3211                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3212                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3213         }
3214 }
3215
3216 /*
3217  * Merge all low power watermarks for all active pipes.
3218  */
3219 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3220                          const struct intel_wm_config *config,
3221                          const struct ilk_wm_maximums *max,
3222                          struct intel_pipe_wm *merged)
3223 {
3224         int level, max_level = ilk_wm_max_level(dev_priv);
3225         int last_enabled_level = max_level;
3226
3227         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3228         if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3229             config->num_pipes_active > 1)
3230                 last_enabled_level = 0;
3231
3232         /* ILK: FBC WM must be disabled always */
3233         merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3234
3235         /* merge each WM1+ level */
3236         for (level = 1; level <= max_level; level++) {
3237                 struct intel_wm_level *wm = &merged->wm[level];
3238
3239                 ilk_merge_wm_level(dev_priv, level, wm);
3240
3241                 if (level > last_enabled_level)
3242                         wm->enable = false;
3243                 else if (!ilk_validate_wm_level(level, max, wm))
3244                         /* make sure all following levels get disabled */
3245                         last_enabled_level = level - 1;
3246
3247                 /*
3248                  * The spec says it is preferred to disable
3249                  * FBC WMs instead of disabling a WM level.
3250                  */
3251                 if (wm->fbc_val > max->fbc) {
3252                         if (wm->enable)
3253                                 merged->fbc_wm_enabled = false;
3254                         wm->fbc_val = 0;
3255                 }
3256         }
3257
3258         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3259         if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
3260             dev_priv->params.enable_fbc && !merged->fbc_wm_enabled) {
3261                 for (level = 2; level <= max_level; level++) {
3262                         struct intel_wm_level *wm = &merged->wm[level];
3263
3264                         wm->enable = false;
3265                 }
3266         }
3267 }
3268
3269 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3270 {
3271         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3272         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3273 }
3274
3275 /* The value we need to program into the WM_LPx latency field */
3276 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3277                                       int level)
3278 {
3279         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3280                 return 2 * level;
3281         else
3282                 return dev_priv->display.wm.pri_latency[level];
3283 }
3284
3285 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3286                                    const struct intel_pipe_wm *merged,
3287                                    enum intel_ddb_partitioning partitioning,
3288                                    struct ilk_wm_values *results)
3289 {
3290         struct intel_crtc *crtc;
3291         int level, wm_lp;
3292
3293         results->enable_fbc_wm = merged->fbc_wm_enabled;
3294         results->partitioning = partitioning;
3295
3296         /* LP1+ register values */
3297         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3298                 const struct intel_wm_level *r;
3299
3300                 level = ilk_wm_lp_to_level(wm_lp, merged);
3301
3302                 r = &merged->wm[level];
3303
3304                 /*
3305                  * Maintain the watermark values even if the level is
3306                  * disabled. Doing otherwise could cause underruns.
3307                  */
3308                 results->wm_lp[wm_lp - 1] =
3309                         WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3310                         WM_LP_PRIMARY(r->pri_val) |
3311                         WM_LP_CURSOR(r->cur_val);
3312
3313                 if (r->enable)
3314                         results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3315
3316                 if (DISPLAY_VER(dev_priv) >= 8)
3317                         results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3318                 else
3319                         results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3320
3321                 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3322
3323                 /*
3324                  * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3325                  * level is disabled. Doing otherwise could cause underruns.
3326                  */
3327                 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3328                         drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3329                         results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3330                 }
3331         }
3332
3333         /* LP0 register values */
3334         for_each_intel_crtc(&dev_priv->drm, crtc) {
3335                 enum pipe pipe = crtc->pipe;
3336                 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3337                 const struct intel_wm_level *r = &pipe_wm->wm[0];
3338
3339                 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3340                         continue;
3341
3342                 results->wm_pipe[pipe] =
3343                         WM0_PIPE_PRIMARY(r->pri_val) |
3344                         WM0_PIPE_SPRITE(r->spr_val) |
3345                         WM0_PIPE_CURSOR(r->cur_val);
3346         }
3347 }
3348
3349 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3350  * case both are at the same level. Prefer r1 in case they're the same. */
3351 static struct intel_pipe_wm *
3352 ilk_find_best_result(struct drm_i915_private *dev_priv,
3353                      struct intel_pipe_wm *r1,
3354                      struct intel_pipe_wm *r2)
3355 {
3356         int level, max_level = ilk_wm_max_level(dev_priv);
3357         int level1 = 0, level2 = 0;
3358
3359         for (level = 1; level <= max_level; level++) {
3360                 if (r1->wm[level].enable)
3361                         level1 = level;
3362                 if (r2->wm[level].enable)
3363                         level2 = level;
3364         }
3365
3366         if (level1 == level2) {
3367                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3368                         return r2;
3369                 else
3370                         return r1;
3371         } else if (level1 > level2) {
3372                 return r1;
3373         } else {
3374                 return r2;
3375         }
3376 }
3377
3378 /* dirty bits used to track which watermarks need changes */
3379 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3380 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3381 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3382 #define WM_DIRTY_FBC (1 << 24)
3383 #define WM_DIRTY_DDB (1 << 25)
3384
3385 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3386                                          const struct ilk_wm_values *old,
3387                                          const struct ilk_wm_values *new)
3388 {
3389         unsigned int dirty = 0;
3390         enum pipe pipe;
3391         int wm_lp;
3392
3393         for_each_pipe(dev_priv, pipe) {
3394                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3395                         dirty |= WM_DIRTY_PIPE(pipe);
3396                         /* Must disable LP1+ watermarks too */
3397                         dirty |= WM_DIRTY_LP_ALL;
3398                 }
3399         }
3400
3401         if (old->enable_fbc_wm != new->enable_fbc_wm) {
3402                 dirty |= WM_DIRTY_FBC;
3403                 /* Must disable LP1+ watermarks too */
3404                 dirty |= WM_DIRTY_LP_ALL;
3405         }
3406
3407         if (old->partitioning != new->partitioning) {
3408                 dirty |= WM_DIRTY_DDB;
3409                 /* Must disable LP1+ watermarks too */
3410                 dirty |= WM_DIRTY_LP_ALL;
3411         }
3412
3413         /* LP1+ watermarks already deemed dirty, no need to continue */
3414         if (dirty & WM_DIRTY_LP_ALL)
3415                 return dirty;
3416
3417         /* Find the lowest numbered LP1+ watermark in need of an update... */
3418         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3419                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3420                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3421                         break;
3422         }
3423
3424         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3425         for (; wm_lp <= 3; wm_lp++)
3426                 dirty |= WM_DIRTY_LP(wm_lp);
3427
3428         return dirty;
3429 }
3430
3431 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3432                                unsigned int dirty)
3433 {
3434         struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3435         bool changed = false;
3436
3437         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3438                 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3439                 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3440                 changed = true;
3441         }
3442         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3443                 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3444                 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3445                 changed = true;
3446         }
3447         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3448                 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3449                 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3450                 changed = true;
3451         }
3452
3453         /*
3454          * Don't touch WM_LP_SPRITE_ENABLE here.
3455          * Doing so could cause underruns.
3456          */
3457
3458         return changed;
3459 }
3460
3461 /*
3462  * The spec says we shouldn't write when we don't need, because every write
3463  * causes WMs to be re-evaluated, expending some power.
3464  */
3465 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3466                                 struct ilk_wm_values *results)
3467 {
3468         struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3469         unsigned int dirty;
3470
3471         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3472         if (!dirty)
3473                 return;
3474
3475         _ilk_disable_lp_wm(dev_priv, dirty);
3476
3477         if (dirty & WM_DIRTY_PIPE(PIPE_A))
3478                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3479         if (dirty & WM_DIRTY_PIPE(PIPE_B))
3480                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3481         if (dirty & WM_DIRTY_PIPE(PIPE_C))
3482                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3483
3484         if (dirty & WM_DIRTY_DDB) {
3485                 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3486                         intel_uncore_rmw(&dev_priv->uncore, WM_MISC, WM_MISC_DATA_PARTITION_5_6,
3487                                          results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3488                                          WM_MISC_DATA_PARTITION_5_6);
3489                 else
3490                         intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,
3491                                          results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3492                                          DISP_DATA_PARTITION_5_6);
3493         }
3494
3495         if (dirty & WM_DIRTY_FBC)
3496                 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, DISP_FBC_WM_DIS,
3497                                  results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);
3498
3499         if (dirty & WM_DIRTY_LP(1) &&
3500             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3501                 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3502
3503         if (DISPLAY_VER(dev_priv) >= 7) {
3504                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3505                         intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3506                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3507                         intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3508         }
3509
3510         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3511                 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3512         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3513                 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3514         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3515                 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3516
3517         dev_priv->display.wm.hw = *results;
3518 }
3519
3520 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3521 {
3522         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3523 }
3524
3525 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
3526                                   struct intel_wm_config *config)
3527 {
3528         struct intel_crtc *crtc;
3529
3530         /* Compute the currently _active_ config */
3531         for_each_intel_crtc(&dev_priv->drm, crtc) {
3532                 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3533
3534                 if (!wm->pipe_enabled)
3535                         continue;
3536
3537                 config->sprites_enabled |= wm->sprites_enabled;
3538                 config->sprites_scaled |= wm->sprites_scaled;
3539                 config->num_pipes_active++;
3540         }
3541 }
3542
3543 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
3544 {
3545         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3546         struct ilk_wm_maximums max;
3547         struct intel_wm_config config = {};
3548         struct ilk_wm_values results = {};
3549         enum intel_ddb_partitioning partitioning;
3550
3551         ilk_compute_wm_config(dev_priv, &config);
3552
3553         ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
3554         ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
3555
3556         /* 5/6 split only in single pipe config on IVB+ */
3557         if (DISPLAY_VER(dev_priv) >= 7 &&
3558             config.num_pipes_active == 1 && config.sprites_enabled) {
3559                 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
3560                 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
3561
3562                 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
3563         } else {
3564                 best_lp_wm = &lp_wm_1_2;
3565         }
3566
3567         partitioning = (best_lp_wm == &lp_wm_1_2) ?
3568                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3569
3570         ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
3571
3572         ilk_write_wm_values(dev_priv, &results);
3573 }
3574
3575 static void ilk_initial_watermarks(struct intel_atomic_state *state,
3576                                    struct intel_crtc *crtc)
3577 {
3578         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3579         const struct intel_crtc_state *crtc_state =
3580                 intel_atomic_get_new_crtc_state(state, crtc);
3581
3582         mutex_lock(&dev_priv->display.wm.wm_mutex);
3583         crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
3584         ilk_program_watermarks(dev_priv);
3585         mutex_unlock(&dev_priv->display.wm.wm_mutex);
3586 }
3587
3588 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
3589                                     struct intel_crtc *crtc)
3590 {
3591         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3592         const struct intel_crtc_state *crtc_state =
3593                 intel_atomic_get_new_crtc_state(state, crtc);
3594
3595         if (!crtc_state->wm.need_postvbl_update)
3596                 return;
3597
3598         mutex_lock(&dev_priv->display.wm.wm_mutex);
3599         crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
3600         ilk_program_watermarks(dev_priv);
3601         mutex_unlock(&dev_priv->display.wm.wm_mutex);
3602 }
3603
3604 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
3605 {
3606         struct drm_device *dev = crtc->base.dev;
3607         struct drm_i915_private *dev_priv = to_i915(dev);
3608         struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3609         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
3610         struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
3611         enum pipe pipe = crtc->pipe;
3612
3613         hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
3614
3615         memset(active, 0, sizeof(*active));
3616
3617         active->pipe_enabled = crtc->active;
3618
3619         if (active->pipe_enabled) {
3620                 u32 tmp = hw->wm_pipe[pipe];
3621
3622                 /*
3623                  * For active pipes LP0 watermark is marked as
3624                  * enabled, and LP1+ watermaks as disabled since
3625                  * we can't really reverse compute them in case
3626                  * multiple pipes are active.
3627                  */
3628                 active->wm[0].enable = true;
3629                 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
3630                 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
3631                 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
3632         } else {
3633                 int level, max_level = ilk_wm_max_level(dev_priv);
3634
3635                 /*
3636                  * For inactive pipes, all watermark levels
3637                  * should be marked as enabled but zeroed,
3638                  * which is what we'd compute them to.
3639                  */
3640                 for (level = 0; level <= max_level; level++)
3641                         active->wm[level].enable = true;
3642         }
3643
3644         crtc->wm.active.ilk = *active;
3645 }
3646
3647 #define _FW_WM(value, plane) \
3648         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3649 #define _FW_WM_VLV(value, plane) \
3650         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3651
3652 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
3653                                struct g4x_wm_values *wm)
3654 {
3655         u32 tmp;
3656
3657         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3658         wm->sr.plane = _FW_WM(tmp, SR);
3659         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3660         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
3661         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
3662
3663         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3664         wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
3665         wm->sr.fbc = _FW_WM(tmp, FBC_SR);
3666         wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
3667         wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
3668         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3669         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
3670
3671         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3672         wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
3673         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3674         wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
3675         wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
3676 }
3677
3678 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
3679                                struct vlv_wm_values *wm)
3680 {
3681         enum pipe pipe;
3682         u32 tmp;
3683
3684         for_each_pipe(dev_priv, pipe) {
3685                 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
3686
3687                 wm->ddl[pipe].plane[PLANE_PRIMARY] =
3688                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3689                 wm->ddl[pipe].plane[PLANE_CURSOR] =
3690                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3691                 wm->ddl[pipe].plane[PLANE_SPRITE0] =
3692                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3693                 wm->ddl[pipe].plane[PLANE_SPRITE1] =
3694                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3695         }
3696
3697         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3698         wm->sr.plane = _FW_WM(tmp, SR);
3699         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3700         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
3701         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
3702
3703         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3704         wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
3705         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3706         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
3707
3708         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3709         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3710
3711         if (IS_CHERRYVIEW(dev_priv)) {
3712                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
3713                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3714                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3715
3716                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
3717                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
3718                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
3719
3720                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
3721                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
3722                 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
3723
3724                 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3725                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3726                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3727                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3728                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
3729                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3730                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3731                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3732                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3733                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3734                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3735         } else {
3736                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
3737                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3738                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3739
3740                 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3741                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3742                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3743                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3744                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3745                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3746                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3747                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3748         }
3749 }
3750
3751 #undef _FW_WM
3752 #undef _FW_WM_VLV
3753
3754 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
3755 {
3756         struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;
3757         struct intel_crtc *crtc;
3758
3759         g4x_read_wm_values(dev_priv, wm);
3760
3761         wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
3762
3763         for_each_intel_crtc(&dev_priv->drm, crtc) {
3764                 struct intel_crtc_state *crtc_state =
3765                         to_intel_crtc_state(crtc->base.state);
3766                 struct g4x_wm_state *active = &crtc->wm.active.g4x;
3767                 struct g4x_pipe_wm *raw;
3768                 enum pipe pipe = crtc->pipe;
3769                 enum plane_id plane_id;
3770                 int level, max_level;
3771
3772                 active->cxsr = wm->cxsr;
3773                 active->hpll_en = wm->hpll_en;
3774                 active->fbc_en = wm->fbc_en;
3775
3776                 active->sr = wm->sr;
3777                 active->hpll = wm->hpll;
3778
3779                 for_each_plane_id_on_crtc(crtc, plane_id) {
3780                         active->wm.plane[plane_id] =
3781                                 wm->pipe[pipe].plane[plane_id];
3782                 }
3783
3784                 if (wm->cxsr && wm->hpll_en)
3785                         max_level = G4X_WM_LEVEL_HPLL;
3786                 else if (wm->cxsr)
3787                         max_level = G4X_WM_LEVEL_SR;
3788                 else
3789                         max_level = G4X_WM_LEVEL_NORMAL;
3790
3791                 level = G4X_WM_LEVEL_NORMAL;
3792                 raw = &crtc_state->wm.g4x.raw[level];
3793                 for_each_plane_id_on_crtc(crtc, plane_id)
3794                         raw->plane[plane_id] = active->wm.plane[plane_id];
3795
3796                 level = G4X_WM_LEVEL_SR;
3797                 if (level > max_level)
3798                         goto out;
3799
3800                 raw = &crtc_state->wm.g4x.raw[level];
3801                 raw->plane[PLANE_PRIMARY] = active->sr.plane;
3802                 raw->plane[PLANE_CURSOR] = active->sr.cursor;
3803                 raw->plane[PLANE_SPRITE0] = 0;
3804                 raw->fbc = active->sr.fbc;
3805
3806                 level = G4X_WM_LEVEL_HPLL;
3807                 if (level > max_level)
3808                         goto out;
3809
3810                 raw = &crtc_state->wm.g4x.raw[level];
3811                 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
3812                 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
3813                 raw->plane[PLANE_SPRITE0] = 0;
3814                 raw->fbc = active->hpll.fbc;
3815
3816                 level++;
3817         out:
3818                 for_each_plane_id_on_crtc(crtc, plane_id)
3819                         g4x_raw_plane_wm_set(crtc_state, level,
3820                                              plane_id, USHRT_MAX);
3821                 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
3822
3823                 g4x_invalidate_wms(crtc, active, level);
3824
3825                 crtc_state->wm.g4x.optimal = *active;
3826                 crtc_state->wm.g4x.intermediate = *active;
3827
3828                 drm_dbg_kms(&dev_priv->drm,
3829                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
3830                             pipe_name(pipe),
3831                             wm->pipe[pipe].plane[PLANE_PRIMARY],
3832                             wm->pipe[pipe].plane[PLANE_CURSOR],
3833                             wm->pipe[pipe].plane[PLANE_SPRITE0]);
3834         }
3835
3836         drm_dbg_kms(&dev_priv->drm,
3837                     "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
3838                     wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
3839         drm_dbg_kms(&dev_priv->drm,
3840                     "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
3841                     wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
3842         drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
3843                     str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
3844                     str_yes_no(wm->fbc_en));
3845 }
3846
3847 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
3848 {
3849         struct intel_plane *plane;
3850         struct intel_crtc *crtc;
3851
3852         mutex_lock(&dev_priv->display.wm.wm_mutex);
3853
3854         for_each_intel_plane(&dev_priv->drm, plane) {
3855                 struct intel_crtc *crtc =
3856                         intel_crtc_for_pipe(dev_priv, plane->pipe);
3857                 struct intel_crtc_state *crtc_state =
3858                         to_intel_crtc_state(crtc->base.state);
3859                 struct intel_plane_state *plane_state =
3860                         to_intel_plane_state(plane->base.state);
3861                 enum plane_id plane_id = plane->id;
3862                 int level, num_levels = intel_wm_num_levels(dev_priv);
3863
3864                 if (plane_state->uapi.visible)
3865                         continue;
3866
3867                 for (level = 0; level < num_levels; level++) {
3868                         struct g4x_pipe_wm *raw =
3869                                 &crtc_state->wm.g4x.raw[level];
3870
3871                         raw->plane[plane_id] = 0;
3872
3873                         if (plane_id == PLANE_PRIMARY)
3874                                 raw->fbc = 0;
3875                 }
3876         }
3877
3878         for_each_intel_crtc(&dev_priv->drm, crtc) {
3879                 struct intel_crtc_state *crtc_state =
3880                         to_intel_crtc_state(crtc->base.state);
3881                 int ret;
3882
3883                 ret = _g4x_compute_pipe_wm(crtc_state);
3884                 drm_WARN_ON(&dev_priv->drm, ret);
3885
3886                 crtc_state->wm.g4x.intermediate =
3887                         crtc_state->wm.g4x.optimal;
3888                 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
3889         }
3890
3891         g4x_program_watermarks(dev_priv);
3892
3893         mutex_unlock(&dev_priv->display.wm.wm_mutex);
3894 }
3895
3896 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
3897 {
3898         struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;
3899         struct intel_crtc *crtc;
3900         u32 val;
3901
3902         vlv_read_wm_values(dev_priv, wm);
3903
3904         wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3905         wm->level = VLV_WM_LEVEL_PM2;
3906
3907         if (IS_CHERRYVIEW(dev_priv)) {
3908                 vlv_punit_get(dev_priv);
3909
3910                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
3911                 if (val & DSP_MAXFIFO_PM5_ENABLE)
3912                         wm->level = VLV_WM_LEVEL_PM5;
3913
3914                 /*
3915                  * If DDR DVFS is disabled in the BIOS, Punit
3916                  * will never ack the request. So if that happens
3917                  * assume we don't have to enable/disable DDR DVFS
3918                  * dynamically. To test that just set the REQ_ACK
3919                  * bit to poke the Punit, but don't change the
3920                  * HIGH/LOW bits so that we don't actually change
3921                  * the current state.
3922                  */
3923                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3924                 val |= FORCE_DDR_FREQ_REQ_ACK;
3925                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
3926
3927                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
3928                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
3929                         drm_dbg_kms(&dev_priv->drm,
3930                                     "Punit not acking DDR DVFS request, "
3931                                     "assuming DDR DVFS is disabled\n");
3932                         dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM5;
3933                 } else {
3934                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3935                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
3936                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
3937                 }
3938
3939                 vlv_punit_put(dev_priv);
3940         }
3941
3942         for_each_intel_crtc(&dev_priv->drm, crtc) {
3943                 struct intel_crtc_state *crtc_state =
3944                         to_intel_crtc_state(crtc->base.state);
3945                 struct vlv_wm_state *active = &crtc->wm.active.vlv;
3946                 const struct vlv_fifo_state *fifo_state =
3947                         &crtc_state->wm.vlv.fifo_state;
3948                 enum pipe pipe = crtc->pipe;
3949                 enum plane_id plane_id;
3950                 int level;
3951
3952                 vlv_get_fifo_size(crtc_state);
3953
3954                 active->num_levels = wm->level + 1;
3955                 active->cxsr = wm->cxsr;
3956
3957                 for (level = 0; level < active->num_levels; level++) {
3958                         struct g4x_pipe_wm *raw =
3959                                 &crtc_state->wm.vlv.raw[level];
3960
3961                         active->sr[level].plane = wm->sr.plane;
3962                         active->sr[level].cursor = wm->sr.cursor;
3963
3964                         for_each_plane_id_on_crtc(crtc, plane_id) {
3965                                 active->wm[level].plane[plane_id] =
3966                                         wm->pipe[pipe].plane[plane_id];
3967
3968                                 raw->plane[plane_id] =
3969                                         vlv_invert_wm_value(active->wm[level].plane[plane_id],
3970                                                             fifo_state->plane[plane_id]);
3971                         }
3972                 }
3973
3974                 for_each_plane_id_on_crtc(crtc, plane_id)
3975                         vlv_raw_plane_wm_set(crtc_state, level,
3976                                              plane_id, USHRT_MAX);
3977                 vlv_invalidate_wms(crtc, active, level);
3978
3979                 crtc_state->wm.vlv.optimal = *active;
3980                 crtc_state->wm.vlv.intermediate = *active;
3981
3982                 drm_dbg_kms(&dev_priv->drm,
3983                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
3984                             pipe_name(pipe),
3985                             wm->pipe[pipe].plane[PLANE_PRIMARY],
3986                             wm->pipe[pipe].plane[PLANE_CURSOR],
3987                             wm->pipe[pipe].plane[PLANE_SPRITE0],
3988                             wm->pipe[pipe].plane[PLANE_SPRITE1]);
3989         }
3990
3991         drm_dbg_kms(&dev_priv->drm,
3992                     "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
3993                     wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
3994 }
3995
3996 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
3997 {
3998         struct intel_plane *plane;
3999         struct intel_crtc *crtc;
4000
4001         mutex_lock(&dev_priv->display.wm.wm_mutex);
4002
4003         for_each_intel_plane(&dev_priv->drm, plane) {
4004                 struct intel_crtc *crtc =
4005                         intel_crtc_for_pipe(dev_priv, plane->pipe);
4006                 struct intel_crtc_state *crtc_state =
4007                         to_intel_crtc_state(crtc->base.state);
4008                 struct intel_plane_state *plane_state =
4009                         to_intel_plane_state(plane->base.state);
4010                 enum plane_id plane_id = plane->id;
4011                 int level, num_levels = intel_wm_num_levels(dev_priv);
4012
4013                 if (plane_state->uapi.visible)
4014                         continue;
4015
4016                 for (level = 0; level < num_levels; level++) {
4017                         struct g4x_pipe_wm *raw =
4018                                 &crtc_state->wm.vlv.raw[level];
4019
4020                         raw->plane[plane_id] = 0;
4021                 }
4022         }
4023
4024         for_each_intel_crtc(&dev_priv->drm, crtc) {
4025                 struct intel_crtc_state *crtc_state =
4026                         to_intel_crtc_state(crtc->base.state);
4027                 int ret;
4028
4029                 ret = _vlv_compute_pipe_wm(crtc_state);
4030                 drm_WARN_ON(&dev_priv->drm, ret);
4031
4032                 crtc_state->wm.vlv.intermediate =
4033                         crtc_state->wm.vlv.optimal;
4034                 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
4035         }
4036
4037         vlv_program_watermarks(dev_priv);
4038
4039         mutex_unlock(&dev_priv->display.wm.wm_mutex);
4040 }
4041
4042 /*
4043  * FIXME should probably kill this and improve
4044  * the real watermark readout/sanitation instead
4045  */
4046 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
4047 {
4048         intel_uncore_rmw(&dev_priv->uncore, WM3_LP_ILK, WM_LP_ENABLE, 0);
4049         intel_uncore_rmw(&dev_priv->uncore, WM2_LP_ILK, WM_LP_ENABLE, 0);
4050         intel_uncore_rmw(&dev_priv->uncore, WM1_LP_ILK, WM_LP_ENABLE, 0);
4051
4052         /*
4053          * Don't touch WM_LP_SPRITE_ENABLE here.
4054          * Doing so could cause underruns.
4055          */
4056 }
4057
4058 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
4059 {
4060         struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
4061         struct intel_crtc *crtc;
4062
4063         ilk_init_lp_watermarks(dev_priv);
4064
4065         for_each_intel_crtc(&dev_priv->drm, crtc)
4066                 ilk_pipe_wm_get_hw_state(crtc);
4067
4068         hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
4069         hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
4070         hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
4071
4072         hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
4073         if (DISPLAY_VER(dev_priv) >= 7) {
4074                 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
4075                 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
4076         }
4077
4078         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
4079                 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
4080                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4081         else if (IS_IVYBRIDGE(dev_priv))
4082                 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
4083                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4084
4085         hw->enable_fbc_wm =
4086                 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
4087 }
4088
4089 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
4090 {
4091         /*
4092          * On Ibex Peak and Cougar Point, we need to disable clock
4093          * gating for the panel power sequencer or it will fail to
4094          * start up when no ports are active.
4095          */
4096         intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
4097 }
4098
4099 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
4100 {
4101         enum pipe pipe;
4102
4103         for_each_pipe(dev_priv, pipe) {
4104                 intel_uncore_rmw(&dev_priv->uncore, DSPCNTR(pipe), 0, DISP_TRICKLE_FEED_DISABLE);
4105
4106                 intel_uncore_rmw(&dev_priv->uncore, DSPSURF(pipe), 0, 0);
4107                 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
4108         }
4109 }
4110
4111 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
4112 {
4113         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4114
4115         /*
4116          * Required for FBC
4117          * WaFbcDisableDpfcClockGating:ilk
4118          */
4119         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
4120                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
4121                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
4122
4123         intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
4124                    MARIUNIT_CLOCK_GATE_DISABLE |
4125                    SVSMUNIT_CLOCK_GATE_DISABLE);
4126         intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
4127                    VFMUNIT_CLOCK_GATE_DISABLE);
4128
4129         /*
4130          * According to the spec the following bits should be set in
4131          * order to enable memory self-refresh
4132          * The bit 22/21 of 0x42004
4133          * The bit 5 of 0x42020
4134          * The bit 15 of 0x45000
4135          */
4136         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4137                    (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4138                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
4139         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
4140         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
4141                    (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
4142                     DISP_FBC_WM_DIS));
4143
4144         /*
4145          * Based on the document from hardware guys the following bits
4146          * should be set unconditionally in order to enable FBC.
4147          * The bit 22 of 0x42000
4148          * The bit 22 of 0x42004
4149          * The bit 7,8,9 of 0x42020.
4150          */
4151         if (IS_IRONLAKE_M(dev_priv)) {
4152                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
4153                 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
4154                 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
4155         }
4156
4157         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
4158
4159         intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
4160
4161         g4x_disable_trickle_feed(dev_priv);
4162
4163         ibx_init_clock_gating(dev_priv);
4164 }
4165
4166 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
4167 {
4168         enum pipe pipe;
4169         u32 val;
4170
4171         /*
4172          * On Ibex Peak and Cougar Point, we need to disable clock
4173          * gating for the panel power sequencer or it will fail to
4174          * start up when no ports are active.
4175          */
4176         intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
4177                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
4178                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
4179         intel_uncore_rmw(&dev_priv->uncore, SOUTH_CHICKEN2, 0, DPLS_EDP_PPS_FIX_DIS);
4180         /* The below fixes the weird display corruption, a few pixels shifted
4181          * downward, on (only) LVDS of some HP laptops with IVY.
4182          */
4183         for_each_pipe(dev_priv, pipe) {
4184                 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
4185                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
4186                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4187                 if (dev_priv->display.vbt.fdi_rx_polarity_inverted)
4188                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4189                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
4190                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
4191                 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
4192         }
4193         /* WADP0ClockGatingDisable */
4194         for_each_pipe(dev_priv, pipe) {
4195                 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
4196                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
4197         }
4198 }
4199
4200 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
4201 {
4202         u32 tmp;
4203
4204         tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
4205         if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
4206                 drm_dbg_kms(&dev_priv->drm,
4207                             "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
4208                             tmp);
4209 }
4210
4211 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
4212 {
4213         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4214
4215         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
4216
4217         intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
4218
4219         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
4220                    intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
4221                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
4222                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
4223
4224         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
4225          * gating disable must be set.  Failure to set it results in
4226          * flickering pixels due to Z write ordering failures after
4227          * some amount of runtime in the Mesa "fire" demo, and Unigine
4228          * Sanctuary and Tropics, and apparently anything else with
4229          * alpha test or pixel discard.
4230          *
4231          * According to the spec, bit 11 (RCCUNIT) must also be set,
4232          * but we didn't debug actual testcases to find it out.
4233          *
4234          * WaDisableRCCUnitClockGating:snb
4235          * WaDisableRCPBUnitClockGating:snb
4236          */
4237         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4238                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
4239                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
4240
4241         /*
4242          * According to the spec the following bits should be
4243          * set in order to enable memory self-refresh and fbc:
4244          * The bit21 and bit22 of 0x42000
4245          * The bit21 and bit22 of 0x42004
4246          * The bit5 and bit7 of 0x42020
4247          * The bit14 of 0x70180
4248          * The bit14 of 0x71180
4249          *
4250          * WaFbcAsynchFlipDisableFbcQueue:snb
4251          */
4252         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
4253                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
4254                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
4255         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4256                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4257                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
4258         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
4259                    intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
4260                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
4261                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
4262
4263         g4x_disable_trickle_feed(dev_priv);
4264
4265         cpt_init_clock_gating(dev_priv);
4266
4267         gen6_check_mch_setup(dev_priv);
4268 }
4269
4270 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
4271 {
4272         /*
4273          * TODO: this bit should only be enabled when really needed, then
4274          * disabled when not needed anymore in order to save power.
4275          */
4276         if (HAS_PCH_LPT_LP(dev_priv))
4277                 intel_uncore_rmw(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
4278                                  0, PCH_LP_PARTITION_LEVEL_DISABLE);
4279
4280         /* WADPOClockGatingDisable:hsw */
4281         intel_uncore_rmw(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
4282                          0, TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
4283 }
4284
4285 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
4286 {
4287         if (HAS_PCH_LPT_LP(dev_priv)) {
4288                 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
4289
4290                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
4291                 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
4292         }
4293 }
4294
4295 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
4296                                    int general_prio_credits,
4297                                    int high_prio_credits)
4298 {
4299         u32 misccpctl;
4300         u32 val;
4301
4302         /* WaTempDisableDOPClkGating:bdw */
4303         misccpctl = intel_uncore_rmw(&dev_priv->uncore, GEN7_MISCCPCTL,
4304                                      GEN7_DOP_CLOCK_GATE_ENABLE, 0);
4305
4306         val = intel_gt_mcr_read_any(to_gt(dev_priv), GEN8_L3SQCREG1);
4307         val &= ~L3_PRIO_CREDITS_MASK;
4308         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
4309         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
4310         intel_gt_mcr_multicast_write(to_gt(dev_priv), GEN8_L3SQCREG1, val);
4311
4312         /*
4313          * Wait at least 100 clocks before re-enabling clock gating.
4314          * See the definition of L3SQCREG1 in BSpec.
4315          */
4316         intel_gt_mcr_read_any(to_gt(dev_priv), GEN8_L3SQCREG1);
4317         udelay(1);
4318         intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
4319 }
4320
4321 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
4322 {
4323         /* Wa_1409120013:icl,ehl */
4324         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4325                            DPFC_CHICKEN_COMP_DUMMY_PIXEL);
4326
4327         /*Wa_14010594013:icl, ehl */
4328         intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
4329                          0, ICL_DELAY_PMRSP);
4330 }
4331
4332 static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
4333 {
4334         /* Wa_1409120013 */
4335         if (DISPLAY_VER(dev_priv) == 12)
4336                 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4337                                    DPFC_CHICKEN_COMP_DUMMY_PIXEL);
4338
4339         /* Wa_1409825376:tgl (pre-prod)*/
4340         if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
4341                 intel_uncore_rmw(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, 0, TGL_VRH_GATING_DIS);
4342
4343         /* Wa_14013723622:tgl,rkl,dg1,adl-s */
4344         if (DISPLAY_VER(dev_priv) == 12)
4345                 intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
4346                                  CLKREQ_POLICY_MEM_UP_OVRD, 0);
4347 }
4348
4349 static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
4350 {
4351         gen12lp_init_clock_gating(dev_priv);
4352
4353         /* Wa_22011091694:adlp */
4354         intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
4355
4356         /* Bspec/49189 Initialize Sequence */
4357         intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1, DDI_CLOCK_REG_ACCESS, 0);
4358 }
4359
4360 static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
4361 {
4362         gen12lp_init_clock_gating(dev_priv);
4363
4364         /* Wa_1409836686:dg1[a0] */
4365         if (IS_DG1_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
4366                 intel_uncore_rmw(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, 0, DPT_GATING_DIS);
4367 }
4368
4369 static void xehpsdv_init_clock_gating(struct drm_i915_private *dev_priv)
4370 {
4371         /* Wa_22010146351:xehpsdv */
4372         if (IS_XEHPSDV_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
4373                 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
4374 }
4375
4376 static void dg2_init_clock_gating(struct drm_i915_private *i915)
4377 {
4378         /* Wa_22010954014:dg2 */
4379         intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
4380                          SGSI_SIDECLK_DIS);
4381
4382         /*
4383          * Wa_14010733611:dg2_g10
4384          * Wa_22010146351:dg2_g10
4385          */
4386         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
4387                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
4388                                  SGR_DIS | SGGI_DIS);
4389 }
4390
4391 static void pvc_init_clock_gating(struct drm_i915_private *dev_priv)
4392 {
4393         /* Wa_14012385139:pvc */
4394         if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
4395                 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
4396
4397         /* Wa_22010954014:pvc */
4398         if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
4399                 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);
4400 }
4401
4402 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
4403 {
4404         if (!HAS_PCH_CNP(dev_priv))
4405                 return;
4406
4407         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
4408         intel_uncore_rmw(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, 0, CNP_PWM_CGE_GATING_DISABLE);
4409 }
4410
4411 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
4412 {
4413         cnp_init_clock_gating(dev_priv);
4414         gen9_init_clock_gating(dev_priv);
4415
4416         /* WAC6entrylatency:cfl */
4417         intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4418
4419         /*
4420          * WaFbcTurnOffFbcWatermark:cfl
4421          * Display WA #0562: cfl
4422          */
4423         intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4424
4425         /*
4426          * WaFbcNukeOnHostModify:cfl
4427          * Display WA #0873: cfl
4428          */
4429         intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4430                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
4431 }
4432
4433 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
4434 {
4435         gen9_init_clock_gating(dev_priv);
4436
4437         /* WAC6entrylatency:kbl */
4438         intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4439
4440         /* WaDisableSDEUnitClockGating:kbl */
4441         if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
4442                 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6,
4443                                  0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4444
4445         /* WaDisableGamClockGating:kbl */
4446         if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
4447                 intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1,
4448                                  0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
4449
4450         /*
4451          * WaFbcTurnOffFbcWatermark:kbl
4452          * Display WA #0562: kbl
4453          */
4454         intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4455
4456         /*
4457          * WaFbcNukeOnHostModify:kbl
4458          * Display WA #0873: kbl
4459          */
4460         intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4461                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
4462 }
4463
4464 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
4465 {
4466         gen9_init_clock_gating(dev_priv);
4467
4468         /* WaDisableDopClockGating:skl */
4469         intel_uncore_rmw(&dev_priv->uncore, GEN7_MISCCPCTL,
4470                          GEN7_DOP_CLOCK_GATE_ENABLE, 0);
4471
4472         /* WAC6entrylatency:skl */
4473         intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4474
4475         /*
4476          * WaFbcTurnOffFbcWatermark:skl
4477          * Display WA #0562: skl
4478          */
4479         intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4480
4481         /*
4482          * WaFbcNukeOnHostModify:skl
4483          * Display WA #0873: skl
4484          */
4485         intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4486                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
4487
4488         /*
4489          * WaFbcHighMemBwCorruptionAvoidance:skl
4490          * Display WA #0883: skl
4491          */
4492         intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
4493 }
4494
4495 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
4496 {
4497         enum pipe pipe;
4498
4499         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
4500         intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
4501
4502         /* WaSwitchSolVfFArbitrationPriority:bdw */
4503         intel_uncore_rmw(&dev_priv->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
4504
4505         /* WaPsrDPAMaskVBlankInSRD:bdw */
4506         intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, DPA_MASK_VBLANK_SRD);
4507
4508         for_each_pipe(dev_priv, pipe) {
4509                 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
4510                 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
4511                                  0, BDW_DPRS_MASK_VBLANK_SRD);
4512         }
4513
4514         /* WaVSRefCountFullforceMissDisable:bdw */
4515         /* WaDSRefCountFullforceMissDisable:bdw */
4516         intel_uncore_rmw(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
4517                          GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
4518
4519         intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
4520                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4521
4522         /* WaDisableSDEUnitClockGating:bdw */
4523         intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4524
4525         /* WaProgramL3SqcReg1Default:bdw */
4526         gen8_set_l3sqc_credits(dev_priv, 30, 2);
4527
4528         /* WaKVMNotificationOnConfigChange:bdw */
4529         intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR2_1,
4530                          0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
4531
4532         lpt_init_clock_gating(dev_priv);
4533
4534         /* WaDisableDopClockGating:bdw
4535          *
4536          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
4537          * clock gating.
4538          */
4539         intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1, 0, GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
4540 }
4541
4542 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
4543 {
4544         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
4545         intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
4546
4547         /* This is required by WaCatErrorRejectionIssue:hsw */
4548         intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4549                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4550
4551         /* WaSwitchSolVfFArbitrationPriority:hsw */
4552         intel_uncore_rmw(&dev_priv->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
4553
4554         lpt_init_clock_gating(dev_priv);
4555 }
4556
4557 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
4558 {
4559         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
4560
4561         /* WaFbcAsynchFlipDisableFbcQueue:ivb */
4562         intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
4563
4564         /* WaDisableBackToBackFlipFix:ivb */
4565         intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
4566                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
4567                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
4568
4569         if (IS_IVB_GT1(dev_priv))
4570                 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4571                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4572         else {
4573                 /* must write both registers */
4574                 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4575                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4576                 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
4577                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4578         }
4579
4580         /*
4581          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4582          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
4583          */
4584         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4585                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
4586
4587         /* This is required by WaCatErrorRejectionIssue:ivb */
4588         intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4589                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4590
4591         g4x_disable_trickle_feed(dev_priv);
4592
4593         intel_uncore_rmw(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
4594                          GEN6_MBC_SNPCR_MED);
4595
4596         if (!HAS_PCH_NOP(dev_priv))
4597                 cpt_init_clock_gating(dev_priv);
4598
4599         gen6_check_mch_setup(dev_priv);
4600 }
4601
4602 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
4603 {
4604         /* WaDisableBackToBackFlipFix:vlv */
4605         intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
4606                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
4607                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
4608
4609         /* WaDisableDopClockGating:vlv */
4610         intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4611                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4612
4613         /* This is required by WaCatErrorRejectionIssue:vlv */
4614         intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4615                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4616
4617         /*
4618          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4619          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
4620          */
4621         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4622                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
4623
4624         /* WaDisableL3Bank2xClockGate:vlv
4625          * Disabling L3 clock gating- MMIO 940c[25] = 1
4626          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
4627         intel_uncore_rmw(&dev_priv->uncore, GEN7_UCGCTL4, 0, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
4628
4629         /*
4630          * WaDisableVLVClockGating_VBIIssue:vlv
4631          * Disable clock gating on th GCFG unit to prevent a delay
4632          * in the reporting of vblank events.
4633          */
4634         intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
4635 }
4636
4637 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
4638 {
4639         /* WaVSRefCountFullforceMissDisable:chv */
4640         /* WaDSRefCountFullforceMissDisable:chv */
4641         intel_uncore_rmw(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
4642                          GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
4643
4644         /* WaDisableSemaphoreAndSyncFlipWait:chv */
4645         intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
4646                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4647
4648         /* WaDisableCSUnitClockGating:chv */
4649         intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
4650
4651         /* WaDisableSDEUnitClockGating:chv */
4652         intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4653
4654         /*
4655          * WaProgramL3SqcReg1Default:chv
4656          * See gfxspecs/Related Documents/Performance Guide/
4657          * LSQC Setting Recommendations.
4658          */
4659         gen8_set_l3sqc_credits(dev_priv, 38, 2);
4660 }
4661
4662 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
4663 {
4664         u32 dspclk_gate;
4665
4666         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
4667         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
4668                    GS_UNIT_CLOCK_GATE_DISABLE |
4669                    CL_UNIT_CLOCK_GATE_DISABLE);
4670         intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
4671         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
4672                 OVRUNIT_CLOCK_GATE_DISABLE |
4673                 OVCUNIT_CLOCK_GATE_DISABLE;
4674         if (IS_GM45(dev_priv))
4675                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
4676         intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D(dev_priv), dspclk_gate);
4677
4678         g4x_disable_trickle_feed(dev_priv);
4679 }
4680
4681 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
4682 {
4683         struct intel_uncore *uncore = &dev_priv->uncore;
4684
4685         intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
4686         intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
4687         intel_uncore_write(uncore, DSPCLK_GATE_D(dev_priv), 0);
4688         intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
4689         intel_uncore_write16(uncore, DEUC, 0);
4690         intel_uncore_write(uncore,
4691                            MI_ARB_STATE,
4692                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4693 }
4694
4695 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
4696 {
4697         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
4698                    I965_RCC_CLOCK_GATE_DISABLE |
4699                    I965_RCPB_CLOCK_GATE_DISABLE |
4700                    I965_ISC_CLOCK_GATE_DISABLE |
4701                    I965_FBC_CLOCK_GATE_DISABLE);
4702         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
4703         intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
4704                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4705 }
4706
4707 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
4708 {
4709         u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
4710
4711         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
4712                 DSTATE_DOT_CLOCK_GATING;
4713         intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
4714
4715         if (IS_PINEVIEW(dev_priv))
4716                 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
4717                                    _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
4718
4719         /* IIR "flip pending" means done if this bit is set */
4720         intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
4721                            _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
4722
4723         /* interrupts should cause a wake up from C3 */
4724         intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
4725
4726         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4727         intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
4728
4729         intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
4730                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4731 }
4732
4733 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
4734 {
4735         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
4736
4737         /* interrupts should cause a wake up from C3 */
4738         intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
4739                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
4740
4741         intel_uncore_write(&dev_priv->uncore, MEM_MODE,
4742                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
4743
4744         /*
4745          * Have FBC ignore 3D activity since we use software
4746          * render tracking, and otherwise a pure 3D workload
4747          * (even if it just renders a single frame and then does
4748          * abosultely nothing) would not allow FBC to recompress
4749          * until a 2D blit occurs.
4750          */
4751         intel_uncore_write(&dev_priv->uncore, SCPD0,
4752                    _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
4753 }
4754
4755 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
4756 {
4757         intel_uncore_write(&dev_priv->uncore, MEM_MODE,
4758                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
4759                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
4760 }
4761
4762 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
4763 {
4764         dev_priv->clock_gating_funcs->init_clock_gating(dev_priv);
4765 }
4766
4767 void intel_suspend_hw(struct drm_i915_private *dev_priv)
4768 {
4769         if (HAS_PCH_LPT(dev_priv))
4770                 lpt_suspend_hw(dev_priv);
4771 }
4772
4773 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
4774 {
4775         drm_dbg_kms(&dev_priv->drm,
4776                     "No clock gating settings or workarounds applied.\n");
4777 }
4778
4779 #define CG_FUNCS(platform)                                              \
4780 static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
4781         .init_clock_gating = platform##_init_clock_gating,              \
4782 }
4783
4784 CG_FUNCS(pvc);
4785 CG_FUNCS(dg2);
4786 CG_FUNCS(xehpsdv);
4787 CG_FUNCS(adlp);
4788 CG_FUNCS(dg1);
4789 CG_FUNCS(gen12lp);
4790 CG_FUNCS(icl);
4791 CG_FUNCS(cfl);
4792 CG_FUNCS(skl);
4793 CG_FUNCS(kbl);
4794 CG_FUNCS(bxt);
4795 CG_FUNCS(glk);
4796 CG_FUNCS(bdw);
4797 CG_FUNCS(chv);
4798 CG_FUNCS(hsw);
4799 CG_FUNCS(ivb);
4800 CG_FUNCS(vlv);
4801 CG_FUNCS(gen6);
4802 CG_FUNCS(ilk);
4803 CG_FUNCS(g4x);
4804 CG_FUNCS(i965gm);
4805 CG_FUNCS(i965g);
4806 CG_FUNCS(gen3);
4807 CG_FUNCS(i85x);
4808 CG_FUNCS(i830);
4809 CG_FUNCS(nop);
4810 #undef CG_FUNCS
4811
4812 /**
4813  * intel_init_clock_gating_hooks - setup the clock gating hooks
4814  * @dev_priv: device private
4815  *
4816  * Setup the hooks that configure which clocks of a given platform can be
4817  * gated and also apply various GT and display specific workarounds for these
4818  * platforms. Note that some GT specific workarounds are applied separately
4819  * when GPU contexts or batchbuffers start their execution.
4820  */
4821 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
4822 {
4823         if (IS_PONTEVECCHIO(dev_priv))
4824                 dev_priv->clock_gating_funcs = &pvc_clock_gating_funcs;
4825         else if (IS_DG2(dev_priv))
4826                 dev_priv->clock_gating_funcs = &dg2_clock_gating_funcs;
4827         else if (IS_XEHPSDV(dev_priv))
4828                 dev_priv->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
4829         else if (IS_ALDERLAKE_P(dev_priv))
4830                 dev_priv->clock_gating_funcs = &adlp_clock_gating_funcs;
4831         else if (IS_DG1(dev_priv))
4832                 dev_priv->clock_gating_funcs = &dg1_clock_gating_funcs;
4833         else if (GRAPHICS_VER(dev_priv) == 12)
4834                 dev_priv->clock_gating_funcs = &gen12lp_clock_gating_funcs;
4835         else if (GRAPHICS_VER(dev_priv) == 11)
4836                 dev_priv->clock_gating_funcs = &icl_clock_gating_funcs;
4837         else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
4838                 dev_priv->clock_gating_funcs = &cfl_clock_gating_funcs;
4839         else if (IS_SKYLAKE(dev_priv))
4840                 dev_priv->clock_gating_funcs = &skl_clock_gating_funcs;
4841         else if (IS_KABYLAKE(dev_priv))
4842                 dev_priv->clock_gating_funcs = &kbl_clock_gating_funcs;
4843         else if (IS_BROXTON(dev_priv))
4844                 dev_priv->clock_gating_funcs = &bxt_clock_gating_funcs;
4845         else if (IS_GEMINILAKE(dev_priv))
4846                 dev_priv->clock_gating_funcs = &glk_clock_gating_funcs;
4847         else if (IS_BROADWELL(dev_priv))
4848                 dev_priv->clock_gating_funcs = &bdw_clock_gating_funcs;
4849         else if (IS_CHERRYVIEW(dev_priv))
4850                 dev_priv->clock_gating_funcs = &chv_clock_gating_funcs;
4851         else if (IS_HASWELL(dev_priv))
4852                 dev_priv->clock_gating_funcs = &hsw_clock_gating_funcs;
4853         else if (IS_IVYBRIDGE(dev_priv))
4854                 dev_priv->clock_gating_funcs = &ivb_clock_gating_funcs;
4855         else if (IS_VALLEYVIEW(dev_priv))
4856                 dev_priv->clock_gating_funcs = &vlv_clock_gating_funcs;
4857         else if (GRAPHICS_VER(dev_priv) == 6)
4858                 dev_priv->clock_gating_funcs = &gen6_clock_gating_funcs;
4859         else if (GRAPHICS_VER(dev_priv) == 5)
4860                 dev_priv->clock_gating_funcs = &ilk_clock_gating_funcs;
4861         else if (IS_G4X(dev_priv))
4862                 dev_priv->clock_gating_funcs = &g4x_clock_gating_funcs;
4863         else if (IS_I965GM(dev_priv))
4864                 dev_priv->clock_gating_funcs = &i965gm_clock_gating_funcs;
4865         else if (IS_I965G(dev_priv))
4866                 dev_priv->clock_gating_funcs = &i965g_clock_gating_funcs;
4867         else if (GRAPHICS_VER(dev_priv) == 3)
4868                 dev_priv->clock_gating_funcs = &gen3_clock_gating_funcs;
4869         else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
4870                 dev_priv->clock_gating_funcs = &i85x_clock_gating_funcs;
4871         else if (GRAPHICS_VER(dev_priv) == 2)
4872                 dev_priv->clock_gating_funcs = &i830_clock_gating_funcs;
4873         else {
4874                 MISSING_CASE(INTEL_DEVID(dev_priv));
4875                 dev_priv->clock_gating_funcs = &nop_clock_gating_funcs;
4876         }
4877 }
4878
4879 static const struct intel_wm_funcs ilk_wm_funcs = {
4880         .compute_pipe_wm = ilk_compute_pipe_wm,
4881         .compute_intermediate_wm = ilk_compute_intermediate_wm,
4882         .initial_watermarks = ilk_initial_watermarks,
4883         .optimize_watermarks = ilk_optimize_watermarks,
4884 };
4885
4886 static const struct intel_wm_funcs vlv_wm_funcs = {
4887         .compute_pipe_wm = vlv_compute_pipe_wm,
4888         .compute_intermediate_wm = vlv_compute_intermediate_wm,
4889         .initial_watermarks = vlv_initial_watermarks,
4890         .optimize_watermarks = vlv_optimize_watermarks,
4891         .atomic_update_watermarks = vlv_atomic_update_fifo,
4892 };
4893
4894 static const struct intel_wm_funcs g4x_wm_funcs = {
4895         .compute_pipe_wm = g4x_compute_pipe_wm,
4896         .compute_intermediate_wm = g4x_compute_intermediate_wm,
4897         .initial_watermarks = g4x_initial_watermarks,
4898         .optimize_watermarks = g4x_optimize_watermarks,
4899 };
4900
4901 static const struct intel_wm_funcs pnv_wm_funcs = {
4902         .update_wm = pnv_update_wm,
4903 };
4904
4905 static const struct intel_wm_funcs i965_wm_funcs = {
4906         .update_wm = i965_update_wm,
4907 };
4908
4909 static const struct intel_wm_funcs i9xx_wm_funcs = {
4910         .update_wm = i9xx_update_wm,
4911 };
4912
4913 static const struct intel_wm_funcs i845_wm_funcs = {
4914         .update_wm = i845_update_wm,
4915 };
4916
4917 static const struct intel_wm_funcs nop_funcs = {
4918 };
4919
4920 /* Set up chip specific power management-related functions */
4921 void intel_init_pm(struct drm_i915_private *dev_priv)
4922 {
4923         if (DISPLAY_VER(dev_priv) >= 9) {
4924                 skl_wm_init(dev_priv);
4925                 return;
4926         }
4927
4928         /* For cxsr */
4929         if (IS_PINEVIEW(dev_priv))
4930                 pnv_get_mem_freq(dev_priv);
4931         else if (GRAPHICS_VER(dev_priv) == 5)
4932                 ilk_get_mem_freq(dev_priv);
4933
4934         /* For FIFO watermark updates */
4935         if (HAS_PCH_SPLIT(dev_priv)) {
4936                 ilk_setup_wm_latency(dev_priv);
4937
4938                 if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->display.wm.pri_latency[1] &&
4939                      dev_priv->display.wm.spr_latency[1] && dev_priv->display.wm.cur_latency[1]) ||
4940                     (DISPLAY_VER(dev_priv) != 5 && dev_priv->display.wm.pri_latency[0] &&
4941                      dev_priv->display.wm.spr_latency[0] && dev_priv->display.wm.cur_latency[0])) {
4942                         dev_priv->display.funcs.wm = &ilk_wm_funcs;
4943                 } else {
4944                         drm_dbg_kms(&dev_priv->drm,
4945                                     "Failed to read display plane latency. "
4946                                     "Disable CxSR\n");
4947                         dev_priv->display.funcs.wm = &nop_funcs;
4948                 }
4949         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
4950                 vlv_setup_wm_latency(dev_priv);
4951                 dev_priv->display.funcs.wm = &vlv_wm_funcs;
4952         } else if (IS_G4X(dev_priv)) {
4953                 g4x_setup_wm_latency(dev_priv);
4954                 dev_priv->display.funcs.wm = &g4x_wm_funcs;
4955         } else if (IS_PINEVIEW(dev_priv)) {
4956                 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
4957                                             dev_priv->is_ddr3,
4958                                             dev_priv->fsb_freq,
4959                                             dev_priv->mem_freq)) {
4960                         drm_info(&dev_priv->drm,
4961                                  "failed to find known CxSR latency "
4962                                  "(found ddr%s fsb freq %d, mem freq %d), "
4963                                  "disabling CxSR\n",
4964                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
4965                                  dev_priv->fsb_freq, dev_priv->mem_freq);
4966                         /* Disable CxSR and never update its watermark again */
4967                         intel_set_memory_cxsr(dev_priv, false);
4968                         dev_priv->display.funcs.wm = &nop_funcs;
4969                 } else
4970                         dev_priv->display.funcs.wm = &pnv_wm_funcs;
4971         } else if (DISPLAY_VER(dev_priv) == 4) {
4972                 dev_priv->display.funcs.wm = &i965_wm_funcs;
4973         } else if (DISPLAY_VER(dev_priv) == 3) {
4974                 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4975         } else if (DISPLAY_VER(dev_priv) == 2) {
4976                 if (INTEL_NUM_PIPES(dev_priv) == 1)
4977                         dev_priv->display.funcs.wm = &i845_wm_funcs;
4978                 else
4979                         dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4980         } else {
4981                 drm_err(&dev_priv->drm,
4982                         "unexpected fall-through in %s\n", __func__);
4983                 dev_priv->display.funcs.wm = &nop_funcs;
4984         }
4985 }
4986
4987 void intel_pm_setup(struct drm_i915_private *dev_priv)
4988 {
4989         dev_priv->runtime_pm.suspended = false;
4990         atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
4991 }