drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <[email protected]>
  25  *
  26  */
  27
  28 #include "display/intel_de.h"
  29 #include "display/intel_display_trace.h"
  30 #include "display/skl_watermark.h"
  31
  32 #include "gt/intel_engine_regs.h"
  33 #include "gt/intel_gt.h"
  34 #include "gt/intel_gt_mcr.h"
  35 #include "gt/intel_gt_regs.h"
  36
  37 #include "i915_drv.h"
  38 #include "intel_mchbar_regs.h"
  39 #include "intel_pm.h"
  40 #include "vlv_sideband.h"
  41
  42 struct drm_i915_clock_gating_funcs {
  43         void (*init_clock_gating)(struct drm_i915_private *i915);
  44 };
  45
  46 /* used in computing the new watermarks state */
  47 struct intel_wm_config {
  48         unsigned int num_pipes_active;
  49         bool sprites_enabled;
  50         bool sprites_scaled;
  51 };
  52
  53 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
  54 {
  55         if (HAS_LLC(dev_priv)) {
  56                 /*
  57                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  58                  * Display WA #0390: skl,kbl
  59                  *
  60                  * Must match Sampler, Pixel Back End, and Media. See
  61                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  62                  */
  63                 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
  64         }
  65
  66         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  67         intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
  68
  69         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  70         intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
  71
  72         /*
  73          * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
  74          * Display WA #0859: skl,bxt,kbl,glk,cfl
  75          */
  76         intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
  77 }
  78
  79 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
  80 {
  81         gen9_init_clock_gating(dev_priv);
  82
  83         /* WaDisableSDEUnitClockGating:bxt */
  84         intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
  85
  86         /*
  87          * FIXME:
  88          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
  89          */
  90         intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
  91
  92         /*
  93          * Wa: Backlight PWM may stop in the asserted state, causing backlight
  94          * to stay fully on.
  95          */
  96         intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
  97                    PWM1_GATING_DIS | PWM2_GATING_DIS);
  98
  99         /*
 100          * Lower the display internal timeout.
 101          * This is needed to avoid any hard hangs when DSI port PLL
 102          * is off and a MMIO access is attempted by any privilege
 103          * application, using batch buffers or any other means.
 104          */
 105         intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
 106
 107         /*
 108          * WaFbcTurnOffFbcWatermark:bxt
 109          * Display WA #0562: bxt
 110          */
 111         intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 112
 113         /*
 114          * WaFbcHighMemBwCorruptionAvoidance:bxt
 115          * Display WA #0883: bxt
 116          */
 117         intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
 118 }
 119
 120 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
 121 {
 122         gen9_init_clock_gating(dev_priv);
 123
 124         /*
 125          * WaDisablePWMClockGating:glk
 126          * Backlight PWM may stop in the asserted state, causing backlight
 127          * to stay fully on.
 128          */
 129         intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
 130                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 131 }
 132
 133 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
 134 {
 135         u32 tmp;
 136
 137         tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
 138
 139         switch (tmp & CLKCFG_FSB_MASK) {
 140         case CLKCFG_FSB_533:
 141                 dev_priv->fsb_freq = 533; /* 133*4 */
 142                 break;
 143         case CLKCFG_FSB_800:
 144                 dev_priv->fsb_freq = 800; /* 200*4 */
 145                 break;
 146         case CLKCFG_FSB_667:
 147                 dev_priv->fsb_freq =  667; /* 167*4 */
 148                 break;
 149         case CLKCFG_FSB_400:
 150                 dev_priv->fsb_freq = 400; /* 100*4 */
 151                 break;
 152         }
 153
 154         switch (tmp & CLKCFG_MEM_MASK) {
 155         case CLKCFG_MEM_533:
 156                 dev_priv->mem_freq = 533;
 157                 break;
 158         case CLKCFG_MEM_667:
 159                 dev_priv->mem_freq = 667;
 160                 break;
 161         case CLKCFG_MEM_800:
 162                 dev_priv->mem_freq = 800;
 163                 break;
 164         }
 165
 166         /* detect pineview DDR3 setting */
 167         tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
 168         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 169 }
 170
 171 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
 172 {
 173         u16 ddrpll, csipll;
 174
 175         ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
 176         csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
 177
 178         switch (ddrpll & 0xff) {
 179         case 0xc:
 180                 dev_priv->mem_freq = 800;
 181                 break;
 182         case 0x10:
 183                 dev_priv->mem_freq = 1066;
 184                 break;
 185         case 0x14:
 186                 dev_priv->mem_freq = 1333;
 187                 break;
 188         case 0x18:
 189                 dev_priv->mem_freq = 1600;
 190                 break;
 191         default:
 192                 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
 193                         ddrpll & 0xff);
 194                 dev_priv->mem_freq = 0;
 195                 break;
 196         }
 197
 198         switch (csipll & 0x3ff) {
 199         case 0x00c:
 200                 dev_priv->fsb_freq = 3200;
 201                 break;
 202         case 0x00e:
 203                 dev_priv->fsb_freq = 3733;
 204                 break;
 205         case 0x010:
 206                 dev_priv->fsb_freq = 4266;
 207                 break;
 208         case 0x012:
 209                 dev_priv->fsb_freq = 4800;
 210                 break;
 211         case 0x014:
 212                 dev_priv->fsb_freq = 5333;
 213                 break;
 214         case 0x016:
 215                 dev_priv->fsb_freq = 5866;
 216                 break;
 217         case 0x018:
 218                 dev_priv->fsb_freq = 6400;
 219                 break;
 220         default:
 221                 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
 222                         csipll & 0x3ff);
 223                 dev_priv->fsb_freq = 0;
 224                 break;
 225         }
 226 }
 227
 228 static const struct cxsr_latency cxsr_latency_table[] = {
 229         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 230         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 231         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 232         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 233         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 234
 235         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 236         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 237         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 238         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 239         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 240
 241         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 242         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 243         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 244         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 245         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 246
 247         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 248         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 249         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 250         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 251         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 252
 253         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 254         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 255         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 256         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 257         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 258
 259         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 260         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 261         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 262         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 263         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 264 };
 265
 266 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
 267                                                          bool is_ddr3,
 268                                                          int fsb,
 269                                                          int mem)
 270 {
 271         const struct cxsr_latency *latency;
 272         int i;
 273
 274         if (fsb == 0 || mem == 0)
 275                 return NULL;
 276
 277         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 278                 latency = &cxsr_latency_table[i];
 279                 if (is_desktop == latency->is_desktop &&
 280                     is_ddr3 == latency->is_ddr3 &&
 281                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 282                         return latency;
 283         }
 284
 285         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 286
 287         return NULL;
 288 }
 289
 290 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 291 {
 292         u32 val;
 293
 294         vlv_punit_get(dev_priv);
 295
 296         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 297         if (enable)
 298                 val &= ~FORCE_DDR_HIGH_FREQ;
 299         else
 300                 val |= FORCE_DDR_HIGH_FREQ;
 301         val &= ~FORCE_DDR_LOW_FREQ;
 302         val |= FORCE_DDR_FREQ_REQ_ACK;
 303         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 304
 305         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 306                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 307                 drm_err(&dev_priv->drm,
 308                         "timed out waiting for Punit DDR DVFS request\n");
 309
 310         vlv_punit_put(dev_priv);
 311 }
 312
 313 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 314 {
 315         u32 val;
 316
 317         vlv_punit_get(dev_priv);
 318
 319         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 320         if (enable)
 321                 val |= DSP_MAXFIFO_PM5_ENABLE;
 322         else
 323                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 324         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 325
 326         vlv_punit_put(dev_priv);
 327 }
 328
 329 #define FW_WM(value, plane) \
 330         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 331
 332 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 333 {
 334         bool was_enabled;
 335         u32 val;
 336
 337         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 338                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 339                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 340                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
 341         } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
 342                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
 343                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 344                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
 345         } else if (IS_PINEVIEW(dev_priv)) {
 346                 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
 347                 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 348                 if (enable)
 349                         val |= PINEVIEW_SELF_REFRESH_EN;
 350                 else
 351                         val &= ~PINEVIEW_SELF_REFRESH_EN;
 352                 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
 353                 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
 354         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 355                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
 356                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 357                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 358                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
 359                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
 360         } else if (IS_I915GM(dev_priv)) {
 361                 /*
 362                  * FIXME can't find a bit like this for 915G, and
 363                  * and yet it does have the related watermark in
 364                  * FW_BLC_SELF. What's going on?
 365                  */
 366                 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
 367                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 368                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 369                 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
 370                 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
 371         } else {
 372                 return false;
 373         }
 374
 375         trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
 376
 377         drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
 378                     str_enabled_disabled(enable),
 379                     str_enabled_disabled(was_enabled));
 380
 381         return was_enabled;
 382 }
 383
 384 /**
 385  * intel_set_memory_cxsr - Configure CxSR state
 386  * @dev_priv: i915 device
 387  * @enable: Allow vs. disallow CxSR
 388  *
 389  * Allow or disallow the system to enter a special CxSR
 390  * (C-state self refresh) state. What typically happens in CxSR mode
 391  * is that several display FIFOs may get combined into a single larger
 392  * FIFO for a particular plane (so called max FIFO mode) to allow the
 393  * system to defer memory fetches longer, and the memory will enter
 394  * self refresh.
 395  *
 396  * Note that enabling CxSR does not guarantee that the system enter
 397  * this special mode, nor does it guarantee that the system stays
 398  * in that mode once entered. So this just allows/disallows the system
 399  * to autonomously utilize the CxSR mode. Other factors such as core
 400  * C-states will affect when/if the system actually enters/exits the
 401  * CxSR mode.
 402  *
 403  * Note that on VLV/CHV this actually only controls the max FIFO mode,
 404  * and the system is free to enter/exit memory self refresh at any time
 405  * even when the use of CxSR has been disallowed.
 406  *
 407  * While the system is actually in the CxSR/max FIFO mode, some plane
 408  * control registers will not get latched on vblank. Thus in order to
 409  * guarantee the system will respond to changes in the plane registers
 410  * we must always disallow CxSR prior to making changes to those registers.
 411  * Unfortunately the system will re-evaluate the CxSR conditions at
 412  * frame start which happens after vblank start (which is when the plane
 413  * registers would get latched), so we can't proceed with the plane update
 414  * during the same frame where we disallowed CxSR.
 415  *
 416  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 417  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 418  * the hardware w.r.t. HPLL SR when writing to plane registers.
 419  * Disallowing just CxSR is sufficient.
 420  */
 421 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 422 {
 423         bool ret;
 424
 425         mutex_lock(&dev_priv->display.wm.wm_mutex);
 426         ret = _intel_set_memory_cxsr(dev_priv, enable);
 427         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 428                 dev_priv->display.wm.vlv.cxsr = enable;
 429         else if (IS_G4X(dev_priv))
 430                 dev_priv->display.wm.g4x.cxsr = enable;
 431         mutex_unlock(&dev_priv->display.wm.wm_mutex);
 432
 433         return ret;
 434 }
 435
 436 /*
 437  * Latency for FIFO fetches is dependent on several factors:
 438  *   - memory configuration (speed, channels)
 439  *   - chipset
 440  *   - current MCH state
 441  * It can be fairly high in some situations, so here we assume a fairly
 442  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 443  * set this value too high, the FIFO will fetch frequently to stay full)
 444  * and power consumption (set it too low to save power and we might see
 445  * FIFO underruns and display "flicker").
 446  *
 447  * A value of 5us seems to be a good balance; safe for very low end
 448  * platforms but not overly aggressive on lower latency configs.
 449  */
 450 static const int pessimal_latency_ns = 5000;
 451
 452 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 453         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 454
 455 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 456 {
 457         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 458         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 459         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 460         enum pipe pipe = crtc->pipe;
 461         int sprite0_start, sprite1_start;
 462         u32 dsparb, dsparb2, dsparb3;
 463
 464         switch (pipe) {
 465         case PIPE_A:
 466                 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 467                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 468                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 469                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 470                 break;
 471         case PIPE_B:
 472                 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 473                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 474                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 475                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 476                 break;
 477         case PIPE_C:
 478                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 479                 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
 480                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 481                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 482                 break;
 483         default:
 484                 MISSING_CASE(pipe);
 485                 return;
 486         }
 487
 488         fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 489         fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 490         fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 491         fifo_state->plane[PLANE_CURSOR] = 63;
 492 }
 493
 494 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
 495                               enum i9xx_plane_id i9xx_plane)
 496 {
 497         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 498         int size;
 499
 500         size = dsparb & 0x7f;
 501         if (i9xx_plane == PLANE_B)
 502                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 503
 504         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 505                     dsparb, plane_name(i9xx_plane), size);
 506
 507         return size;
 508 }
 509
 510 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
 511                               enum i9xx_plane_id i9xx_plane)
 512 {
 513         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 514         int size;
 515
 516         size = dsparb & 0x1ff;
 517         if (i9xx_plane == PLANE_B)
 518                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 519         size >>= 1; /* Convert to cachelines */
 520
 521         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 522                     dsparb, plane_name(i9xx_plane), size);
 523
 524         return size;
 525 }
 526
 527 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
 528                               enum i9xx_plane_id i9xx_plane)
 529 {
 530         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 531         int size;
 532
 533         size = dsparb & 0x7f;
 534         size >>= 2; /* Convert to cachelines */
 535
 536         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 537                     dsparb, plane_name(i9xx_plane), size);
 538
 539         return size;
 540 }
 541
 542 /* Pineview has different values for various configs */
 543 static const struct intel_watermark_params pnv_display_wm = {
 544         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 545         .max_wm = PINEVIEW_MAX_WM,
 546         .default_wm = PINEVIEW_DFT_WM,
 547         .guard_size = PINEVIEW_GUARD_WM,
 548         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 549 };
 550
 551 static const struct intel_watermark_params pnv_display_hplloff_wm = {
 552         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 553         .max_wm = PINEVIEW_MAX_WM,
 554         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 555         .guard_size = PINEVIEW_GUARD_WM,
 556         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 557 };
 558
 559 static const struct intel_watermark_params pnv_cursor_wm = {
 560         .fifo_size = PINEVIEW_CURSOR_FIFO,
 561         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 562         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 563         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 564         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 565 };
 566
 567 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
 568         .fifo_size = PINEVIEW_CURSOR_FIFO,
 569         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 570         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 571         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 572         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 573 };
 574
 575 static const struct intel_watermark_params i965_cursor_wm_info = {
 576         .fifo_size = I965_CURSOR_FIFO,
 577         .max_wm = I965_CURSOR_MAX_WM,
 578         .default_wm = I965_CURSOR_DFT_WM,
 579         .guard_size = 2,
 580         .cacheline_size = I915_FIFO_LINE_SIZE,
 581 };
 582
 583 static const struct intel_watermark_params i945_wm_info = {
 584         .fifo_size = I945_FIFO_SIZE,
 585         .max_wm = I915_MAX_WM,
 586         .default_wm = 1,
 587         .guard_size = 2,
 588         .cacheline_size = I915_FIFO_LINE_SIZE,
 589 };
 590
 591 static const struct intel_watermark_params i915_wm_info = {
 592         .fifo_size = I915_FIFO_SIZE,
 593         .max_wm = I915_MAX_WM,
 594         .default_wm = 1,
 595         .guard_size = 2,
 596         .cacheline_size = I915_FIFO_LINE_SIZE,
 597 };
 598
 599 static const struct intel_watermark_params i830_a_wm_info = {
 600         .fifo_size = I855GM_FIFO_SIZE,
 601         .max_wm = I915_MAX_WM,
 602         .default_wm = 1,
 603         .guard_size = 2,
 604         .cacheline_size = I830_FIFO_LINE_SIZE,
 605 };
 606
 607 static const struct intel_watermark_params i830_bc_wm_info = {
 608         .fifo_size = I855GM_FIFO_SIZE,
 609         .max_wm = I915_MAX_WM/2,
 610         .default_wm = 1,
 611         .guard_size = 2,
 612         .cacheline_size = I830_FIFO_LINE_SIZE,
 613 };
 614
 615 static const struct intel_watermark_params i845_wm_info = {
 616         .fifo_size = I830_FIFO_SIZE,
 617         .max_wm = I915_MAX_WM,
 618         .default_wm = 1,
 619         .guard_size = 2,
 620         .cacheline_size = I830_FIFO_LINE_SIZE,
 621 };
 622
 623 /**
 624  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 625  * @pixel_rate: Pipe pixel rate in kHz
 626  * @cpp: Plane bytes per pixel
 627  * @latency: Memory wakeup latency in 0.1us units
 628  *
 629  * Compute the watermark using the method 1 or "small buffer"
 630  * formula. The caller may additonally add extra cachelines
 631  * to account for TLB misses and clock crossings.
 632  *
 633  * This method is concerned with the short term drain rate
 634  * of the FIFO, ie. it does not account for blanking periods
 635  * which would effectively reduce the average drain rate across
 636  * a longer period. The name "small" refers to the fact the
 637  * FIFO is relatively small compared to the amount of data
 638  * fetched.
 639  *
 640  * The FIFO level vs. time graph might look something like:
 641  *
 642  *   |\   |\
 643  *   | \  | \
 644  * __---__---__ (- plane active, _ blanking)
 645  * -> time
 646  *
 647  * or perhaps like this:
 648  *
 649  *   |\|\  |\|\
 650  * __----__----__ (- plane active, _ blanking)
 651  * -> time
 652  *
 653  * Returns:
 654  * The watermark in bytes
 655  */
 656 static unsigned int intel_wm_method1(unsigned int pixel_rate,
 657                                      unsigned int cpp,
 658                                      unsigned int latency)
 659 {
 660         u64 ret;
 661
 662         ret = mul_u32_u32(pixel_rate, cpp * latency);
 663         ret = DIV_ROUND_UP_ULL(ret, 10000);
 664
 665         return ret;
 666 }
 667
 668 /**
 669  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 670  * @pixel_rate: Pipe pixel rate in kHz
 671  * @htotal: Pipe horizontal total
 672  * @width: Plane width in pixels
 673  * @cpp: Plane bytes per pixel
 674  * @latency: Memory wakeup latency in 0.1us units
 675  *
 676  * Compute the watermark using the method 2 or "large buffer"
 677  * formula. The caller may additonally add extra cachelines
 678  * to account for TLB misses and clock crossings.
 679  *
 680  * This method is concerned with the long term drain rate
 681  * of the FIFO, ie. it does account for blanking periods
 682  * which effectively reduce the average drain rate across
 683  * a longer period. The name "large" refers to the fact the
 684  * FIFO is relatively large compared to the amount of data
 685  * fetched.
 686  *
 687  * The FIFO level vs. time graph might look something like:
 688  *
 689  *    |\___       |\___
 690  *    |    \___   |    \___
 691  *    |        \  |        \
 692  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 693  * -> time
 694  *
 695  * Returns:
 696  * The watermark in bytes
 697  */
 698 static unsigned int intel_wm_method2(unsigned int pixel_rate,
 699                                      unsigned int htotal,
 700                                      unsigned int width,
 701                                      unsigned int cpp,
 702                                      unsigned int latency)
 703 {
 704         unsigned int ret;
 705
 706         /*
 707          * FIXME remove once all users are computing
 708          * watermarks in the correct place.
 709          */
 710         if (WARN_ON_ONCE(htotal == 0))
 711                 htotal = 1;
 712
 713         ret = (latency * pixel_rate) / (htotal * 10000);
 714         ret = (ret + 1) * width * cpp;
 715
 716         return ret;
 717 }
 718
 719 /**
 720  * intel_calculate_wm - calculate watermark level
 721  * @pixel_rate: pixel clock
 722  * @wm: chip FIFO params
 723  * @fifo_size: size of the FIFO buffer
 724  * @cpp: bytes per pixel
 725  * @latency_ns: memory latency for the platform
 726  *
 727  * Calculate the watermark level (the level at which the display plane will
 728  * start fetching from memory again).  Each chip has a different display
 729  * FIFO size and allocation, so the caller needs to figure that out and pass
 730  * in the correct intel_watermark_params structure.
 731  *
 732  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 733  * on the pixel size.  When it reaches the watermark level, it'll start
 734  * fetching FIFO line sized based chunks from memory until the FIFO fills
 735  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 736  * will occur, and a display engine hang could result.
 737  */
 738 static unsigned int intel_calculate_wm(int pixel_rate,
 739                                        const struct intel_watermark_params *wm,
 740                                        int fifo_size, int cpp,
 741                                        unsigned int latency_ns)
 742 {
 743         int entries, wm_size;
 744
 745         /*
 746          * Note: we need to make sure we don't overflow for various clock &
 747          * latency values.
 748          * clocks go from a few thousand to several hundred thousand.
 749          * latency is usually a few thousand
 750          */
 751         entries = intel_wm_method1(pixel_rate, cpp,
 752                                    latency_ns / 100);
 753         entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 754                 wm->guard_size;
 755         DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
 756
 757         wm_size = fifo_size - entries;
 758         DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
 759
 760         /* Don't promote wm_size to unsigned... */
 761         if (wm_size > wm->max_wm)
 762                 wm_size = wm->max_wm;
 763         if (wm_size <= 0)
 764                 wm_size = wm->default_wm;
 765
 766         /*
 767          * Bspec seems to indicate that the value shouldn't be lower than
 768          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 769          * Lets go for 8 which is the burst size since certain platforms
 770          * already use a hardcoded 8 (which is what the spec says should be
 771          * done).
 772          */
 773         if (wm_size <= 8)
 774                 wm_size = 8;
 775
 776         return wm_size;
 777 }
 778
 779 static bool is_disabling(int old, int new, int threshold)
 780 {
 781         return old >= threshold && new < threshold;
 782 }
 783
 784 static bool is_enabling(int old, int new, int threshold)
 785 {
 786         return old < threshold && new >= threshold;
 787 }
 788
 789 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
 790 {
 791         return dev_priv->display.wm.max_level + 1;
 792 }
 793
 794 bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
 795                             const struct intel_plane_state *plane_state)
 796 {
 797         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 798
 799         /* FIXME check the 'enable' instead */
 800         if (!crtc_state->hw.active)
 801                 return false;
 802
 803         /*
 804          * Treat cursor with fb as always visible since cursor updates
 805          * can happen faster than the vrefresh rate, and the current
 806          * watermark code doesn't handle that correctly. Cursor updates
 807          * which set/clear the fb or change the cursor size are going
 808          * to get throttled by intel_legacy_cursor_update() to work
 809          * around this problem with the watermark code.
 810          */
 811         if (plane->id == PLANE_CURSOR)
 812                 return plane_state->hw.fb != NULL;
 813         else
 814                 return plane_state->uapi.visible;
 815 }
 816
 817 static bool intel_crtc_active(struct intel_crtc *crtc)
 818 {
 819         /* Be paranoid as we can arrive here with only partial
 820          * state retrieved from the hardware during setup.
 821          *
 822          * We can ditch the adjusted_mode.crtc_clock check as soon
 823          * as Haswell has gained clock readout/fastboot support.
 824          *
 825          * We can ditch the crtc->primary->state->fb check as soon as we can
 826          * properly reconstruct framebuffers.
 827          *
 828          * FIXME: The intel_crtc->active here should be switched to
 829          * crtc->state->active once we have proper CRTC states wired up
 830          * for atomic.
 831          */
 832         return crtc && crtc->active && crtc->base.primary->state->fb &&
 833                 crtc->config->hw.adjusted_mode.crtc_clock;
 834 }
 835
 836 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 837 {
 838         struct intel_crtc *crtc, *enabled = NULL;
 839
 840         for_each_intel_crtc(&dev_priv->drm, crtc) {
 841                 if (intel_crtc_active(crtc)) {
 842                         if (enabled)
 843                                 return NULL;
 844                         enabled = crtc;
 845                 }
 846         }
 847
 848         return enabled;
 849 }
 850
 851 static void pnv_update_wm(struct drm_i915_private *dev_priv)
 852 {
 853         struct intel_crtc *crtc;
 854         const struct cxsr_latency *latency;
 855         u32 reg;
 856         unsigned int wm;
 857
 858         latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
 859                                          dev_priv->is_ddr3,
 860                                          dev_priv->fsb_freq,
 861                                          dev_priv->mem_freq);
 862         if (!latency) {
 863                 drm_dbg_kms(&dev_priv->drm,
 864                             "Unknown FSB/MEM found, disable CxSR\n");
 865                 intel_set_memory_cxsr(dev_priv, false);
 866                 return;
 867         }
 868
 869         crtc = single_enabled_crtc(dev_priv);
 870         if (crtc) {
 871                 const struct drm_framebuffer *fb =
 872                         crtc->base.primary->state->fb;
 873                 int pixel_rate = crtc->config->pixel_rate;
 874                 int cpp = fb->format->cpp[0];
 875
 876                 /* Display SR */
 877                 wm = intel_calculate_wm(pixel_rate, &pnv_display_wm,
 878                                         pnv_display_wm.fifo_size,
 879                                         cpp, latency->display_sr);
 880                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
 881                 reg &= ~DSPFW_SR_MASK;
 882                 reg |= FW_WM(wm, SR);
 883                 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
 884                 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
 885
 886                 /* cursor SR */
 887                 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_wm,
 888                                         pnv_display_wm.fifo_size,
 889                                         4, latency->cursor_sr);
 890                 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_CURSOR_SR_MASK,
 891                                  FW_WM(wm, CURSOR_SR));
 892
 893                 /* Display HPLL off SR */
 894                 wm = intel_calculate_wm(pixel_rate, &pnv_display_hplloff_wm,
 895                                         pnv_display_hplloff_wm.fifo_size,
 896                                         cpp, latency->display_hpll_disable);
 897                 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR));
 898
 899                 /* cursor HPLL off SR */
 900                 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_hplloff_wm,
 901                                         pnv_display_hplloff_wm.fifo_size,
 902                                         4, latency->cursor_hpll_disable);
 903                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
 904                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 905                 reg |= FW_WM(wm, HPLL_CURSOR);
 906                 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
 907                 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
 908
 909                 intel_set_memory_cxsr(dev_priv, true);
 910         } else {
 911                 intel_set_memory_cxsr(dev_priv, false);
 912         }
 913 }
 914
 915 /*
 916  * Documentation says:
 917  * "If the line size is small, the TLB fetches can get in the way of the
 918  *  data fetches, causing some lag in the pixel data return which is not
 919  *  accounted for in the above formulas. The following adjustment only
 920  *  needs to be applied if eight whole lines fit in the buffer at once.
 921  *  The WM is adjusted upwards by the difference between the FIFO size
 922  *  and the size of 8 whole lines. This adjustment is always performed
 923  *  in the actual pixel depth regardless of whether FBC is enabled or not."
 924  */
 925 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 926 {
 927         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 928
 929         return max(0, tlb_miss);
 930 }
 931
 932 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 933                                 const struct g4x_wm_values *wm)
 934 {
 935         enum pipe pipe;
 936
 937         for_each_pipe(dev_priv, pipe)
 938                 trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
 939
 940         intel_uncore_write(&dev_priv->uncore, DSPFW1,
 941                    FW_WM(wm->sr.plane, SR) |
 942                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 943                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 944                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 945         intel_uncore_write(&dev_priv->uncore, DSPFW2,
 946                    (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 947                    FW_WM(wm->sr.fbc, FBC_SR) |
 948                    FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
 949                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
 950                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 951                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 952         intel_uncore_write(&dev_priv->uncore, DSPFW3,
 953                    (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
 954                    FW_WM(wm->sr.cursor, CURSOR_SR) |
 955                    FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
 956                    FW_WM(wm->hpll.plane, HPLL_SR));
 957
 958         intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
 959 }
 960
 961 #define FW_WM_VLV(value, plane) \
 962         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
 963
 964 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
 965                                 const struct vlv_wm_values *wm)
 966 {
 967         enum pipe pipe;
 968
 969         for_each_pipe(dev_priv, pipe) {
 970                 trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
 971
 972                 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
 973                            (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
 974                            (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
 975                            (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
 976                            (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
 977         }
 978
 979         /*
 980          * Zero the (unused) WM1 watermarks, and also clear all the
 981          * high order bits so that there are no out of bounds values
 982          * present in the registers during the reprogramming.
 983          */
 984         intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
 985         intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
 986         intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
 987         intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
 988         intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
 989
 990         intel_uncore_write(&dev_priv->uncore, DSPFW1,
 991                    FW_WM(wm->sr.plane, SR) |
 992                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 993                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 994                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 995         intel_uncore_write(&dev_priv->uncore, DSPFW2,
 996                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
 997                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 998                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 999         intel_uncore_write(&dev_priv->uncore, DSPFW3,
1000                    FW_WM(wm->sr.cursor, CURSOR_SR));
1001
1002         if (IS_CHERRYVIEW(dev_priv)) {
1003                 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1004                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1005                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1006                 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1007                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1008                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1009                 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1010                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1011                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1012                 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1013                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1014                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1015                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1016                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1017                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1018                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1019                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1020                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1021                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1022                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1023         } else {
1024                 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1025                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1026                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1027                 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1028                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1029                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1030                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1031                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1032                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1033                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1034                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1035         }
1036
1037         intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1038 }
1039
1040 #undef FW_WM_VLV
1041
1042 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1043 {
1044         /* all latencies in usec */
1045         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1046         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1047         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1048
1049         dev_priv->display.wm.max_level = G4X_WM_LEVEL_HPLL;
1050 }
1051
1052 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1053 {
1054         /*
1055          * DSPCNTR[13] supposedly controls whether the
1056          * primary plane can use the FIFO space otherwise
1057          * reserved for the sprite plane. It's not 100% clear
1058          * what the actual FIFO size is, but it looks like we
1059          * can happily set both primary and sprite watermarks
1060          * up to 127 cachelines. So that would seem to mean
1061          * that either DSPCNTR[13] doesn't do anything, or that
1062          * the total FIFO is >= 256 cachelines in size. Either
1063          * way, we don't seem to have to worry about this
1064          * repartitioning as the maximum watermark value the
1065          * register can hold for each plane is lower than the
1066          * minimum FIFO size.
1067          */
1068         switch (plane_id) {
1069         case PLANE_CURSOR:
1070                 return 63;
1071         case PLANE_PRIMARY:
1072                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1073         case PLANE_SPRITE0:
1074                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1075         default:
1076                 MISSING_CASE(plane_id);
1077                 return 0;
1078         }
1079 }
1080
1081 static int g4x_fbc_fifo_size(int level)
1082 {
1083         switch (level) {
1084         case G4X_WM_LEVEL_SR:
1085                 return 7;
1086         case G4X_WM_LEVEL_HPLL:
1087                 return 15;
1088         default:
1089                 MISSING_CASE(level);
1090                 return 0;
1091         }
1092 }
1093
1094 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1095                           const struct intel_plane_state *plane_state,
1096                           int level)
1097 {
1098         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1099         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1100         const struct drm_display_mode *pipe_mode =
1101                 &crtc_state->hw.pipe_mode;
1102         unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;
1103         unsigned int pixel_rate, htotal, cpp, width, wm;
1104
1105         if (latency == 0)
1106                 return USHRT_MAX;
1107
1108         if (!intel_wm_plane_visible(crtc_state, plane_state))
1109                 return 0;
1110
1111         cpp = plane_state->hw.fb->format->cpp[0];
1112
1113         /*
1114          * WaUse32BppForSRWM:ctg,elk
1115          *
1116          * The spec fails to list this restriction for the
1117          * HPLL watermark, which seems a little strange.
1118          * Let's use 32bpp for the HPLL watermark as well.
1119          */
1120         if (plane->id == PLANE_PRIMARY &&
1121             level != G4X_WM_LEVEL_NORMAL)
1122                 cpp = max(cpp, 4u);
1123
1124         pixel_rate = crtc_state->pixel_rate;
1125         htotal = pipe_mode->crtc_htotal;
1126         width = drm_rect_width(&plane_state->uapi.src) >> 16;
1127
1128         if (plane->id == PLANE_CURSOR) {
1129                 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1130         } else if (plane->id == PLANE_PRIMARY &&
1131                    level == G4X_WM_LEVEL_NORMAL) {
1132                 wm = intel_wm_method1(pixel_rate, cpp, latency);
1133         } else {
1134                 unsigned int small, large;
1135
1136                 small = intel_wm_method1(pixel_rate, cpp, latency);
1137                 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1138
1139                 wm = min(small, large);
1140         }
1141
1142         wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1143                               width, cpp);
1144
1145         wm = DIV_ROUND_UP(wm, 64) + 2;
1146
1147         return min_t(unsigned int, wm, USHRT_MAX);
1148 }
1149
1150 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1151                                  int level, enum plane_id plane_id, u16 value)
1152 {
1153         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1154         bool dirty = false;
1155
1156         for (; level < intel_wm_num_levels(dev_priv); level++) {
1157                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1158
1159                 dirty |= raw->plane[plane_id] != value;
1160                 raw->plane[plane_id] = value;
1161         }
1162
1163         return dirty;
1164 }
1165
1166 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1167                                int level, u16 value)
1168 {
1169         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1170         bool dirty = false;
1171
1172         /* NORMAL level doesn't have an FBC watermark */
1173         level = max(level, G4X_WM_LEVEL_SR);
1174
1175         for (; level < intel_wm_num_levels(dev_priv); level++) {
1176                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1177
1178                 dirty |= raw->fbc != value;
1179                 raw->fbc = value;
1180         }
1181
1182         return dirty;
1183 }
1184
1185 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1186                               const struct intel_plane_state *plane_state,
1187                               u32 pri_val);
1188
1189 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1190                                      const struct intel_plane_state *plane_state)
1191 {
1192         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1193         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1194         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1195         enum plane_id plane_id = plane->id;
1196         bool dirty = false;
1197         int level;
1198
1199         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1200                 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1201                 if (plane_id == PLANE_PRIMARY)
1202                         dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1203                 goto out;
1204         }
1205
1206         for (level = 0; level < num_levels; level++) {
1207                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1208                 int wm, max_wm;
1209
1210                 wm = g4x_compute_wm(crtc_state, plane_state, level);
1211                 max_wm = g4x_plane_fifo_size(plane_id, level);
1212
1213                 if (wm > max_wm)
1214                         break;
1215
1216                 dirty |= raw->plane[plane_id] != wm;
1217                 raw->plane[plane_id] = wm;
1218
1219                 if (plane_id != PLANE_PRIMARY ||
1220                     level == G4X_WM_LEVEL_NORMAL)
1221                         continue;
1222
1223                 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1224                                         raw->plane[plane_id]);
1225                 max_wm = g4x_fbc_fifo_size(level);
1226
1227                 /*
1228                  * FBC wm is not mandatory as we
1229                  * can always just disable its use.
1230                  */
1231                 if (wm > max_wm)
1232                         wm = USHRT_MAX;
1233
1234                 dirty |= raw->fbc != wm;
1235                 raw->fbc = wm;
1236         }
1237
1238         /* mark watermarks as invalid */
1239         dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1240
1241         if (plane_id == PLANE_PRIMARY)
1242                 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1243
1244  out:
1245         if (dirty) {
1246                 drm_dbg_kms(&dev_priv->drm,
1247                             "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1248                             plane->base.name,
1249                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1250                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1251                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1252
1253                 if (plane_id == PLANE_PRIMARY)
1254                         drm_dbg_kms(&dev_priv->drm,
1255                                     "FBC watermarks: SR=%d, HPLL=%d\n",
1256                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1257                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1258         }
1259
1260         return dirty;
1261 }
1262
1263 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1264                                       enum plane_id plane_id, int level)
1265 {
1266         const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1267
1268         return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1269 }
1270
1271 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1272                                      int level)
1273 {
1274         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1275
1276         if (level > dev_priv->display.wm.max_level)
1277                 return false;
1278
1279         return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1280                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1281                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1282 }
1283
1284 /* mark all levels starting from 'level' as invalid */
1285 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1286                                struct g4x_wm_state *wm_state, int level)
1287 {
1288         if (level <= G4X_WM_LEVEL_NORMAL) {
1289                 enum plane_id plane_id;
1290
1291                 for_each_plane_id_on_crtc(crtc, plane_id)
1292                         wm_state->wm.plane[plane_id] = USHRT_MAX;
1293         }
1294
1295         if (level <= G4X_WM_LEVEL_SR) {
1296                 wm_state->cxsr = false;
1297                 wm_state->sr.cursor = USHRT_MAX;
1298                 wm_state->sr.plane = USHRT_MAX;
1299                 wm_state->sr.fbc = USHRT_MAX;
1300         }
1301
1302         if (level <= G4X_WM_LEVEL_HPLL) {
1303                 wm_state->hpll_en = false;
1304                 wm_state->hpll.cursor = USHRT_MAX;
1305                 wm_state->hpll.plane = USHRT_MAX;
1306                 wm_state->hpll.fbc = USHRT_MAX;
1307         }
1308 }
1309
1310 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1311                                int level)
1312 {
1313         if (level < G4X_WM_LEVEL_SR)
1314                 return false;
1315
1316         if (level >= G4X_WM_LEVEL_SR &&
1317             wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1318                 return false;
1319
1320         if (level >= G4X_WM_LEVEL_HPLL &&
1321             wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1322                 return false;
1323
1324         return true;
1325 }
1326
1327 static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1328 {
1329         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1330         struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1331         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1332         const struct g4x_pipe_wm *raw;
1333         enum plane_id plane_id;
1334         int level;
1335
1336         level = G4X_WM_LEVEL_NORMAL;
1337         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1338                 goto out;
1339
1340         raw = &crtc_state->wm.g4x.raw[level];
1341         for_each_plane_id_on_crtc(crtc, plane_id)
1342                 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1343
1344         level = G4X_WM_LEVEL_SR;
1345         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1346                 goto out;
1347
1348         raw = &crtc_state->wm.g4x.raw[level];
1349         wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1350         wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1351         wm_state->sr.fbc = raw->fbc;
1352
1353         wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1354
1355         level = G4X_WM_LEVEL_HPLL;
1356         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1357                 goto out;
1358
1359         raw = &crtc_state->wm.g4x.raw[level];
1360         wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1361         wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1362         wm_state->hpll.fbc = raw->fbc;
1363
1364         wm_state->hpll_en = wm_state->cxsr;
1365
1366         level++;
1367
1368  out:
1369         if (level == G4X_WM_LEVEL_NORMAL)
1370                 return -EINVAL;
1371
1372         /* invalidate the higher levels */
1373         g4x_invalidate_wms(crtc, wm_state, level);
1374
1375         /*
1376          * Determine if the FBC watermark(s) can be used. IF
1377          * this isn't the case we prefer to disable the FBC
1378          * watermark(s) rather than disable the SR/HPLL
1379          * level(s) entirely. 'level-1' is the highest valid
1380          * level here.
1381          */
1382         wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1383
1384         return 0;
1385 }
1386
1387 static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1388                                struct intel_crtc *crtc)
1389 {
1390         struct intel_crtc_state *crtc_state =
1391                 intel_atomic_get_new_crtc_state(state, crtc);
1392         const struct intel_plane_state *old_plane_state;
1393         const struct intel_plane_state *new_plane_state;
1394         struct intel_plane *plane;
1395         unsigned int dirty = 0;
1396         int i;
1397
1398         for_each_oldnew_intel_plane_in_state(state, plane,
1399                                              old_plane_state,
1400                                              new_plane_state, i) {
1401                 if (new_plane_state->hw.crtc != &crtc->base &&
1402                     old_plane_state->hw.crtc != &crtc->base)
1403                         continue;
1404
1405                 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1406                         dirty |= BIT(plane->id);
1407         }
1408
1409         if (!dirty)
1410                 return 0;
1411
1412         return _g4x_compute_pipe_wm(crtc_state);
1413 }
1414
1415 static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1416                                        struct intel_crtc *crtc)
1417 {
1418         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1419         struct intel_crtc_state *new_crtc_state =
1420                 intel_atomic_get_new_crtc_state(state, crtc);
1421         const struct intel_crtc_state *old_crtc_state =
1422                 intel_atomic_get_old_crtc_state(state, crtc);
1423         struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1424         const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1425         const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1426         enum plane_id plane_id;
1427
1428         if (!new_crtc_state->hw.active ||
1429             intel_crtc_needs_modeset(new_crtc_state)) {
1430                 *intermediate = *optimal;
1431
1432                 intermediate->cxsr = false;
1433                 intermediate->hpll_en = false;
1434                 goto out;
1435         }
1436
1437         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1438                 !new_crtc_state->disable_cxsr;
1439         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1440                 !new_crtc_state->disable_cxsr;
1441         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1442
1443         for_each_plane_id_on_crtc(crtc, plane_id) {
1444                 intermediate->wm.plane[plane_id] =
1445                         max(optimal->wm.plane[plane_id],
1446                             active->wm.plane[plane_id]);
1447
1448                 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1449                             g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1450         }
1451
1452         intermediate->sr.plane = max(optimal->sr.plane,
1453                                      active->sr.plane);
1454         intermediate->sr.cursor = max(optimal->sr.cursor,
1455                                       active->sr.cursor);
1456         intermediate->sr.fbc = max(optimal->sr.fbc,
1457                                    active->sr.fbc);
1458
1459         intermediate->hpll.plane = max(optimal->hpll.plane,
1460                                        active->hpll.plane);
1461         intermediate->hpll.cursor = max(optimal->hpll.cursor,
1462                                         active->hpll.cursor);
1463         intermediate->hpll.fbc = max(optimal->hpll.fbc,
1464                                      active->hpll.fbc);
1465
1466         drm_WARN_ON(&dev_priv->drm,
1467                     (intermediate->sr.plane >
1468                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1469                      intermediate->sr.cursor >
1470                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1471                     intermediate->cxsr);
1472         drm_WARN_ON(&dev_priv->drm,
1473                     (intermediate->sr.plane >
1474                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1475                      intermediate->sr.cursor >
1476                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1477                     intermediate->hpll_en);
1478
1479         drm_WARN_ON(&dev_priv->drm,
1480                     intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1481                     intermediate->fbc_en && intermediate->cxsr);
1482         drm_WARN_ON(&dev_priv->drm,
1483                     intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1484                     intermediate->fbc_en && intermediate->hpll_en);
1485
1486 out:
1487         /*
1488          * If our intermediate WM are identical to the final WM, then we can
1489          * omit the post-vblank programming; only update if it's different.
1490          */
1491         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1492                 new_crtc_state->wm.need_postvbl_update = true;
1493
1494         return 0;
1495 }
1496
1497 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1498                          struct g4x_wm_values *wm)
1499 {
1500         struct intel_crtc *crtc;
1501         int num_active_pipes = 0;
1502
1503         wm->cxsr = true;
1504         wm->hpll_en = true;
1505         wm->fbc_en = true;
1506
1507         for_each_intel_crtc(&dev_priv->drm, crtc) {
1508                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1509
1510                 if (!crtc->active)
1511                         continue;
1512
1513                 if (!wm_state->cxsr)
1514                         wm->cxsr = false;
1515                 if (!wm_state->hpll_en)
1516                         wm->hpll_en = false;
1517                 if (!wm_state->fbc_en)
1518                         wm->fbc_en = false;
1519
1520                 num_active_pipes++;
1521         }
1522
1523         if (num_active_pipes != 1) {
1524                 wm->cxsr = false;
1525                 wm->hpll_en = false;
1526                 wm->fbc_en = false;
1527         }
1528
1529         for_each_intel_crtc(&dev_priv->drm, crtc) {
1530                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1531                 enum pipe pipe = crtc->pipe;
1532
1533                 wm->pipe[pipe] = wm_state->wm;
1534                 if (crtc->active && wm->cxsr)
1535                         wm->sr = wm_state->sr;
1536                 if (crtc->active && wm->hpll_en)
1537                         wm->hpll = wm_state->hpll;
1538         }
1539 }
1540
1541 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1542 {
1543         struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;
1544         struct g4x_wm_values new_wm = {};
1545
1546         g4x_merge_wm(dev_priv, &new_wm);
1547
1548         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1549                 return;
1550
1551         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1552                 _intel_set_memory_cxsr(dev_priv, false);
1553
1554         g4x_write_wm_values(dev_priv, &new_wm);
1555
1556         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1557                 _intel_set_memory_cxsr(dev_priv, true);
1558
1559         *old_wm = new_wm;
1560 }
1561
1562 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1563                                    struct intel_crtc *crtc)
1564 {
1565         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1566         const struct intel_crtc_state *crtc_state =
1567                 intel_atomic_get_new_crtc_state(state, crtc);
1568
1569         mutex_lock(&dev_priv->display.wm.wm_mutex);
1570         crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1571         g4x_program_watermarks(dev_priv);
1572         mutex_unlock(&dev_priv->display.wm.wm_mutex);
1573 }
1574
1575 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1576                                     struct intel_crtc *crtc)
1577 {
1578         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1579         const struct intel_crtc_state *crtc_state =
1580                 intel_atomic_get_new_crtc_state(state, crtc);
1581
1582         if (!crtc_state->wm.need_postvbl_update)
1583                 return;
1584
1585         mutex_lock(&dev_priv->display.wm.wm_mutex);
1586         crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1587         g4x_program_watermarks(dev_priv);
1588         mutex_unlock(&dev_priv->display.wm.wm_mutex);
1589 }
1590
1591 /* latency must be in 0.1us units. */
1592 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1593                                    unsigned int htotal,
1594                                    unsigned int width,
1595                                    unsigned int cpp,
1596                                    unsigned int latency)
1597 {
1598         unsigned int ret;
1599
1600         ret = intel_wm_method2(pixel_rate, htotal,
1601                                width, cpp, latency);
1602         ret = DIV_ROUND_UP(ret, 64);
1603
1604         return ret;
1605 }
1606
1607 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1608 {
1609         /* all latencies in usec */
1610         dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1611
1612         dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM2;
1613
1614         if (IS_CHERRYVIEW(dev_priv)) {
1615                 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1616                 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1617
1618                 dev_priv->display.wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1619         }
1620 }
1621
1622 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1623                                 const struct intel_plane_state *plane_state,
1624                                 int level)
1625 {
1626         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1627         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1628         const struct drm_display_mode *pipe_mode =
1629                 &crtc_state->hw.pipe_mode;
1630         unsigned int pixel_rate, htotal, cpp, width, wm;
1631
1632         if (dev_priv->display.wm.pri_latency[level] == 0)
1633                 return USHRT_MAX;
1634
1635         if (!intel_wm_plane_visible(crtc_state, plane_state))
1636                 return 0;
1637
1638         cpp = plane_state->hw.fb->format->cpp[0];
1639         pixel_rate = crtc_state->pixel_rate;
1640         htotal = pipe_mode->crtc_htotal;
1641         width = drm_rect_width(&plane_state->uapi.src) >> 16;
1642
1643         if (plane->id == PLANE_CURSOR) {
1644                 /*
1645                  * FIXME the formula gives values that are
1646                  * too big for the cursor FIFO, and hence we
1647                  * would never be able to use cursors. For
1648                  * now just hardcode the watermark.
1649                  */
1650                 wm = 63;
1651         } else {
1652                 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1653                                     dev_priv->display.wm.pri_latency[level] * 10);
1654         }
1655
1656         return min_t(unsigned int, wm, USHRT_MAX);
1657 }
1658
1659 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1660 {
1661         return (active_planes & (BIT(PLANE_SPRITE0) |
1662                                  BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1663 }
1664
1665 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1666 {
1667         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1668         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1669         const struct g4x_pipe_wm *raw =
1670                 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1671         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1672         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1673         int num_active_planes = hweight8(active_planes);
1674         const int fifo_size = 511;
1675         int fifo_extra, fifo_left = fifo_size;
1676         int sprite0_fifo_extra = 0;
1677         unsigned int total_rate;
1678         enum plane_id plane_id;
1679
1680         /*
1681          * When enabling sprite0 after sprite1 has already been enabled
1682          * we tend to get an underrun unless sprite0 already has some
1683          * FIFO space allcoated. Hence we always allocate at least one
1684          * cacheline for sprite0 whenever sprite1 is enabled.
1685          *
1686          * All other plane enable sequences appear immune to this problem.
1687          */
1688         if (vlv_need_sprite0_fifo_workaround(active_planes))
1689                 sprite0_fifo_extra = 1;
1690
1691         total_rate = raw->plane[PLANE_PRIMARY] +
1692                 raw->plane[PLANE_SPRITE0] +
1693                 raw->plane[PLANE_SPRITE1] +
1694                 sprite0_fifo_extra;
1695
1696         if (total_rate > fifo_size)
1697                 return -EINVAL;
1698
1699         if (total_rate == 0)
1700                 total_rate = 1;
1701
1702         for_each_plane_id_on_crtc(crtc, plane_id) {
1703                 unsigned int rate;
1704
1705                 if ((active_planes & BIT(plane_id)) == 0) {
1706                         fifo_state->plane[plane_id] = 0;
1707                         continue;
1708                 }
1709
1710                 rate = raw->plane[plane_id];
1711                 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1712                 fifo_left -= fifo_state->plane[plane_id];
1713         }
1714
1715         fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1716         fifo_left -= sprite0_fifo_extra;
1717
1718         fifo_state->plane[PLANE_CURSOR] = 63;
1719
1720         fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1721
1722         /* spread the remainder evenly */
1723         for_each_plane_id_on_crtc(crtc, plane_id) {
1724                 int plane_extra;
1725
1726                 if (fifo_left == 0)
1727                         break;
1728
1729                 if ((active_planes & BIT(plane_id)) == 0)
1730                         continue;
1731
1732                 plane_extra = min(fifo_extra, fifo_left);
1733                 fifo_state->plane[plane_id] += plane_extra;
1734                 fifo_left -= plane_extra;
1735         }
1736
1737         drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1738
1739         /* give it all to the first plane if none are active */
1740         if (active_planes == 0) {
1741                 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1742                 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1743         }
1744
1745         return 0;
1746 }
1747
1748 /* mark all levels starting from 'level' as invalid */
1749 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1750                                struct vlv_wm_state *wm_state, int level)
1751 {
1752         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1753
1754         for (; level < intel_wm_num_levels(dev_priv); level++) {
1755                 enum plane_id plane_id;
1756
1757                 for_each_plane_id_on_crtc(crtc, plane_id)
1758                         wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1759
1760                 wm_state->sr[level].cursor = USHRT_MAX;
1761                 wm_state->sr[level].plane = USHRT_MAX;
1762         }
1763 }
1764
1765 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1766 {
1767         if (wm > fifo_size)
1768                 return USHRT_MAX;
1769         else
1770                 return fifo_size - wm;
1771 }
1772
1773 /*
1774  * Starting from 'level' set all higher
1775  * levels to 'value' in the "raw" watermarks.
1776  */
1777 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1778                                  int level, enum plane_id plane_id, u16 value)
1779 {
1780         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1781         int num_levels = intel_wm_num_levels(dev_priv);
1782         bool dirty = false;
1783
1784         for (; level < num_levels; level++) {
1785                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1786
1787                 dirty |= raw->plane[plane_id] != value;
1788                 raw->plane[plane_id] = value;
1789         }
1790
1791         return dirty;
1792 }
1793
1794 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1795                                      const struct intel_plane_state *plane_state)
1796 {
1797         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1798         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1799         enum plane_id plane_id = plane->id;
1800         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1801         int level;
1802         bool dirty = false;
1803
1804         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1805                 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1806                 goto out;
1807         }
1808
1809         for (level = 0; level < num_levels; level++) {
1810                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1811                 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1812                 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1813
1814                 if (wm > max_wm)
1815                         break;
1816
1817                 dirty |= raw->plane[plane_id] != wm;
1818                 raw->plane[plane_id] = wm;
1819         }
1820
1821         /* mark all higher levels as invalid */
1822         dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1823
1824 out:
1825         if (dirty)
1826                 drm_dbg_kms(&dev_priv->drm,
1827                             "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1828                             plane->base.name,
1829                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1830                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1831                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1832
1833         return dirty;
1834 }
1835
1836 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1837                                       enum plane_id plane_id, int level)
1838 {
1839         const struct g4x_pipe_wm *raw =
1840                 &crtc_state->wm.vlv.raw[level];
1841         const struct vlv_fifo_state *fifo_state =
1842                 &crtc_state->wm.vlv.fifo_state;
1843
1844         return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1845 }
1846
1847 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1848 {
1849         return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1850                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1851                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1852                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1853 }
1854
1855 static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1856 {
1857         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1858         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1859         struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1860         const struct vlv_fifo_state *fifo_state =
1861                 &crtc_state->wm.vlv.fifo_state;
1862         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1863         int num_active_planes = hweight8(active_planes);
1864         enum plane_id plane_id;
1865         int level;
1866
1867         /* initially allow all levels */
1868         wm_state->num_levels = intel_wm_num_levels(dev_priv);
1869         /*
1870          * Note that enabling cxsr with no primary/sprite planes
1871          * enabled can wedge the pipe. Hence we only allow cxsr
1872          * with exactly one enabled primary/sprite plane.
1873          */
1874         wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1875
1876         for (level = 0; level < wm_state->num_levels; level++) {
1877                 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1878                 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1879
1880                 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1881                         break;
1882
1883                 for_each_plane_id_on_crtc(crtc, plane_id) {
1884                         wm_state->wm[level].plane[plane_id] =
1885                                 vlv_invert_wm_value(raw->plane[plane_id],
1886                                                     fifo_state->plane[plane_id]);
1887                 }
1888
1889                 wm_state->sr[level].plane =
1890                         vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1891                                                  raw->plane[PLANE_SPRITE0],
1892                                                  raw->plane[PLANE_SPRITE1]),
1893                                             sr_fifo_size);
1894
1895                 wm_state->sr[level].cursor =
1896                         vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1897                                             63);
1898         }
1899
1900         if (level == 0)
1901                 return -EINVAL;
1902
1903         /* limit to only levels we can actually handle */
1904         wm_state->num_levels = level;
1905
1906         /* invalidate the higher levels */
1907         vlv_invalidate_wms(crtc, wm_state, level);
1908
1909         return 0;
1910 }
1911
1912 static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1913                                struct intel_crtc *crtc)
1914 {
1915         struct intel_crtc_state *crtc_state =
1916                 intel_atomic_get_new_crtc_state(state, crtc);
1917         const struct intel_plane_state *old_plane_state;
1918         const struct intel_plane_state *new_plane_state;
1919         struct intel_plane *plane;
1920         unsigned int dirty = 0;
1921         int i;
1922
1923         for_each_oldnew_intel_plane_in_state(state, plane,
1924                                              old_plane_state,
1925                                              new_plane_state, i) {
1926                 if (new_plane_state->hw.crtc != &crtc->base &&
1927                     old_plane_state->hw.crtc != &crtc->base)
1928                         continue;
1929
1930                 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1931                         dirty |= BIT(plane->id);
1932         }
1933
1934         /*
1935          * DSPARB registers may have been reset due to the
1936          * power well being turned off. Make sure we restore
1937          * them to a consistent state even if no primary/sprite
1938          * planes are initially active. We also force a FIFO
1939          * recomputation so that we are sure to sanitize the
1940          * FIFO setting we took over from the BIOS even if there
1941          * are no active planes on the crtc.
1942          */
1943         if (intel_crtc_needs_modeset(crtc_state))
1944                 dirty = ~0;
1945
1946         if (!dirty)
1947                 return 0;
1948
1949         /* cursor changes don't warrant a FIFO recompute */
1950         if (dirty & ~BIT(PLANE_CURSOR)) {
1951                 const struct intel_crtc_state *old_crtc_state =
1952                         intel_atomic_get_old_crtc_state(state, crtc);
1953                 const struct vlv_fifo_state *old_fifo_state =
1954                         &old_crtc_state->wm.vlv.fifo_state;
1955                 const struct vlv_fifo_state *new_fifo_state =
1956                         &crtc_state->wm.vlv.fifo_state;
1957                 int ret;
1958
1959                 ret = vlv_compute_fifo(crtc_state);
1960                 if (ret)
1961                         return ret;
1962
1963                 if (intel_crtc_needs_modeset(crtc_state) ||
1964                     memcmp(old_fifo_state, new_fifo_state,
1965                            sizeof(*new_fifo_state)) != 0)
1966                         crtc_state->fifo_changed = true;
1967         }
1968
1969         return _vlv_compute_pipe_wm(crtc_state);
1970 }
1971
1972 #define VLV_FIFO(plane, value) \
1973         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1974
1975 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1976                                    struct intel_crtc *crtc)
1977 {
1978         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1979         struct intel_uncore *uncore = &dev_priv->uncore;
1980         const struct intel_crtc_state *crtc_state =
1981                 intel_atomic_get_new_crtc_state(state, crtc);
1982         const struct vlv_fifo_state *fifo_state =
1983                 &crtc_state->wm.vlv.fifo_state;
1984         int sprite0_start, sprite1_start, fifo_size;
1985         u32 dsparb, dsparb2, dsparb3;
1986
1987         if (!crtc_state->fifo_changed)
1988                 return;
1989
1990         sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1991         sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1992         fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1993
1994         drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1995         drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
1996
1997         trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1998
1999         /*
2000          * uncore.lock serves a double purpose here. It allows us to
2001          * use the less expensive I915_{READ,WRITE}_FW() functions, and
2002          * it protects the DSPARB registers from getting clobbered by
2003          * parallel updates from multiple pipes.
2004          *
2005          * intel_pipe_update_start() has already disabled interrupts
2006          * for us, so a plain spin_lock() is sufficient here.
2007          */
2008         spin_lock(&uncore->lock);
2009
2010         switch (crtc->pipe) {
2011         case PIPE_A:
2012                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2013                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2014
2015                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2016                             VLV_FIFO(SPRITEB, 0xff));
2017                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2018                            VLV_FIFO(SPRITEB, sprite1_start));
2019
2020                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2021                              VLV_FIFO(SPRITEB_HI, 0x1));
2022                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2023                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2024
2025                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2026                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2027                 break;
2028         case PIPE_B:
2029                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2030                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2031
2032                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2033                             VLV_FIFO(SPRITED, 0xff));
2034                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2035                            VLV_FIFO(SPRITED, sprite1_start));
2036
2037                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2038                              VLV_FIFO(SPRITED_HI, 0xff));
2039                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2040                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2041
2042                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2043                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2044                 break;
2045         case PIPE_C:
2046                 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2047                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2048
2049                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2050                              VLV_FIFO(SPRITEF, 0xff));
2051                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2052                             VLV_FIFO(SPRITEF, sprite1_start));
2053
2054                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2055                              VLV_FIFO(SPRITEF_HI, 0xff));
2056                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2057                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2058
2059                 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2060                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2061                 break;
2062         default:
2063                 break;
2064         }
2065
2066         intel_uncore_posting_read_fw(uncore, DSPARB);
2067
2068         spin_unlock(&uncore->lock);
2069 }
2070
2071 #undef VLV_FIFO
2072
2073 static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
2074                                        struct intel_crtc *crtc)
2075 {
2076         struct intel_crtc_state *new_crtc_state =
2077                 intel_atomic_get_new_crtc_state(state, crtc);
2078         const struct intel_crtc_state *old_crtc_state =
2079                 intel_atomic_get_old_crtc_state(state, crtc);
2080         struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2081         const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2082         const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2083         int level;
2084
2085         if (!new_crtc_state->hw.active ||
2086             intel_crtc_needs_modeset(new_crtc_state)) {
2087                 *intermediate = *optimal;
2088
2089                 intermediate->cxsr = false;
2090                 goto out;
2091         }
2092
2093         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2094         intermediate->cxsr = optimal->cxsr && active->cxsr &&
2095                 !new_crtc_state->disable_cxsr;
2096
2097         for (level = 0; level < intermediate->num_levels; level++) {
2098                 enum plane_id plane_id;
2099
2100                 for_each_plane_id_on_crtc(crtc, plane_id) {
2101                         intermediate->wm[level].plane[plane_id] =
2102                                 min(optimal->wm[level].plane[plane_id],
2103                                     active->wm[level].plane[plane_id]);
2104                 }
2105
2106                 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2107                                                     active->sr[level].plane);
2108                 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2109                                                      active->sr[level].cursor);
2110         }
2111
2112         vlv_invalidate_wms(crtc, intermediate, level);
2113
2114 out:
2115         /*
2116          * If our intermediate WM are identical to the final WM, then we can
2117          * omit the post-vblank programming; only update if it's different.
2118          */
2119         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2120                 new_crtc_state->wm.need_postvbl_update = true;
2121
2122         return 0;
2123 }
2124
2125 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2126                          struct vlv_wm_values *wm)
2127 {
2128         struct intel_crtc *crtc;
2129         int num_active_pipes = 0;
2130
2131         wm->level = dev_priv->display.wm.max_level;
2132         wm->cxsr = true;
2133
2134         for_each_intel_crtc(&dev_priv->drm, crtc) {
2135                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2136
2137                 if (!crtc->active)
2138                         continue;
2139
2140                 if (!wm_state->cxsr)
2141                         wm->cxsr = false;
2142
2143                 num_active_pipes++;
2144                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2145         }
2146
2147         if (num_active_pipes != 1)
2148                 wm->cxsr = false;
2149
2150         if (num_active_pipes > 1)
2151                 wm->level = VLV_WM_LEVEL_PM2;
2152
2153         for_each_intel_crtc(&dev_priv->drm, crtc) {
2154                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2155                 enum pipe pipe = crtc->pipe;
2156
2157                 wm->pipe[pipe] = wm_state->wm[wm->level];
2158                 if (crtc->active && wm->cxsr)
2159                         wm->sr = wm_state->sr[wm->level];
2160
2161                 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2162                 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2163                 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2164                 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2165         }
2166 }
2167
2168 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2169 {
2170         struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;
2171         struct vlv_wm_values new_wm = {};
2172
2173         vlv_merge_wm(dev_priv, &new_wm);
2174
2175         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2176                 return;
2177
2178         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2179                 chv_set_memory_dvfs(dev_priv, false);
2180
2181         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2182                 chv_set_memory_pm5(dev_priv, false);
2183
2184         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2185                 _intel_set_memory_cxsr(dev_priv, false);
2186
2187         vlv_write_wm_values(dev_priv, &new_wm);
2188
2189         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2190                 _intel_set_memory_cxsr(dev_priv, true);
2191
2192         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2193                 chv_set_memory_pm5(dev_priv, true);
2194
2195         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2196                 chv_set_memory_dvfs(dev_priv, true);
2197
2198         *old_wm = new_wm;
2199 }
2200
2201 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2202                                    struct intel_crtc *crtc)
2203 {
2204         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2205         const struct intel_crtc_state *crtc_state =
2206                 intel_atomic_get_new_crtc_state(state, crtc);
2207
2208         mutex_lock(&dev_priv->display.wm.wm_mutex);
2209         crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2210         vlv_program_watermarks(dev_priv);
2211         mutex_unlock(&dev_priv->display.wm.wm_mutex);
2212 }
2213
2214 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2215                                     struct intel_crtc *crtc)
2216 {
2217         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2218         const struct intel_crtc_state *crtc_state =
2219                 intel_atomic_get_new_crtc_state(state, crtc);
2220
2221         if (!crtc_state->wm.need_postvbl_update)
2222                 return;
2223
2224         mutex_lock(&dev_priv->display.wm.wm_mutex);
2225         crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2226         vlv_program_watermarks(dev_priv);
2227         mutex_unlock(&dev_priv->display.wm.wm_mutex);
2228 }
2229
2230 static void i965_update_wm(struct drm_i915_private *dev_priv)
2231 {
2232         struct intel_crtc *crtc;
2233         int srwm = 1;
2234         int cursor_sr = 16;
2235         bool cxsr_enabled;
2236
2237         /* Calc sr entries for one plane configs */
2238         crtc = single_enabled_crtc(dev_priv);
2239         if (crtc) {
2240                 /* self-refresh has much higher latency */
2241                 static const int sr_latency_ns = 12000;
2242                 const struct drm_display_mode *pipe_mode =
2243                         &crtc->config->hw.pipe_mode;
2244                 const struct drm_framebuffer *fb =
2245                         crtc->base.primary->state->fb;
2246                 int pixel_rate = crtc->config->pixel_rate;
2247                 int htotal = pipe_mode->crtc_htotal;
2248                 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2249                 int cpp = fb->format->cpp[0];
2250                 int entries;
2251
2252                 entries = intel_wm_method2(pixel_rate, htotal,
2253                                            width, cpp, sr_latency_ns / 100);
2254                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2255                 srwm = I965_FIFO_SIZE - entries;
2256                 if (srwm < 0)
2257                         srwm = 1;
2258                 srwm &= 0x1ff;
2259                 drm_dbg_kms(&dev_priv->drm,
2260                             "self-refresh entries: %d, wm: %d\n",
2261                             entries, srwm);
2262
2263                 entries = intel_wm_method2(pixel_rate, htotal,
2264                                            crtc->base.cursor->state->crtc_w, 4,
2265                                            sr_latency_ns / 100);
2266                 entries = DIV_ROUND_UP(entries,
2267                                        i965_cursor_wm_info.cacheline_size) +
2268                         i965_cursor_wm_info.guard_size;
2269
2270                 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2271                 if (cursor_sr > i965_cursor_wm_info.max_wm)
2272                         cursor_sr = i965_cursor_wm_info.max_wm;
2273
2274                 drm_dbg_kms(&dev_priv->drm,
2275                             "self-refresh watermark: display plane %d "
2276                             "cursor %d\n", srwm, cursor_sr);
2277
2278                 cxsr_enabled = true;
2279         } else {
2280                 cxsr_enabled = false;
2281                 /* Turn off self refresh if both pipes are enabled */
2282                 intel_set_memory_cxsr(dev_priv, false);
2283         }
2284
2285         drm_dbg_kms(&dev_priv->drm,
2286                     "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2287                     srwm);
2288
2289         /* 965 has limitations... */
2290         intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2291                    FW_WM(8, CURSORB) |
2292                    FW_WM(8, PLANEB) |
2293                    FW_WM(8, PLANEA));
2294         intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2295                    FW_WM(8, PLANEC_OLD));
2296         /* update cursor SR watermark */
2297         intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2298
2299         if (cxsr_enabled)
2300                 intel_set_memory_cxsr(dev_priv, true);
2301 }
2302
2303 #undef FW_WM
2304
2305 static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2306                                                enum i9xx_plane_id i9xx_plane)
2307 {
2308         struct intel_plane *plane;
2309
2310         for_each_intel_plane(&i915->drm, plane) {
2311                 if (plane->id == PLANE_PRIMARY &&
2312                     plane->i9xx_plane == i9xx_plane)
2313                         return intel_crtc_for_pipe(i915, plane->pipe);
2314         }
2315
2316         return NULL;
2317 }
2318
2319 static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2320 {
2321         const struct intel_watermark_params *wm_info;
2322         u32 fwater_lo;
2323         u32 fwater_hi;
2324         int cwm, srwm = 1;
2325         int fifo_size;
2326         int planea_wm, planeb_wm;
2327         struct intel_crtc *crtc;
2328
2329         if (IS_I945GM(dev_priv))
2330                 wm_info = &i945_wm_info;
2331         else if (DISPLAY_VER(dev_priv) != 2)
2332                 wm_info = &i915_wm_info;
2333         else
2334                 wm_info = &i830_a_wm_info;
2335
2336         if (DISPLAY_VER(dev_priv) == 2)
2337                 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2338         else
2339                 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2340         crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2341         if (intel_crtc_active(crtc)) {
2342                 const struct drm_framebuffer *fb =
2343                         crtc->base.primary->state->fb;
2344                 int cpp;
2345
2346                 if (DISPLAY_VER(dev_priv) == 2)
2347                         cpp = 4;
2348                 else
2349                         cpp = fb->format->cpp[0];
2350
2351                 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2352                                                wm_info, fifo_size, cpp,
2353                                                pessimal_latency_ns);
2354         } else {
2355                 planea_wm = fifo_size - wm_info->guard_size;
2356                 if (planea_wm > (long)wm_info->max_wm)
2357                         planea_wm = wm_info->max_wm;
2358         }
2359
2360         if (DISPLAY_VER(dev_priv) == 2)
2361                 wm_info = &i830_bc_wm_info;
2362
2363         if (DISPLAY_VER(dev_priv) == 2)
2364                 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2365         else
2366                 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2367         crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2368         if (intel_crtc_active(crtc)) {
2369                 const struct drm_framebuffer *fb =
2370                         crtc->base.primary->state->fb;
2371                 int cpp;
2372
2373                 if (DISPLAY_VER(dev_priv) == 2)
2374                         cpp = 4;
2375                 else
2376                         cpp = fb->format->cpp[0];
2377
2378                 planeb_wm = intel_calculate_wm(crtc->config->pixel_rate,
2379                                                wm_info, fifo_size, cpp,
2380                                                pessimal_latency_ns);
2381         } else {
2382                 planeb_wm = fifo_size - wm_info->guard_size;
2383                 if (planeb_wm > (long)wm_info->max_wm)
2384                         planeb_wm = wm_info->max_wm;
2385         }
2386
2387         drm_dbg_kms(&dev_priv->drm,
2388                     "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2389
2390         crtc = single_enabled_crtc(dev_priv);
2391         if (IS_I915GM(dev_priv) && crtc) {
2392                 struct drm_i915_gem_object *obj;
2393
2394                 obj = intel_fb_obj(crtc->base.primary->state->fb);
2395
2396                 /* self-refresh seems busted with untiled */
2397                 if (!i915_gem_object_is_tiled(obj))
2398                         crtc = NULL;
2399         }
2400
2401         /*
2402          * Overlay gets an aggressive default since video jitter is bad.
2403          */
2404         cwm = 2;
2405
2406         /* Play safe and disable self-refresh before adjusting watermarks. */
2407         intel_set_memory_cxsr(dev_priv, false);
2408
2409         /* Calc sr entries for one plane configs */
2410         if (HAS_FW_BLC(dev_priv) && crtc) {
2411                 /* self-refresh has much higher latency */
2412                 static const int sr_latency_ns = 6000;
2413                 const struct drm_display_mode *pipe_mode =
2414                         &crtc->config->hw.pipe_mode;
2415                 const struct drm_framebuffer *fb =
2416                         crtc->base.primary->state->fb;
2417                 int pixel_rate = crtc->config->pixel_rate;
2418                 int htotal = pipe_mode->crtc_htotal;
2419                 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2420                 int cpp;
2421                 int entries;
2422
2423                 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2424                         cpp = 4;
2425                 else
2426                         cpp = fb->format->cpp[0];
2427
2428                 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2429                                            sr_latency_ns / 100);
2430                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2431                 drm_dbg_kms(&dev_priv->drm,
2432                             "self-refresh entries: %d\n", entries);
2433                 srwm = wm_info->fifo_size - entries;
2434                 if (srwm < 0)
2435                         srwm = 1;
2436
2437                 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2438                         intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2439                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2440                 else
2441                         intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2442         }
2443
2444         drm_dbg_kms(&dev_priv->drm,
2445                     "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2446                      planea_wm, planeb_wm, cwm, srwm);
2447
2448         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2449         fwater_hi = (cwm & 0x1f);
2450
2451         /* Set request length to 8 cachelines per fetch */
2452         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2453         fwater_hi = fwater_hi | (1 << 8);
2454
2455         intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2456         intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2457
2458         if (crtc)
2459                 intel_set_memory_cxsr(dev_priv, true);
2460 }
2461
2462 static void i845_update_wm(struct drm_i915_private *dev_priv)
2463 {
2464         struct intel_crtc *crtc;
2465         u32 fwater_lo;
2466         int planea_wm;
2467
2468         crtc = single_enabled_crtc(dev_priv);
2469         if (crtc == NULL)
2470                 return;
2471
2472         planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2473                                        &i845_wm_info,
2474                                        i845_get_fifo_size(dev_priv, PLANE_A),
2475                                        4, pessimal_latency_ns);
2476         fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2477         fwater_lo |= (3<<8) | planea_wm;
2478
2479         drm_dbg_kms(&dev_priv->drm,
2480                     "Setting FIFO watermarks - A: %d\n", planea_wm);
2481
2482         intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2483 }
2484
2485 /* latency must be in 0.1us units. */
2486 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2487                                    unsigned int cpp,
2488                                    unsigned int latency)
2489 {
2490         unsigned int ret;
2491
2492         ret = intel_wm_method1(pixel_rate, cpp, latency);
2493         ret = DIV_ROUND_UP(ret, 64) + 2;
2494
2495         return ret;
2496 }
2497
2498 /* latency must be in 0.1us units. */
2499 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2500                                    unsigned int htotal,
2501                                    unsigned int width,
2502                                    unsigned int cpp,
2503                                    unsigned int latency)
2504 {
2505         unsigned int ret;
2506
2507         ret = intel_wm_method2(pixel_rate, htotal,
2508                                width, cpp, latency);
2509         ret = DIV_ROUND_UP(ret, 64) + 2;
2510
2511         return ret;
2512 }
2513
2514 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2515 {
2516         /*
2517          * Neither of these should be possible since this function shouldn't be
2518          * called if the CRTC is off or the plane is invisible.  But let's be
2519          * extra paranoid to avoid a potential divide-by-zero if we screw up
2520          * elsewhere in the driver.
2521          */
2522         if (WARN_ON(!cpp))
2523                 return 0;
2524         if (WARN_ON(!horiz_pixels))
2525                 return 0;
2526
2527         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2528 }
2529
2530 struct ilk_wm_maximums {
2531         u16 pri;
2532         u16 spr;
2533         u16 cur;
2534         u16 fbc;
2535 };
2536
2537 /*
2538  * For both WM_PIPE and WM_LP.
2539  * mem_value must be in 0.1us units.
2540  */
2541 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2542                               const struct intel_plane_state *plane_state,
2543                               u32 mem_value, bool is_lp)
2544 {
2545         u32 method1, method2;
2546         int cpp;
2547
2548         if (mem_value == 0)
2549                 return U32_MAX;
2550
2551         if (!intel_wm_plane_visible(crtc_state, plane_state))
2552                 return 0;
2553
2554         cpp = plane_state->hw.fb->format->cpp[0];
2555
2556         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2557
2558         if (!is_lp)
2559                 return method1;
2560
2561         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2562                                  crtc_state->hw.pipe_mode.crtc_htotal,
2563                                  drm_rect_width(&plane_state->uapi.src) >> 16,
2564                                  cpp, mem_value);
2565
2566         return min(method1, method2);
2567 }
2568
2569 /*
2570  * For both WM_PIPE and WM_LP.
2571  * mem_value must be in 0.1us units.
2572  */
2573 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2574                               const struct intel_plane_state *plane_state,
2575                               u32 mem_value)
2576 {
2577         u32 method1, method2;
2578         int cpp;
2579
2580         if (mem_value == 0)
2581                 return U32_MAX;
2582
2583         if (!intel_wm_plane_visible(crtc_state, plane_state))
2584                 return 0;
2585
2586         cpp = plane_state->hw.fb->format->cpp[0];
2587
2588         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2589         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2590                                  crtc_state->hw.pipe_mode.crtc_htotal,
2591                                  drm_rect_width(&plane_state->uapi.src) >> 16,
2592                                  cpp, mem_value);
2593         return min(method1, method2);
2594 }
2595
2596 /*
2597  * For both WM_PIPE and WM_LP.
2598  * mem_value must be in 0.1us units.
2599  */
2600 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2601                               const struct intel_plane_state *plane_state,
2602                               u32 mem_value)
2603 {
2604         int cpp;
2605
2606         if (mem_value == 0)
2607                 return U32_MAX;
2608
2609         if (!intel_wm_plane_visible(crtc_state, plane_state))
2610                 return 0;
2611
2612         cpp = plane_state->hw.fb->format->cpp[0];
2613
2614         return ilk_wm_method2(crtc_state->pixel_rate,
2615                               crtc_state->hw.pipe_mode.crtc_htotal,
2616                               drm_rect_width(&plane_state->uapi.src) >> 16,
2617                               cpp, mem_value);
2618 }
2619
2620 /* Only for WM_LP. */
2621 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2622                               const struct intel_plane_state *plane_state,
2623                               u32 pri_val)
2624 {
2625         int cpp;
2626
2627         if (!intel_wm_plane_visible(crtc_state, plane_state))
2628                 return 0;
2629
2630         cpp = plane_state->hw.fb->format->cpp[0];
2631
2632         return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2633                           cpp);
2634 }
2635
2636 static unsigned int
2637 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2638 {
2639         if (DISPLAY_VER(dev_priv) >= 8)
2640                 return 3072;
2641         else if (DISPLAY_VER(dev_priv) >= 7)
2642                 return 768;
2643         else
2644                 return 512;
2645 }
2646
2647 static unsigned int
2648 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2649                      int level, bool is_sprite)
2650 {
2651         if (DISPLAY_VER(dev_priv) >= 8)
2652                 /* BDW primary/sprite plane watermarks */
2653                 return level == 0 ? 255 : 2047;
2654         else if (DISPLAY_VER(dev_priv) >= 7)
2655                 /* IVB/HSW primary/sprite plane watermarks */
2656                 return level == 0 ? 127 : 1023;
2657         else if (!is_sprite)
2658                 /* ILK/SNB primary plane watermarks */
2659                 return level == 0 ? 127 : 511;
2660         else
2661                 /* ILK/SNB sprite plane watermarks */
2662                 return level == 0 ? 63 : 255;
2663 }
2664
2665 static unsigned int
2666 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2667 {
2668         if (DISPLAY_VER(dev_priv) >= 7)
2669                 return level == 0 ? 63 : 255;
2670         else
2671                 return level == 0 ? 31 : 63;
2672 }
2673
2674 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2675 {
2676         if (DISPLAY_VER(dev_priv) >= 8)
2677                 return 31;
2678         else
2679                 return 15;
2680 }
2681
2682 /* Calculate the maximum primary/sprite plane watermark */
2683 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2684                                      int level,
2685                                      const struct intel_wm_config *config,
2686                                      enum intel_ddb_partitioning ddb_partitioning,
2687                                      bool is_sprite)
2688 {
2689         unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2690
2691         /* if sprites aren't enabled, sprites get nothing */
2692         if (is_sprite && !config->sprites_enabled)
2693                 return 0;
2694
2695         /* HSW allows LP1+ watermarks even with multiple pipes */
2696         if (level == 0 || config->num_pipes_active > 1) {
2697                 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2698
2699                 /*
2700                  * For some reason the non self refresh
2701                  * FIFO size is only half of the self
2702                  * refresh FIFO size on ILK/SNB.
2703                  */
2704                 if (DISPLAY_VER(dev_priv) <= 6)
2705                         fifo_size /= 2;
2706         }
2707
2708         if (config->sprites_enabled) {
2709                 /* level 0 is always calculated with 1:1 split */
2710                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2711                         if (is_sprite)
2712                                 fifo_size *= 5;
2713                         fifo_size /= 6;
2714                 } else {
2715                         fifo_size /= 2;
2716                 }
2717         }
2718
2719         /* clamp to max that the registers can hold */
2720         return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2721 }
2722
2723 /* Calculate the maximum cursor plane watermark */
2724 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2725                                       int level,
2726                                       const struct intel_wm_config *config)
2727 {
2728         /* HSW LP1+ watermarks w/ multiple pipes */
2729         if (level > 0 && config->num_pipes_active > 1)
2730                 return 64;
2731
2732         /* otherwise just report max that registers can hold */
2733         return ilk_cursor_wm_reg_max(dev_priv, level);
2734 }
2735
2736 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2737                                     int level,
2738                                     const struct intel_wm_config *config,
2739                                     enum intel_ddb_partitioning ddb_partitioning,
2740                                     struct ilk_wm_maximums *max)
2741 {
2742         max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2743         max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2744         max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2745         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2746 }
2747
2748 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2749                                         int level,
2750                                         struct ilk_wm_maximums *max)
2751 {
2752         max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2753         max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2754         max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2755         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2756 }
2757
2758 static bool ilk_validate_wm_level(int level,
2759                                   const struct ilk_wm_maximums *max,
2760                                   struct intel_wm_level *result)
2761 {
2762         bool ret;
2763
2764         /* already determined to be invalid? */
2765         if (!result->enable)
2766                 return false;
2767
2768         result->enable = result->pri_val <= max->pri &&
2769                          result->spr_val <= max->spr &&
2770                          result->cur_val <= max->cur;
2771
2772         ret = result->enable;
2773
2774         /*
2775          * HACK until we can pre-compute everything,
2776          * and thus fail gracefully if LP0 watermarks
2777          * are exceeded...
2778          */
2779         if (level == 0 && !result->enable) {
2780                 if (result->pri_val > max->pri)
2781                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2782                                       level, result->pri_val, max->pri);
2783                 if (result->spr_val > max->spr)
2784                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2785                                       level, result->spr_val, max->spr);
2786                 if (result->cur_val > max->cur)
2787                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2788                                       level, result->cur_val, max->cur);
2789
2790                 result->pri_val = min_t(u32, result->pri_val, max->pri);
2791                 result->spr_val = min_t(u32, result->spr_val, max->spr);
2792                 result->cur_val = min_t(u32, result->cur_val, max->cur);
2793                 result->enable = true;
2794         }
2795
2796         return ret;
2797 }
2798
2799 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2800                                  const struct intel_crtc *crtc,
2801                                  int level,
2802                                  struct intel_crtc_state *crtc_state,
2803                                  const struct intel_plane_state *pristate,
2804                                  const struct intel_plane_state *sprstate,
2805                                  const struct intel_plane_state *curstate,
2806                                  struct intel_wm_level *result)
2807 {
2808         u16 pri_latency = dev_priv->display.wm.pri_latency[level];
2809         u16 spr_latency = dev_priv->display.wm.spr_latency[level];
2810         u16 cur_latency = dev_priv->display.wm.cur_latency[level];
2811
2812         /* WM1+ latency values stored in 0.5us units */
2813         if (level > 0) {
2814                 pri_latency *= 5;
2815                 spr_latency *= 5;
2816                 cur_latency *= 5;
2817         }
2818
2819         if (pristate) {
2820                 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2821                                                      pri_latency, level);
2822                 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2823         }
2824
2825         if (sprstate)
2826                 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2827
2828         if (curstate)
2829                 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2830
2831         result->enable = true;
2832 }
2833
2834 static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2835 {
2836         u64 sskpd;
2837
2838         sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);
2839
2840         wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2841         if (wm[0] == 0)
2842                 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2843         wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2844         wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2845         wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2846         wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2847 }
2848
2849 static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2850 {
2851         u32 sskpd;
2852
2853         sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);
2854
2855         wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2856         wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2857         wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2858         wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2859 }
2860
2861 static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2862 {
2863         u32 mltr;
2864
2865         mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);
2866
2867         /* ILK primary LP0 latency is 700 ns */
2868         wm[0] = 7;
2869         wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2870         wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2871 }
2872
2873 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2874                                        u16 wm[5])
2875 {
2876         /* ILK sprite LP0 latency is 1300 ns */
2877         if (DISPLAY_VER(dev_priv) == 5)
2878                 wm[0] = 13;
2879 }
2880
2881 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2882                                        u16 wm[5])
2883 {
2884         /* ILK cursor LP0 latency is 1300 ns */
2885         if (DISPLAY_VER(dev_priv) == 5)
2886                 wm[0] = 13;
2887 }
2888
2889 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2890 {
2891         /* how many WM levels are we expecting */
2892         if (HAS_HW_SAGV_WM(dev_priv))
2893                 return 5;
2894         else if (DISPLAY_VER(dev_priv) >= 9)
2895                 return 7;
2896         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2897                 return 4;
2898         else if (DISPLAY_VER(dev_priv) >= 6)
2899                 return 3;
2900         else
2901                 return 2;
2902 }
2903
2904 void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2905                             const char *name, const u16 wm[])
2906 {
2907         int level, max_level = ilk_wm_max_level(dev_priv);
2908
2909         for (level = 0; level <= max_level; level++) {
2910                 unsigned int latency = wm[level];
2911
2912                 if (latency == 0) {
2913                         drm_dbg_kms(&dev_priv->drm,
2914                                     "%s WM%d latency not provided\n",
2915                                     name, level);
2916                         continue;
2917                 }
2918
2919                 /*
2920                  * - latencies are in us on gen9.
2921                  * - before then, WM1+ latency values are in 0.5us units
2922                  */
2923                 if (DISPLAY_VER(dev_priv) >= 9)
2924                         latency *= 10;
2925                 else if (level > 0)
2926                         latency *= 5;
2927
2928                 drm_dbg_kms(&dev_priv->drm,
2929                             "%s WM%d latency %u (%u.%u usec)\n", name, level,
2930                             wm[level], latency / 10, latency % 10);
2931         }
2932 }
2933
2934 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2935                                     u16 wm[5], u16 min)
2936 {
2937         int level, max_level = ilk_wm_max_level(dev_priv);
2938
2939         if (wm[0] >= min)
2940                 return false;
2941
2942         wm[0] = max(wm[0], min);
2943         for (level = 1; level <= max_level; level++)
2944                 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2945
2946         return true;
2947 }
2948
2949 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2950 {
2951         bool changed;
2952
2953         /*
2954          * The BIOS provided WM memory latency values are often
2955          * inadequate for high resolution displays. Adjust them.
2956          */
2957         changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);
2958         changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);
2959         changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);
2960
2961         if (!changed)
2962                 return;
2963
2964         drm_dbg_kms(&dev_priv->drm,
2965                     "WM latency values increased to avoid potential underruns\n");
2966         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2967         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2968         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2969 }
2970
2971 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
2972 {
2973         /*
2974          * On some SNB machines (Thinkpad X220 Tablet at least)
2975          * LP3 usage can cause vblank interrupts to be lost.
2976          * The DEIIR bit will go high but it looks like the CPU
2977          * never gets interrupted.
2978          *
2979          * It's not clear whether other interrupt source could
2980          * be affected or if this is somehow limited to vblank
2981          * interrupts only. To play it safe we disable LP3
2982          * watermarks entirely.
2983          */
2984         if (dev_priv->display.wm.pri_latency[3] == 0 &&
2985             dev_priv->display.wm.spr_latency[3] == 0 &&
2986             dev_priv->display.wm.cur_latency[3] == 0)
2987                 return;
2988
2989         dev_priv->display.wm.pri_latency[3] = 0;
2990         dev_priv->display.wm.spr_latency[3] = 0;
2991         dev_priv->display.wm.cur_latency[3] = 0;
2992
2993         drm_dbg_kms(&dev_priv->drm,
2994                     "LP3 watermarks disabled due to potential for lost interrupts\n");
2995         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2996         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2997         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2998 }
2999
3000 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3001 {
3002         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
3003                 hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3004         else if (DISPLAY_VER(dev_priv) >= 6)
3005                 snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3006         else
3007                 ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3008
3009         memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,
3010                sizeof(dev_priv->display.wm.pri_latency));
3011         memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,
3012                sizeof(dev_priv->display.wm.pri_latency));
3013
3014         intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);
3015         intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);
3016
3017         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
3018         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
3019         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
3020
3021         if (DISPLAY_VER(dev_priv) == 6) {
3022                 snb_wm_latency_quirk(dev_priv);
3023                 snb_wm_lp3_irq_quirk(dev_priv);
3024         }
3025 }
3026
3027 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3028                                  struct intel_pipe_wm *pipe_wm)
3029 {
3030         /* LP0 watermark maximums depend on this pipe alone */
3031         const struct intel_wm_config config = {
3032                 .num_pipes_active = 1,
3033                 .sprites_enabled = pipe_wm->sprites_enabled,
3034                 .sprites_scaled = pipe_wm->sprites_scaled,
3035         };
3036         struct ilk_wm_maximums max;
3037
3038         /* LP0 watermarks always use 1/2 DDB partitioning */
3039         ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3040
3041         /* At least LP0 must be valid */
3042         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3043                 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3044                 return false;
3045         }
3046
3047         return true;
3048 }
3049
3050 /* Compute new watermarks for the pipe */
3051 static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
3052                                struct intel_crtc *crtc)
3053 {
3054         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3055         struct intel_crtc_state *crtc_state =
3056                 intel_atomic_get_new_crtc_state(state, crtc);
3057         struct intel_pipe_wm *pipe_wm;
3058         struct intel_plane *plane;
3059         const struct intel_plane_state *plane_state;
3060         const struct intel_plane_state *pristate = NULL;
3061         const struct intel_plane_state *sprstate = NULL;
3062         const struct intel_plane_state *curstate = NULL;
3063         int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3064         struct ilk_wm_maximums max;
3065
3066         pipe_wm = &crtc_state->wm.ilk.optimal;
3067
3068         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3069                 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3070                         pristate = plane_state;
3071                 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3072                         sprstate = plane_state;
3073                 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3074                         curstate = plane_state;
3075         }
3076
3077         pipe_wm->pipe_enabled = crtc_state->hw.active;
3078         pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
3079         pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
3080
3081         usable_level = max_level;
3082
3083         /* ILK/SNB: LP2+ watermarks only w/o sprites */
3084         if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3085                 usable_level = 1;
3086
3087         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3088         if (pipe_wm->sprites_scaled)
3089                 usable_level = 0;
3090
3091         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3092         ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3093                              pristate, sprstate, curstate, &pipe_wm->wm[0]);
3094
3095         if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3096                 return -EINVAL;
3097
3098         ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3099
3100         for (level = 1; level <= usable_level; level++) {
3101                 struct intel_wm_level *wm = &pipe_wm->wm[level];
3102
3103                 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3104                                      pristate, sprstate, curstate, wm);
3105
3106                 /*
3107                  * Disable any watermark level that exceeds the
3108                  * register maximums since such watermarks are
3109                  * always invalid.
3110                  */
3111                 if (!ilk_validate_wm_level(level, &max, wm)) {
3112                         memset(wm, 0, sizeof(*wm));
3113                         break;
3114                 }
3115         }
3116
3117         return 0;
3118 }
3119
3120 /*
3121  * Build a set of 'intermediate' watermark values that satisfy both the old
3122  * state and the new state.  These can be programmed to the hardware
3123  * immediately.
3124  */
3125 static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
3126                                        struct intel_crtc *crtc)
3127 {
3128         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3129         struct intel_crtc_state *new_crtc_state =
3130                 intel_atomic_get_new_crtc_state(state, crtc);
3131         const struct intel_crtc_state *old_crtc_state =
3132                 intel_atomic_get_old_crtc_state(state, crtc);
3133         struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
3134         const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
3135         int level, max_level = ilk_wm_max_level(dev_priv);
3136
3137         /*
3138          * Start with the final, target watermarks, then combine with the
3139          * currently active watermarks to get values that are safe both before
3140          * and after the vblank.
3141          */
3142         *a = new_crtc_state->wm.ilk.optimal;
3143         if (!new_crtc_state->hw.active ||
3144             intel_crtc_needs_modeset(new_crtc_state) ||
3145             state->skip_intermediate_wm)
3146                 return 0;
3147
3148         a->pipe_enabled |= b->pipe_enabled;
3149         a->sprites_enabled |= b->sprites_enabled;
3150         a->sprites_scaled |= b->sprites_scaled;
3151
3152         for (level = 0; level <= max_level; level++) {
3153                 struct intel_wm_level *a_wm = &a->wm[level];
3154                 const struct intel_wm_level *b_wm = &b->wm[level];
3155
3156                 a_wm->enable &= b_wm->enable;
3157                 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3158                 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3159                 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3160                 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3161         }
3162
3163         /*
3164          * We need to make sure that these merged watermark values are
3165          * actually a valid configuration themselves.  If they're not,
3166          * there's no safe way to transition from the old state to
3167          * the new state, so we need to fail the atomic transaction.
3168          */
3169         if (!ilk_validate_pipe_wm(dev_priv, a))
3170                 return -EINVAL;
3171
3172         /*
3173          * If our intermediate WM are identical to the final WM, then we can
3174          * omit the post-vblank programming; only update if it's different.
3175          */
3176         if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
3177                 new_crtc_state->wm.need_postvbl_update = true;
3178
3179         return 0;
3180 }
3181
3182 /*
3183  * Merge the watermarks from all active pipes for a specific level.
3184  */
3185 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3186                                int level,
3187                                struct intel_wm_level *ret_wm)
3188 {
3189         const struct intel_crtc *crtc;
3190
3191         ret_wm->enable = true;
3192
3193         for_each_intel_crtc(&dev_priv->drm, crtc) {
3194                 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3195                 const struct intel_wm_level *wm = &active->wm[level];
3196
3197                 if (!active->pipe_enabled)
3198                         continue;
3199
3200                 /*
3201                  * The watermark values may have been used in the past,
3202                  * so we must maintain them in the registers for some
3203                  * time even if the level is now disabled.
3204                  */
3205                 if (!wm->enable)
3206                         ret_wm->enable = false;
3207
3208                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3209                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3210                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3211                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3212         }
3213 }
3214
3215 /*
3216  * Merge all low power watermarks for all active pipes.
3217  */
3218 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3219                          const struct intel_wm_config *config,
3220                          const struct ilk_wm_maximums *max,
3221                          struct intel_pipe_wm *merged)
3222 {
3223         int level, max_level = ilk_wm_max_level(dev_priv);
3224         int last_enabled_level = max_level;
3225
3226         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3227         if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3228             config->num_pipes_active > 1)
3229                 last_enabled_level = 0;
3230
3231         /* ILK: FBC WM must be disabled always */
3232         merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3233
3234         /* merge each WM1+ level */
3235         for (level = 1; level <= max_level; level++) {
3236                 struct intel_wm_level *wm = &merged->wm[level];
3237
3238                 ilk_merge_wm_level(dev_priv, level, wm);
3239
3240                 if (level > last_enabled_level)
3241                         wm->enable = false;
3242                 else if (!ilk_validate_wm_level(level, max, wm))
3243                         /* make sure all following levels get disabled */
3244                         last_enabled_level = level - 1;
3245
3246                 /*
3247                  * The spec says it is preferred to disable
3248                  * FBC WMs instead of disabling a WM level.
3249                  */
3250                 if (wm->fbc_val > max->fbc) {
3251                         if (wm->enable)
3252                                 merged->fbc_wm_enabled = false;
3253                         wm->fbc_val = 0;
3254                 }
3255         }
3256
3257         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3258         if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
3259             dev_priv->params.enable_fbc && !merged->fbc_wm_enabled) {
3260                 for (level = 2; level <= max_level; level++) {
3261                         struct intel_wm_level *wm = &merged->wm[level];
3262
3263                         wm->enable = false;
3264                 }
3265         }
3266 }
3267
3268 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3269 {
3270         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3271         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3272 }
3273
3274 /* The value we need to program into the WM_LPx latency field */
3275 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3276                                       int level)
3277 {
3278         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3279                 return 2 * level;
3280         else
3281                 return dev_priv->display.wm.pri_latency[level];
3282 }
3283
3284 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3285                                    const struct intel_pipe_wm *merged,
3286                                    enum intel_ddb_partitioning partitioning,
3287                                    struct ilk_wm_values *results)
3288 {
3289         struct intel_crtc *crtc;
3290         int level, wm_lp;
3291
3292         results->enable_fbc_wm = merged->fbc_wm_enabled;
3293         results->partitioning = partitioning;
3294
3295         /* LP1+ register values */
3296         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3297                 const struct intel_wm_level *r;
3298
3299                 level = ilk_wm_lp_to_level(wm_lp, merged);
3300
3301                 r = &merged->wm[level];
3302
3303                 /*
3304                  * Maintain the watermark values even if the level is
3305                  * disabled. Doing otherwise could cause underruns.
3306                  */
3307                 results->wm_lp[wm_lp - 1] =
3308                         WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3309                         WM_LP_PRIMARY(r->pri_val) |
3310                         WM_LP_CURSOR(r->cur_val);
3311
3312                 if (r->enable)
3313                         results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3314
3315                 if (DISPLAY_VER(dev_priv) >= 8)
3316                         results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3317                 else
3318                         results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3319
3320                 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3321
3322                 /*
3323                  * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3324                  * level is disabled. Doing otherwise could cause underruns.
3325                  */
3326                 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3327                         drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3328                         results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3329                 }
3330         }
3331
3332         /* LP0 register values */
3333         for_each_intel_crtc(&dev_priv->drm, crtc) {
3334                 enum pipe pipe = crtc->pipe;
3335                 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3336                 const struct intel_wm_level *r = &pipe_wm->wm[0];
3337
3338                 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3339                         continue;
3340
3341                 results->wm_pipe[pipe] =
3342                         WM0_PIPE_PRIMARY(r->pri_val) |
3343                         WM0_PIPE_SPRITE(r->spr_val) |
3344                         WM0_PIPE_CURSOR(r->cur_val);
3345         }
3346 }
3347
3348 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3349  * case both are at the same level. Prefer r1 in case they're the same. */
3350 static struct intel_pipe_wm *
3351 ilk_find_best_result(struct drm_i915_private *dev_priv,
3352                      struct intel_pipe_wm *r1,
3353                      struct intel_pipe_wm *r2)
3354 {
3355         int level, max_level = ilk_wm_max_level(dev_priv);
3356         int level1 = 0, level2 = 0;
3357
3358         for (level = 1; level <= max_level; level++) {
3359                 if (r1->wm[level].enable)
3360                         level1 = level;
3361                 if (r2->wm[level].enable)
3362                         level2 = level;
3363         }
3364
3365         if (level1 == level2) {
3366                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3367                         return r2;
3368                 else
3369                         return r1;
3370         } else if (level1 > level2) {
3371                 return r1;
3372         } else {
3373                 return r2;
3374         }
3375 }
3376
3377 /* dirty bits used to track which watermarks need changes */
3378 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3379 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3380 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3381 #define WM_DIRTY_FBC (1 << 24)
3382 #define WM_DIRTY_DDB (1 << 25)
3383
3384 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3385                                          const struct ilk_wm_values *old,
3386                                          const struct ilk_wm_values *new)
3387 {
3388         unsigned int dirty = 0;
3389         enum pipe pipe;
3390         int wm_lp;
3391
3392         for_each_pipe(dev_priv, pipe) {
3393                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3394                         dirty |= WM_DIRTY_PIPE(pipe);
3395                         /* Must disable LP1+ watermarks too */
3396                         dirty |= WM_DIRTY_LP_ALL;
3397                 }
3398         }
3399
3400         if (old->enable_fbc_wm != new->enable_fbc_wm) {
3401                 dirty |= WM_DIRTY_FBC;
3402                 /* Must disable LP1+ watermarks too */
3403                 dirty |= WM_DIRTY_LP_ALL;
3404         }
3405
3406         if (old->partitioning != new->partitioning) {
3407                 dirty |= WM_DIRTY_DDB;
3408                 /* Must disable LP1+ watermarks too */
3409                 dirty |= WM_DIRTY_LP_ALL;
3410         }
3411
3412         /* LP1+ watermarks already deemed dirty, no need to continue */
3413         if (dirty & WM_DIRTY_LP_ALL)
3414                 return dirty;
3415
3416         /* Find the lowest numbered LP1+ watermark in need of an update... */
3417         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3418                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3419                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3420                         break;
3421         }
3422
3423         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3424         for (; wm_lp <= 3; wm_lp++)
3425                 dirty |= WM_DIRTY_LP(wm_lp);
3426
3427         return dirty;
3428 }
3429
3430 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3431                                unsigned int dirty)
3432 {
3433         struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3434         bool changed = false;
3435
3436         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3437                 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3438                 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3439                 changed = true;
3440         }
3441         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3442                 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3443                 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3444                 changed = true;
3445         }
3446         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3447                 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3448                 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3449                 changed = true;
3450         }
3451
3452         /*
3453          * Don't touch WM_LP_SPRITE_ENABLE here.
3454          * Doing so could cause underruns.
3455          */
3456
3457         return changed;
3458 }
3459
3460 /*
3461  * The spec says we shouldn't write when we don't need, because every write
3462  * causes WMs to be re-evaluated, expending some power.
3463  */
3464 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3465                                 struct ilk_wm_values *results)
3466 {
3467         struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3468         unsigned int dirty;
3469
3470         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3471         if (!dirty)
3472                 return;
3473
3474         _ilk_disable_lp_wm(dev_priv, dirty);
3475
3476         if (dirty & WM_DIRTY_PIPE(PIPE_A))
3477                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3478         if (dirty & WM_DIRTY_PIPE(PIPE_B))
3479                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3480         if (dirty & WM_DIRTY_PIPE(PIPE_C))
3481                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3482
3483         if (dirty & WM_DIRTY_DDB) {
3484                 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3485                         intel_uncore_rmw(&dev_priv->uncore, WM_MISC, WM_MISC_DATA_PARTITION_5_6,
3486                                          results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3487                                          WM_MISC_DATA_PARTITION_5_6);
3488                 else
3489                         intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,
3490                                          results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3491                                          DISP_DATA_PARTITION_5_6);
3492         }
3493
3494         if (dirty & WM_DIRTY_FBC)
3495                 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, DISP_FBC_WM_DIS,
3496                                  results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);
3497
3498         if (dirty & WM_DIRTY_LP(1) &&
3499             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3500                 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3501
3502         if (DISPLAY_VER(dev_priv) >= 7) {
3503                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3504                         intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3505                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3506                         intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3507         }
3508
3509         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3510                 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3511         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3512                 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3513         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3514                 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3515
3516         dev_priv->display.wm.hw = *results;
3517 }
3518
3519 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3520 {
3521         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3522 }
3523
3524 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
3525                                   struct intel_wm_config *config)
3526 {
3527         struct intel_crtc *crtc;
3528
3529         /* Compute the currently _active_ config */
3530         for_each_intel_crtc(&dev_priv->drm, crtc) {
3531                 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3532
3533                 if (!wm->pipe_enabled)
3534                         continue;
3535
3536                 config->sprites_enabled |= wm->sprites_enabled;
3537                 config->sprites_scaled |= wm->sprites_scaled;
3538                 config->num_pipes_active++;
3539         }
3540 }
3541
3542 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
3543 {
3544         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3545         struct ilk_wm_maximums max;
3546         struct intel_wm_config config = {};
3547         struct ilk_wm_values results = {};
3548         enum intel_ddb_partitioning partitioning;
3549
3550         ilk_compute_wm_config(dev_priv, &config);
3551
3552         ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
3553         ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
3554
3555         /* 5/6 split only in single pipe config on IVB+ */
3556         if (DISPLAY_VER(dev_priv) >= 7 &&
3557             config.num_pipes_active == 1 && config.sprites_enabled) {
3558                 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
3559                 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
3560
3561                 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
3562         } else {
3563                 best_lp_wm = &lp_wm_1_2;
3564         }
3565
3566         partitioning = (best_lp_wm == &lp_wm_1_2) ?
3567                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3568
3569         ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
3570
3571         ilk_write_wm_values(dev_priv, &results);
3572 }
3573
3574 static void ilk_initial_watermarks(struct intel_atomic_state *state,
3575                                    struct intel_crtc *crtc)
3576 {
3577         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3578         const struct intel_crtc_state *crtc_state =
3579                 intel_atomic_get_new_crtc_state(state, crtc);
3580
3581         mutex_lock(&dev_priv->display.wm.wm_mutex);
3582         crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
3583         ilk_program_watermarks(dev_priv);
3584         mutex_unlock(&dev_priv->display.wm.wm_mutex);
3585 }
3586
3587 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
3588                                     struct intel_crtc *crtc)
3589 {
3590         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3591         const struct intel_crtc_state *crtc_state =
3592                 intel_atomic_get_new_crtc_state(state, crtc);
3593
3594         if (!crtc_state->wm.need_postvbl_update)
3595                 return;
3596
3597         mutex_lock(&dev_priv->display.wm.wm_mutex);
3598         crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
3599         ilk_program_watermarks(dev_priv);
3600         mutex_unlock(&dev_priv->display.wm.wm_mutex);
3601 }
3602
3603 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
3604 {
3605         struct drm_device *dev = crtc->base.dev;
3606         struct drm_i915_private *dev_priv = to_i915(dev);
3607         struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3608         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
3609         struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
3610         enum pipe pipe = crtc->pipe;
3611
3612         hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
3613
3614         memset(active, 0, sizeof(*active));
3615
3616         active->pipe_enabled = crtc->active;
3617
3618         if (active->pipe_enabled) {
3619                 u32 tmp = hw->wm_pipe[pipe];
3620
3621                 /*
3622                  * For active pipes LP0 watermark is marked as
3623                  * enabled, and LP1+ watermaks as disabled since
3624                  * we can't really reverse compute them in case
3625                  * multiple pipes are active.
3626                  */
3627                 active->wm[0].enable = true;
3628                 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
3629                 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
3630                 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
3631         } else {
3632                 int level, max_level = ilk_wm_max_level(dev_priv);
3633
3634                 /*
3635                  * For inactive pipes, all watermark levels
3636                  * should be marked as enabled but zeroed,
3637                  * which is what we'd compute them to.
3638                  */
3639                 for (level = 0; level <= max_level; level++)
3640                         active->wm[level].enable = true;
3641         }
3642
3643         crtc->wm.active.ilk = *active;
3644 }
3645
3646 #define _FW_WM(value, plane) \
3647         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3648 #define _FW_WM_VLV(value, plane) \
3649         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3650
3651 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
3652                                struct g4x_wm_values *wm)
3653 {
3654         u32 tmp;
3655
3656         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3657         wm->sr.plane = _FW_WM(tmp, SR);
3658         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3659         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
3660         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
3661
3662         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3663         wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
3664         wm->sr.fbc = _FW_WM(tmp, FBC_SR);
3665         wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
3666         wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
3667         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3668         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
3669
3670         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3671         wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
3672         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3673         wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
3674         wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
3675 }
3676
3677 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
3678                                struct vlv_wm_values *wm)
3679 {
3680         enum pipe pipe;
3681         u32 tmp;
3682
3683         for_each_pipe(dev_priv, pipe) {
3684                 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
3685
3686                 wm->ddl[pipe].plane[PLANE_PRIMARY] =
3687                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3688                 wm->ddl[pipe].plane[PLANE_CURSOR] =
3689                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3690                 wm->ddl[pipe].plane[PLANE_SPRITE0] =
3691                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3692                 wm->ddl[pipe].plane[PLANE_SPRITE1] =
3693                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3694         }
3695
3696         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3697         wm->sr.plane = _FW_WM(tmp, SR);
3698         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3699         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
3700         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
3701
3702         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3703         wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
3704         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3705         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
3706
3707         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3708         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3709
3710         if (IS_CHERRYVIEW(dev_priv)) {
3711                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
3712                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3713                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3714
3715                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
3716                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
3717                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
3718
3719                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
3720                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
3721                 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
3722
3723                 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3724                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3725                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3726                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3727                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
3728                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3729                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3730                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3731                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3732                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3733                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3734         } else {
3735                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
3736                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3737                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3738
3739                 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3740                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3741                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3742                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3743                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3744                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3745                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3746                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3747         }
3748 }
3749
3750 #undef _FW_WM
3751 #undef _FW_WM_VLV
3752
3753 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
3754 {
3755         struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;
3756         struct intel_crtc *crtc;
3757
3758         g4x_read_wm_values(dev_priv, wm);
3759
3760         wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
3761
3762         for_each_intel_crtc(&dev_priv->drm, crtc) {
3763                 struct intel_crtc_state *crtc_state =
3764                         to_intel_crtc_state(crtc->base.state);
3765                 struct g4x_wm_state *active = &crtc->wm.active.g4x;
3766                 struct g4x_pipe_wm *raw;
3767                 enum pipe pipe = crtc->pipe;
3768                 enum plane_id plane_id;
3769                 int level, max_level;
3770
3771                 active->cxsr = wm->cxsr;
3772                 active->hpll_en = wm->hpll_en;
3773                 active->fbc_en = wm->fbc_en;
3774
3775                 active->sr = wm->sr;
3776                 active->hpll = wm->hpll;
3777
3778                 for_each_plane_id_on_crtc(crtc, plane_id) {
3779                         active->wm.plane[plane_id] =
3780                                 wm->pipe[pipe].plane[plane_id];
3781                 }
3782
3783                 if (wm->cxsr && wm->hpll_en)
3784                         max_level = G4X_WM_LEVEL_HPLL;
3785                 else if (wm->cxsr)
3786                         max_level = G4X_WM_LEVEL_SR;
3787                 else
3788                         max_level = G4X_WM_LEVEL_NORMAL;
3789
3790                 level = G4X_WM_LEVEL_NORMAL;
3791                 raw = &crtc_state->wm.g4x.raw[level];
3792                 for_each_plane_id_on_crtc(crtc, plane_id)
3793                         raw->plane[plane_id] = active->wm.plane[plane_id];
3794
3795                 level = G4X_WM_LEVEL_SR;
3796                 if (level > max_level)
3797                         goto out;
3798
3799                 raw = &crtc_state->wm.g4x.raw[level];
3800                 raw->plane[PLANE_PRIMARY] = active->sr.plane;
3801                 raw->plane[PLANE_CURSOR] = active->sr.cursor;
3802                 raw->plane[PLANE_SPRITE0] = 0;
3803                 raw->fbc = active->sr.fbc;
3804
3805                 level = G4X_WM_LEVEL_HPLL;
3806                 if (level > max_level)
3807                         goto out;
3808
3809                 raw = &crtc_state->wm.g4x.raw[level];
3810                 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
3811                 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
3812                 raw->plane[PLANE_SPRITE0] = 0;
3813                 raw->fbc = active->hpll.fbc;
3814
3815                 level++;
3816         out:
3817                 for_each_plane_id_on_crtc(crtc, plane_id)
3818                         g4x_raw_plane_wm_set(crtc_state, level,
3819                                              plane_id, USHRT_MAX);
3820                 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
3821
3822                 g4x_invalidate_wms(crtc, active, level);
3823
3824                 crtc_state->wm.g4x.optimal = *active;
3825                 crtc_state->wm.g4x.intermediate = *active;
3826
3827                 drm_dbg_kms(&dev_priv->drm,
3828                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
3829                             pipe_name(pipe),
3830                             wm->pipe[pipe].plane[PLANE_PRIMARY],
3831                             wm->pipe[pipe].plane[PLANE_CURSOR],
3832                             wm->pipe[pipe].plane[PLANE_SPRITE0]);
3833         }
3834
3835         drm_dbg_kms(&dev_priv->drm,
3836                     "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
3837                     wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
3838         drm_dbg_kms(&dev_priv->drm,
3839                     "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
3840                     wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
3841         drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
3842                     str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
3843                     str_yes_no(wm->fbc_en));
3844 }
3845
3846 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
3847 {
3848         struct intel_plane *plane;
3849         struct intel_crtc *crtc;
3850
3851         mutex_lock(&dev_priv->display.wm.wm_mutex);
3852
3853         for_each_intel_plane(&dev_priv->drm, plane) {
3854                 struct intel_crtc *crtc =
3855                         intel_crtc_for_pipe(dev_priv, plane->pipe);
3856                 struct intel_crtc_state *crtc_state =
3857                         to_intel_crtc_state(crtc->base.state);
3858                 struct intel_plane_state *plane_state =
3859                         to_intel_plane_state(plane->base.state);
3860                 enum plane_id plane_id = plane->id;
3861                 int level, num_levels = intel_wm_num_levels(dev_priv);
3862
3863                 if (plane_state->uapi.visible)
3864                         continue;
3865
3866                 for (level = 0; level < num_levels; level++) {
3867                         struct g4x_pipe_wm *raw =
3868                                 &crtc_state->wm.g4x.raw[level];
3869
3870                         raw->plane[plane_id] = 0;
3871
3872                         if (plane_id == PLANE_PRIMARY)
3873                                 raw->fbc = 0;
3874                 }
3875         }
3876
3877         for_each_intel_crtc(&dev_priv->drm, crtc) {
3878                 struct intel_crtc_state *crtc_state =
3879                         to_intel_crtc_state(crtc->base.state);
3880                 int ret;
3881
3882                 ret = _g4x_compute_pipe_wm(crtc_state);
3883                 drm_WARN_ON(&dev_priv->drm, ret);
3884
3885                 crtc_state->wm.g4x.intermediate =
3886                         crtc_state->wm.g4x.optimal;
3887                 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
3888         }
3889
3890         g4x_program_watermarks(dev_priv);
3891
3892         mutex_unlock(&dev_priv->display.wm.wm_mutex);
3893 }
3894
3895 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
3896 {
3897         struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;
3898         struct intel_crtc *crtc;
3899         u32 val;
3900
3901         vlv_read_wm_values(dev_priv, wm);
3902
3903         wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3904         wm->level = VLV_WM_LEVEL_PM2;
3905
3906         if (IS_CHERRYVIEW(dev_priv)) {
3907                 vlv_punit_get(dev_priv);
3908
3909                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
3910                 if (val & DSP_MAXFIFO_PM5_ENABLE)
3911                         wm->level = VLV_WM_LEVEL_PM5;
3912
3913                 /*
3914                  * If DDR DVFS is disabled in the BIOS, Punit
3915                  * will never ack the request. So if that happens
3916                  * assume we don't have to enable/disable DDR DVFS
3917                  * dynamically. To test that just set the REQ_ACK
3918                  * bit to poke the Punit, but don't change the
3919                  * HIGH/LOW bits so that we don't actually change
3920                  * the current state.
3921                  */
3922                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3923                 val |= FORCE_DDR_FREQ_REQ_ACK;
3924                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
3925
3926                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
3927                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
3928                         drm_dbg_kms(&dev_priv->drm,
3929                                     "Punit not acking DDR DVFS request, "
3930                                     "assuming DDR DVFS is disabled\n");
3931                         dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM5;
3932                 } else {
3933                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3934                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
3935                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
3936                 }
3937
3938                 vlv_punit_put(dev_priv);
3939         }
3940
3941         for_each_intel_crtc(&dev_priv->drm, crtc) {
3942                 struct intel_crtc_state *crtc_state =
3943                         to_intel_crtc_state(crtc->base.state);
3944                 struct vlv_wm_state *active = &crtc->wm.active.vlv;
3945                 const struct vlv_fifo_state *fifo_state =
3946                         &crtc_state->wm.vlv.fifo_state;
3947                 enum pipe pipe = crtc->pipe;
3948                 enum plane_id plane_id;
3949                 int level;
3950
3951                 vlv_get_fifo_size(crtc_state);
3952
3953                 active->num_levels = wm->level + 1;
3954                 active->cxsr = wm->cxsr;
3955
3956                 for (level = 0; level < active->num_levels; level++) {
3957                         struct g4x_pipe_wm *raw =
3958                                 &crtc_state->wm.vlv.raw[level];
3959
3960                         active->sr[level].plane = wm->sr.plane;
3961                         active->sr[level].cursor = wm->sr.cursor;
3962
3963                         for_each_plane_id_on_crtc(crtc, plane_id) {
3964                                 active->wm[level].plane[plane_id] =
3965                                         wm->pipe[pipe].plane[plane_id];
3966
3967                                 raw->plane[plane_id] =
3968                                         vlv_invert_wm_value(active->wm[level].plane[plane_id],
3969                                                             fifo_state->plane[plane_id]);
3970                         }
3971                 }
3972
3973                 for_each_plane_id_on_crtc(crtc, plane_id)
3974                         vlv_raw_plane_wm_set(crtc_state, level,
3975                                              plane_id, USHRT_MAX);
3976                 vlv_invalidate_wms(crtc, active, level);
3977
3978                 crtc_state->wm.vlv.optimal = *active;
3979                 crtc_state->wm.vlv.intermediate = *active;
3980
3981                 drm_dbg_kms(&dev_priv->drm,
3982                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
3983                             pipe_name(pipe),
3984                             wm->pipe[pipe].plane[PLANE_PRIMARY],
3985                             wm->pipe[pipe].plane[PLANE_CURSOR],
3986                             wm->pipe[pipe].plane[PLANE_SPRITE0],
3987                             wm->pipe[pipe].plane[PLANE_SPRITE1]);
3988         }
3989
3990         drm_dbg_kms(&dev_priv->drm,
3991                     "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
3992                     wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
3993 }
3994
3995 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
3996 {
3997         struct intel_plane *plane;
3998         struct intel_crtc *crtc;
3999
4000         mutex_lock(&dev_priv->display.wm.wm_mutex);
4001
4002         for_each_intel_plane(&dev_priv->drm, plane) {
4003                 struct intel_crtc *crtc =
4004                         intel_crtc_for_pipe(dev_priv, plane->pipe);
4005                 struct intel_crtc_state *crtc_state =
4006                         to_intel_crtc_state(crtc->base.state);
4007                 struct intel_plane_state *plane_state =
4008                         to_intel_plane_state(plane->base.state);
4009                 enum plane_id plane_id = plane->id;
4010                 int level, num_levels = intel_wm_num_levels(dev_priv);
4011
4012                 if (plane_state->uapi.visible)
4013                         continue;
4014
4015                 for (level = 0; level < num_levels; level++) {
4016                         struct g4x_pipe_wm *raw =
4017                                 &crtc_state->wm.vlv.raw[level];
4018
4019                         raw->plane[plane_id] = 0;
4020                 }
4021         }
4022
4023         for_each_intel_crtc(&dev_priv->drm, crtc) {
4024                 struct intel_crtc_state *crtc_state =
4025                         to_intel_crtc_state(crtc->base.state);
4026                 int ret;
4027
4028                 ret = _vlv_compute_pipe_wm(crtc_state);
4029                 drm_WARN_ON(&dev_priv->drm, ret);
4030
4031                 crtc_state->wm.vlv.intermediate =
4032                         crtc_state->wm.vlv.optimal;
4033                 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
4034         }
4035
4036         vlv_program_watermarks(dev_priv);
4037
4038         mutex_unlock(&dev_priv->display.wm.wm_mutex);
4039 }
4040
4041 /*
4042  * FIXME should probably kill this and improve
4043  * the real watermark readout/sanitation instead
4044  */
4045 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
4046 {
4047         intel_uncore_rmw(&dev_priv->uncore, WM3_LP_ILK, WM_LP_ENABLE, 0);
4048         intel_uncore_rmw(&dev_priv->uncore, WM2_LP_ILK, WM_LP_ENABLE, 0);
4049         intel_uncore_rmw(&dev_priv->uncore, WM1_LP_ILK, WM_LP_ENABLE, 0);
4050
4051         /*
4052          * Don't touch WM_LP_SPRITE_ENABLE here.
4053          * Doing so could cause underruns.
4054          */
4055 }
4056
4057 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
4058 {
4059         struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
4060         struct intel_crtc *crtc;
4061
4062         ilk_init_lp_watermarks(dev_priv);
4063
4064         for_each_intel_crtc(&dev_priv->drm, crtc)
4065                 ilk_pipe_wm_get_hw_state(crtc);
4066
4067         hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
4068         hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
4069         hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
4070
4071         hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
4072         if (DISPLAY_VER(dev_priv) >= 7) {
4073                 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
4074                 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
4075         }
4076
4077         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
4078                 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
4079                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4080         else if (IS_IVYBRIDGE(dev_priv))
4081                 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
4082                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4083
4084         hw->enable_fbc_wm =
4085                 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
4086 }
4087
4088 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
4089 {
4090         /*
4091          * On Ibex Peak and Cougar Point, we need to disable clock
4092          * gating for the panel power sequencer or it will fail to
4093          * start up when no ports are active.
4094          */
4095         intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
4096 }
4097
4098 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
4099 {
4100         enum pipe pipe;
4101
4102         for_each_pipe(dev_priv, pipe) {
4103                 intel_uncore_rmw(&dev_priv->uncore, DSPCNTR(pipe), 0, DISP_TRICKLE_FEED_DISABLE);
4104
4105                 intel_uncore_rmw(&dev_priv->uncore, DSPSURF(pipe), 0, 0);
4106                 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
4107         }
4108 }
4109
4110 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
4111 {
4112         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4113
4114         /*
4115          * Required for FBC
4116          * WaFbcDisableDpfcClockGating:ilk
4117          */
4118         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
4119                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
4120                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
4121
4122         intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
4123                    MARIUNIT_CLOCK_GATE_DISABLE |
4124                    SVSMUNIT_CLOCK_GATE_DISABLE);
4125         intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
4126                    VFMUNIT_CLOCK_GATE_DISABLE);
4127
4128         /*
4129          * According to the spec the following bits should be set in
4130          * order to enable memory self-refresh
4131          * The bit 22/21 of 0x42004
4132          * The bit 5 of 0x42020
4133          * The bit 15 of 0x45000
4134          */
4135         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4136                    (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4137                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
4138         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
4139         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
4140                    (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
4141                     DISP_FBC_WM_DIS));
4142
4143         /*
4144          * Based on the document from hardware guys the following bits
4145          * should be set unconditionally in order to enable FBC.
4146          * The bit 22 of 0x42000
4147          * The bit 22 of 0x42004
4148          * The bit 7,8,9 of 0x42020.
4149          */
4150         if (IS_IRONLAKE_M(dev_priv)) {
4151                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
4152                 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
4153                 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
4154         }
4155
4156         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
4157
4158         intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
4159
4160         g4x_disable_trickle_feed(dev_priv);
4161
4162         ibx_init_clock_gating(dev_priv);
4163 }
4164
4165 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
4166 {
4167         enum pipe pipe;
4168         u32 val;
4169
4170         /*
4171          * On Ibex Peak and Cougar Point, we need to disable clock
4172          * gating for the panel power sequencer or it will fail to
4173          * start up when no ports are active.
4174          */
4175         intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
4176                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
4177                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
4178         intel_uncore_rmw(&dev_priv->uncore, SOUTH_CHICKEN2, 0, DPLS_EDP_PPS_FIX_DIS);
4179         /* The below fixes the weird display corruption, a few pixels shifted
4180          * downward, on (only) LVDS of some HP laptops with IVY.
4181          */
4182         for_each_pipe(dev_priv, pipe) {
4183                 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
4184                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
4185                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4186                 if (dev_priv->display.vbt.fdi_rx_polarity_inverted)
4187                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4188                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
4189                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
4190                 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
4191         }
4192         /* WADP0ClockGatingDisable */
4193         for_each_pipe(dev_priv, pipe) {
4194                 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
4195                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
4196         }
4197 }
4198
4199 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
4200 {
4201         u32 tmp;
4202
4203         tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
4204         if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
4205                 drm_dbg_kms(&dev_priv->drm,
4206                             "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
4207                             tmp);
4208 }
4209
4210 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
4211 {
4212         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4213
4214         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
4215
4216         intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
4217
4218         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
4219                    intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
4220                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
4221                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
4222
4223         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
4224          * gating disable must be set.  Failure to set it results in
4225          * flickering pixels due to Z write ordering failures after
4226          * some amount of runtime in the Mesa "fire" demo, and Unigine
4227          * Sanctuary and Tropics, and apparently anything else with
4228          * alpha test or pixel discard.
4229          *
4230          * According to the spec, bit 11 (RCCUNIT) must also be set,
4231          * but we didn't debug actual testcases to find it out.
4232          *
4233          * WaDisableRCCUnitClockGating:snb
4234          * WaDisableRCPBUnitClockGating:snb
4235          */
4236         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4237                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
4238                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
4239
4240         /*
4241          * According to the spec the following bits should be
4242          * set in order to enable memory self-refresh and fbc:
4243          * The bit21 and bit22 of 0x42000
4244          * The bit21 and bit22 of 0x42004
4245          * The bit5 and bit7 of 0x42020
4246          * The bit14 of 0x70180
4247          * The bit14 of 0x71180
4248          *
4249          * WaFbcAsynchFlipDisableFbcQueue:snb
4250          */
4251         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
4252                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
4253                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
4254         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4255                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4256                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
4257         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
4258                    intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
4259                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
4260                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
4261
4262         g4x_disable_trickle_feed(dev_priv);
4263
4264         cpt_init_clock_gating(dev_priv);
4265
4266         gen6_check_mch_setup(dev_priv);
4267 }
4268
4269 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
4270 {
4271         /*
4272          * TODO: this bit should only be enabled when really needed, then
4273          * disabled when not needed anymore in order to save power.
4274          */
4275         if (HAS_PCH_LPT_LP(dev_priv))
4276                 intel_uncore_rmw(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
4277                                  0, PCH_LP_PARTITION_LEVEL_DISABLE);
4278
4279         /* WADPOClockGatingDisable:hsw */
4280         intel_uncore_rmw(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
4281                          0, TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
4282 }
4283
4284 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
4285 {
4286         if (HAS_PCH_LPT_LP(dev_priv)) {
4287                 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
4288
4289                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
4290                 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
4291         }
4292 }
4293
4294 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
4295                                    int general_prio_credits,
4296                                    int high_prio_credits)
4297 {
4298         u32 misccpctl;
4299         u32 val;
4300
4301         /* WaTempDisableDOPClkGating:bdw */
4302         misccpctl = intel_gt_mcr_multicast_rmw(to_gt(dev_priv), GEN8_MISCCPCTL,
4303                                                GEN8_DOP_CLOCK_GATE_ENABLE, 0);
4304
4305         val = intel_gt_mcr_read_any(to_gt(dev_priv), GEN8_L3SQCREG1);
4306         val &= ~L3_PRIO_CREDITS_MASK;
4307         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
4308         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
4309         intel_gt_mcr_multicast_write(to_gt(dev_priv), GEN8_L3SQCREG1, val);
4310
4311         /*
4312          * Wait at least 100 clocks before re-enabling clock gating.
4313          * See the definition of L3SQCREG1 in BSpec.
4314          */
4315         intel_gt_mcr_read_any(to_gt(dev_priv), GEN8_L3SQCREG1);
4316         udelay(1);
4317         intel_gt_mcr_multicast_write(to_gt(dev_priv), GEN8_MISCCPCTL, misccpctl);
4318 }
4319
4320 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
4321 {
4322         /* Wa_1409120013:icl,ehl */
4323         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4324                            DPFC_CHICKEN_COMP_DUMMY_PIXEL);
4325
4326         /*Wa_14010594013:icl, ehl */
4327         intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
4328                          0, ICL_DELAY_PMRSP);
4329 }
4330
4331 static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
4332 {
4333         /* Wa_1409120013 */
4334         if (DISPLAY_VER(dev_priv) == 12)
4335                 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4336                                    DPFC_CHICKEN_COMP_DUMMY_PIXEL);
4337
4338         /* Wa_1409825376:tgl (pre-prod)*/
4339         if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
4340                 intel_uncore_rmw(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, 0, TGL_VRH_GATING_DIS);
4341
4342         /* Wa_14013723622:tgl,rkl,dg1,adl-s */
4343         if (DISPLAY_VER(dev_priv) == 12)
4344                 intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
4345                                  CLKREQ_POLICY_MEM_UP_OVRD, 0);
4346 }
4347
4348 static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
4349 {
4350         gen12lp_init_clock_gating(dev_priv);
4351
4352         /* Wa_22011091694:adlp */
4353         intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
4354
4355         /* Bspec/49189 Initialize Sequence */
4356         intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1, DDI_CLOCK_REG_ACCESS, 0);
4357 }
4358
4359 static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
4360 {
4361         gen12lp_init_clock_gating(dev_priv);
4362
4363         /* Wa_1409836686:dg1[a0] */
4364         if (IS_DG1_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
4365                 intel_uncore_rmw(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, 0, DPT_GATING_DIS);
4366 }
4367
4368 static void xehpsdv_init_clock_gating(struct drm_i915_private *dev_priv)
4369 {
4370         /* Wa_22010146351:xehpsdv */
4371         if (IS_XEHPSDV_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
4372                 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
4373 }
4374
4375 static void dg2_init_clock_gating(struct drm_i915_private *i915)
4376 {
4377         /* Wa_22010954014:dg2 */
4378         intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
4379                          SGSI_SIDECLK_DIS);
4380
4381         /*
4382          * Wa_14010733611:dg2_g10
4383          * Wa_22010146351:dg2_g10
4384          */
4385         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
4386                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
4387                                  SGR_DIS | SGGI_DIS);
4388 }
4389
4390 static void pvc_init_clock_gating(struct drm_i915_private *dev_priv)
4391 {
4392         /* Wa_14012385139:pvc */
4393         if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
4394                 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
4395
4396         /* Wa_22010954014:pvc */
4397         if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
4398                 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);
4399 }
4400
4401 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
4402 {
4403         if (!HAS_PCH_CNP(dev_priv))
4404                 return;
4405
4406         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
4407         intel_uncore_rmw(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, 0, CNP_PWM_CGE_GATING_DISABLE);
4408 }
4409
4410 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
4411 {
4412         cnp_init_clock_gating(dev_priv);
4413         gen9_init_clock_gating(dev_priv);
4414
4415         /* WAC6entrylatency:cfl */
4416         intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4417
4418         /*
4419          * WaFbcTurnOffFbcWatermark:cfl
4420          * Display WA #0562: cfl
4421          */
4422         intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4423
4424         /*
4425          * WaFbcNukeOnHostModify:cfl
4426          * Display WA #0873: cfl
4427          */
4428         intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4429                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
4430 }
4431
4432 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
4433 {
4434         gen9_init_clock_gating(dev_priv);
4435
4436         /* WAC6entrylatency:kbl */
4437         intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4438
4439         /* WaDisableSDEUnitClockGating:kbl */
4440         if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
4441                 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6,
4442                                  0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4443
4444         /* WaDisableGamClockGating:kbl */
4445         if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
4446                 intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1,
4447                                  0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
4448
4449         /*
4450          * WaFbcTurnOffFbcWatermark:kbl
4451          * Display WA #0562: kbl
4452          */
4453         intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4454
4455         /*
4456          * WaFbcNukeOnHostModify:kbl
4457          * Display WA #0873: kbl
4458          */
4459         intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4460                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
4461 }
4462
4463 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
4464 {
4465         gen9_init_clock_gating(dev_priv);
4466
4467         /* WaDisableDopClockGating:skl */
4468         intel_gt_mcr_multicast_rmw(to_gt(dev_priv), GEN8_MISCCPCTL,
4469                                    GEN8_DOP_CLOCK_GATE_ENABLE, 0);
4470
4471         /* WAC6entrylatency:skl */
4472         intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4473
4474         /*
4475          * WaFbcTurnOffFbcWatermark:skl
4476          * Display WA #0562: skl
4477          */
4478         intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4479
4480         /*
4481          * WaFbcNukeOnHostModify:skl
4482          * Display WA #0873: skl
4483          */
4484         intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4485                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
4486
4487         /*
4488          * WaFbcHighMemBwCorruptionAvoidance:skl
4489          * Display WA #0883: skl
4490          */
4491         intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
4492 }
4493
4494 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
4495 {
4496         enum pipe pipe;
4497
4498         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
4499         intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
4500
4501         /* WaSwitchSolVfFArbitrationPriority:bdw */
4502         intel_uncore_rmw(&dev_priv->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
4503
4504         /* WaPsrDPAMaskVBlankInSRD:bdw */
4505         intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, DPA_MASK_VBLANK_SRD);
4506
4507         for_each_pipe(dev_priv, pipe) {
4508                 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
4509                 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
4510                                  0, BDW_DPRS_MASK_VBLANK_SRD);
4511         }
4512
4513         /* WaVSRefCountFullforceMissDisable:bdw */
4514         /* WaDSRefCountFullforceMissDisable:bdw */
4515         intel_uncore_rmw(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
4516                          GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
4517
4518         intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
4519                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4520
4521         /* WaDisableSDEUnitClockGating:bdw */
4522         intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4523
4524         /* WaProgramL3SqcReg1Default:bdw */
4525         gen8_set_l3sqc_credits(dev_priv, 30, 2);
4526
4527         /* WaKVMNotificationOnConfigChange:bdw */
4528         intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR2_1,
4529                          0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
4530
4531         lpt_init_clock_gating(dev_priv);
4532
4533         /* WaDisableDopClockGating:bdw
4534          *
4535          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
4536          * clock gating.
4537          */
4538         intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1, 0, GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
4539 }
4540
4541 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
4542 {
4543         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
4544         intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
4545
4546         /* This is required by WaCatErrorRejectionIssue:hsw */
4547         intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4548                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4549
4550         /* WaSwitchSolVfFArbitrationPriority:hsw */
4551         intel_uncore_rmw(&dev_priv->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
4552
4553         lpt_init_clock_gating(dev_priv);
4554 }
4555
4556 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
4557 {
4558         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
4559
4560         /* WaFbcAsynchFlipDisableFbcQueue:ivb */
4561         intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
4562
4563         /* WaDisableBackToBackFlipFix:ivb */
4564         intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
4565                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
4566                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
4567
4568         if (IS_IVB_GT1(dev_priv))
4569                 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4570                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4571         else {
4572                 /* must write both registers */
4573                 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4574                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4575                 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
4576                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4577         }
4578
4579         /*
4580          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4581          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
4582          */
4583         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4584                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
4585
4586         /* This is required by WaCatErrorRejectionIssue:ivb */
4587         intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4588                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4589
4590         g4x_disable_trickle_feed(dev_priv);
4591
4592         intel_uncore_rmw(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
4593                          GEN6_MBC_SNPCR_MED);
4594
4595         if (!HAS_PCH_NOP(dev_priv))
4596                 cpt_init_clock_gating(dev_priv);
4597
4598         gen6_check_mch_setup(dev_priv);
4599 }
4600
4601 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
4602 {
4603         /* WaDisableBackToBackFlipFix:vlv */
4604         intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
4605                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
4606                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
4607
4608         /* WaDisableDopClockGating:vlv */
4609         intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4610                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4611
4612         /* This is required by WaCatErrorRejectionIssue:vlv */
4613         intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4614                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4615
4616         /*
4617          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4618          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
4619          */
4620         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4621                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
4622
4623         /* WaDisableL3Bank2xClockGate:vlv
4624          * Disabling L3 clock gating- MMIO 940c[25] = 1
4625          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
4626         intel_uncore_rmw(&dev_priv->uncore, GEN7_UCGCTL4, 0, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
4627
4628         /*
4629          * WaDisableVLVClockGating_VBIIssue:vlv
4630          * Disable clock gating on th GCFG unit to prevent a delay
4631          * in the reporting of vblank events.
4632          */
4633         intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
4634 }
4635
4636 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
4637 {
4638         /* WaVSRefCountFullforceMissDisable:chv */
4639         /* WaDSRefCountFullforceMissDisable:chv */
4640         intel_uncore_rmw(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
4641                          GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
4642
4643         /* WaDisableSemaphoreAndSyncFlipWait:chv */
4644         intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
4645                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4646
4647         /* WaDisableCSUnitClockGating:chv */
4648         intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
4649
4650         /* WaDisableSDEUnitClockGating:chv */
4651         intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4652
4653         /*
4654          * WaProgramL3SqcReg1Default:chv
4655          * See gfxspecs/Related Documents/Performance Guide/
4656          * LSQC Setting Recommendations.
4657          */
4658         gen8_set_l3sqc_credits(dev_priv, 38, 2);
4659 }
4660
4661 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
4662 {
4663         u32 dspclk_gate;
4664
4665         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
4666         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
4667                    GS_UNIT_CLOCK_GATE_DISABLE |
4668                    CL_UNIT_CLOCK_GATE_DISABLE);
4669         intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
4670         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
4671                 OVRUNIT_CLOCK_GATE_DISABLE |
4672                 OVCUNIT_CLOCK_GATE_DISABLE;
4673         if (IS_GM45(dev_priv))
4674                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
4675         intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D(dev_priv), dspclk_gate);
4676
4677         g4x_disable_trickle_feed(dev_priv);
4678 }
4679
4680 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
4681 {
4682         struct intel_uncore *uncore = &dev_priv->uncore;
4683
4684         intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
4685         intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
4686         intel_uncore_write(uncore, DSPCLK_GATE_D(dev_priv), 0);
4687         intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
4688         intel_uncore_write16(uncore, DEUC, 0);
4689         intel_uncore_write(uncore,
4690                            MI_ARB_STATE,
4691                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4692 }
4693
4694 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
4695 {
4696         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
4697                    I965_RCC_CLOCK_GATE_DISABLE |
4698                    I965_RCPB_CLOCK_GATE_DISABLE |
4699                    I965_ISC_CLOCK_GATE_DISABLE |
4700                    I965_FBC_CLOCK_GATE_DISABLE);
4701         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
4702         intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
4703                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4704 }
4705
4706 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
4707 {
4708         u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
4709
4710         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
4711                 DSTATE_DOT_CLOCK_GATING;
4712         intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
4713
4714         if (IS_PINEVIEW(dev_priv))
4715                 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
4716                                    _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
4717
4718         /* IIR "flip pending" means done if this bit is set */
4719         intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
4720                            _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
4721
4722         /* interrupts should cause a wake up from C3 */
4723         intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
4724
4725         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4726         intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
4727
4728         intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
4729                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4730 }
4731
4732 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
4733 {
4734         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
4735
4736         /* interrupts should cause a wake up from C3 */
4737         intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
4738                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
4739
4740         intel_uncore_write(&dev_priv->uncore, MEM_MODE,
4741                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
4742
4743         /*
4744          * Have FBC ignore 3D activity since we use software
4745          * render tracking, and otherwise a pure 3D workload
4746          * (even if it just renders a single frame and then does
4747          * abosultely nothing) would not allow FBC to recompress
4748          * until a 2D blit occurs.
4749          */
4750         intel_uncore_write(&dev_priv->uncore, SCPD0,
4751                    _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
4752 }
4753
4754 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
4755 {
4756         intel_uncore_write(&dev_priv->uncore, MEM_MODE,
4757                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
4758                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
4759 }
4760
4761 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
4762 {
4763         dev_priv->clock_gating_funcs->init_clock_gating(dev_priv);
4764 }
4765
4766 void intel_suspend_hw(struct drm_i915_private *dev_priv)
4767 {
4768         if (HAS_PCH_LPT(dev_priv))
4769                 lpt_suspend_hw(dev_priv);
4770 }
4771
4772 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
4773 {
4774         drm_dbg_kms(&dev_priv->drm,
4775                     "No clock gating settings or workarounds applied.\n");
4776 }
4777
4778 #define CG_FUNCS(platform)                                              \
4779 static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
4780         .init_clock_gating = platform##_init_clock_gating,              \
4781 }
4782
4783 CG_FUNCS(pvc);
4784 CG_FUNCS(dg2);
4785 CG_FUNCS(xehpsdv);
4786 CG_FUNCS(adlp);
4787 CG_FUNCS(dg1);
4788 CG_FUNCS(gen12lp);
4789 CG_FUNCS(icl);
4790 CG_FUNCS(cfl);
4791 CG_FUNCS(skl);
4792 CG_FUNCS(kbl);
4793 CG_FUNCS(bxt);
4794 CG_FUNCS(glk);
4795 CG_FUNCS(bdw);
4796 CG_FUNCS(chv);
4797 CG_FUNCS(hsw);
4798 CG_FUNCS(ivb);
4799 CG_FUNCS(vlv);
4800 CG_FUNCS(gen6);
4801 CG_FUNCS(ilk);
4802 CG_FUNCS(g4x);
4803 CG_FUNCS(i965gm);
4804 CG_FUNCS(i965g);
4805 CG_FUNCS(gen3);
4806 CG_FUNCS(i85x);
4807 CG_FUNCS(i830);
4808 CG_FUNCS(nop);
4809 #undef CG_FUNCS
4810
4811 /**
4812  * intel_init_clock_gating_hooks - setup the clock gating hooks
4813  * @dev_priv: device private
4814  *
4815  * Setup the hooks that configure which clocks of a given platform can be
4816  * gated and also apply various GT and display specific workarounds for these
4817  * platforms. Note that some GT specific workarounds are applied separately
4818  * when GPU contexts or batchbuffers start their execution.
4819  */
4820 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
4821 {
4822         if (IS_PONTEVECCHIO(dev_priv))
4823                 dev_priv->clock_gating_funcs = &pvc_clock_gating_funcs;
4824         else if (IS_DG2(dev_priv))
4825                 dev_priv->clock_gating_funcs = &dg2_clock_gating_funcs;
4826         else if (IS_XEHPSDV(dev_priv))
4827                 dev_priv->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
4828         else if (IS_ALDERLAKE_P(dev_priv))
4829                 dev_priv->clock_gating_funcs = &adlp_clock_gating_funcs;
4830         else if (IS_DG1(dev_priv))
4831                 dev_priv->clock_gating_funcs = &dg1_clock_gating_funcs;
4832         else if (GRAPHICS_VER(dev_priv) == 12)
4833                 dev_priv->clock_gating_funcs = &gen12lp_clock_gating_funcs;
4834         else if (GRAPHICS_VER(dev_priv) == 11)
4835                 dev_priv->clock_gating_funcs = &icl_clock_gating_funcs;
4836         else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
4837                 dev_priv->clock_gating_funcs = &cfl_clock_gating_funcs;
4838         else if (IS_SKYLAKE(dev_priv))
4839                 dev_priv->clock_gating_funcs = &skl_clock_gating_funcs;
4840         else if (IS_KABYLAKE(dev_priv))
4841                 dev_priv->clock_gating_funcs = &kbl_clock_gating_funcs;
4842         else if (IS_BROXTON(dev_priv))
4843                 dev_priv->clock_gating_funcs = &bxt_clock_gating_funcs;
4844         else if (IS_GEMINILAKE(dev_priv))
4845                 dev_priv->clock_gating_funcs = &glk_clock_gating_funcs;
4846         else if (IS_BROADWELL(dev_priv))
4847                 dev_priv->clock_gating_funcs = &bdw_clock_gating_funcs;
4848         else if (IS_CHERRYVIEW(dev_priv))
4849                 dev_priv->clock_gating_funcs = &chv_clock_gating_funcs;
4850         else if (IS_HASWELL(dev_priv))
4851                 dev_priv->clock_gating_funcs = &hsw_clock_gating_funcs;
4852         else if (IS_IVYBRIDGE(dev_priv))
4853                 dev_priv->clock_gating_funcs = &ivb_clock_gating_funcs;
4854         else if (IS_VALLEYVIEW(dev_priv))
4855                 dev_priv->clock_gating_funcs = &vlv_clock_gating_funcs;
4856         else if (GRAPHICS_VER(dev_priv) == 6)
4857                 dev_priv->clock_gating_funcs = &gen6_clock_gating_funcs;
4858         else if (GRAPHICS_VER(dev_priv) == 5)
4859                 dev_priv->clock_gating_funcs = &ilk_clock_gating_funcs;
4860         else if (IS_G4X(dev_priv))
4861                 dev_priv->clock_gating_funcs = &g4x_clock_gating_funcs;
4862         else if (IS_I965GM(dev_priv))
4863                 dev_priv->clock_gating_funcs = &i965gm_clock_gating_funcs;
4864         else if (IS_I965G(dev_priv))
4865                 dev_priv->clock_gating_funcs = &i965g_clock_gating_funcs;
4866         else if (GRAPHICS_VER(dev_priv) == 3)
4867                 dev_priv->clock_gating_funcs = &gen3_clock_gating_funcs;
4868         else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
4869                 dev_priv->clock_gating_funcs = &i85x_clock_gating_funcs;
4870         else if (GRAPHICS_VER(dev_priv) == 2)
4871                 dev_priv->clock_gating_funcs = &i830_clock_gating_funcs;
4872         else {
4873                 MISSING_CASE(INTEL_DEVID(dev_priv));
4874                 dev_priv->clock_gating_funcs = &nop_clock_gating_funcs;
4875         }
4876 }
4877
4878 static const struct intel_wm_funcs ilk_wm_funcs = {
4879         .compute_pipe_wm = ilk_compute_pipe_wm,
4880         .compute_intermediate_wm = ilk_compute_intermediate_wm,
4881         .initial_watermarks = ilk_initial_watermarks,
4882         .optimize_watermarks = ilk_optimize_watermarks,
4883 };
4884
4885 static const struct intel_wm_funcs vlv_wm_funcs = {
4886         .compute_pipe_wm = vlv_compute_pipe_wm,
4887         .compute_intermediate_wm = vlv_compute_intermediate_wm,
4888         .initial_watermarks = vlv_initial_watermarks,
4889         .optimize_watermarks = vlv_optimize_watermarks,
4890         .atomic_update_watermarks = vlv_atomic_update_fifo,
4891 };
4892
4893 static const struct intel_wm_funcs g4x_wm_funcs = {
4894         .compute_pipe_wm = g4x_compute_pipe_wm,
4895         .compute_intermediate_wm = g4x_compute_intermediate_wm,
4896         .initial_watermarks = g4x_initial_watermarks,
4897         .optimize_watermarks = g4x_optimize_watermarks,
4898 };
4899
4900 static const struct intel_wm_funcs pnv_wm_funcs = {
4901         .update_wm = pnv_update_wm,
4902 };
4903
4904 static const struct intel_wm_funcs i965_wm_funcs = {
4905         .update_wm = i965_update_wm,
4906 };
4907
4908 static const struct intel_wm_funcs i9xx_wm_funcs = {
4909         .update_wm = i9xx_update_wm,
4910 };
4911
4912 static const struct intel_wm_funcs i845_wm_funcs = {
4913         .update_wm = i845_update_wm,
4914 };
4915
4916 static const struct intel_wm_funcs nop_funcs = {
4917 };
4918
4919 /* Set up chip specific power management-related functions */
4920 void intel_init_pm(struct drm_i915_private *dev_priv)
4921 {
4922         if (DISPLAY_VER(dev_priv) >= 9) {
4923                 skl_wm_init(dev_priv);
4924                 return;
4925         }
4926
4927         /* For cxsr */
4928         if (IS_PINEVIEW(dev_priv))
4929                 pnv_get_mem_freq(dev_priv);
4930         else if (GRAPHICS_VER(dev_priv) == 5)
4931                 ilk_get_mem_freq(dev_priv);
4932
4933         /* For FIFO watermark updates */
4934         if (HAS_PCH_SPLIT(dev_priv)) {
4935                 ilk_setup_wm_latency(dev_priv);
4936
4937                 if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->display.wm.pri_latency[1] &&
4938                      dev_priv->display.wm.spr_latency[1] && dev_priv->display.wm.cur_latency[1]) ||
4939                     (DISPLAY_VER(dev_priv) != 5 && dev_priv->display.wm.pri_latency[0] &&
4940                      dev_priv->display.wm.spr_latency[0] && dev_priv->display.wm.cur_latency[0])) {
4941                         dev_priv->display.funcs.wm = &ilk_wm_funcs;
4942                 } else {
4943                         drm_dbg_kms(&dev_priv->drm,
4944                                     "Failed to read display plane latency. "
4945                                     "Disable CxSR\n");
4946                         dev_priv->display.funcs.wm = &nop_funcs;
4947                 }
4948         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
4949                 vlv_setup_wm_latency(dev_priv);
4950                 dev_priv->display.funcs.wm = &vlv_wm_funcs;
4951         } else if (IS_G4X(dev_priv)) {
4952                 g4x_setup_wm_latency(dev_priv);
4953                 dev_priv->display.funcs.wm = &g4x_wm_funcs;
4954         } else if (IS_PINEVIEW(dev_priv)) {
4955                 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
4956                                             dev_priv->is_ddr3,
4957                                             dev_priv->fsb_freq,
4958                                             dev_priv->mem_freq)) {
4959                         drm_info(&dev_priv->drm,
4960                                  "failed to find known CxSR latency "
4961                                  "(found ddr%s fsb freq %d, mem freq %d), "
4962                                  "disabling CxSR\n",
4963                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
4964                                  dev_priv->fsb_freq, dev_priv->mem_freq);
4965                         /* Disable CxSR and never update its watermark again */
4966                         intel_set_memory_cxsr(dev_priv, false);
4967                         dev_priv->display.funcs.wm = &nop_funcs;
4968                 } else
4969                         dev_priv->display.funcs.wm = &pnv_wm_funcs;
4970         } else if (DISPLAY_VER(dev_priv) == 4) {
4971                 dev_priv->display.funcs.wm = &i965_wm_funcs;
4972         } else if (DISPLAY_VER(dev_priv) == 3) {
4973                 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4974         } else if (DISPLAY_VER(dev_priv) == 2) {
4975                 if (INTEL_NUM_PIPES(dev_priv) == 1)
4976                         dev_priv->display.funcs.wm = &i845_wm_funcs;
4977                 else
4978                         dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4979         } else {
4980                 drm_err(&dev_priv->drm,
4981                         "unexpected fall-through in %s\n", __func__);
4982                 dev_priv->display.funcs.wm = &nop_funcs;
4983         }
4984 }
4985
4986 void intel_pm_setup(struct drm_i915_private *dev_priv)
4987 {
4988         dev_priv->runtime_pm.suspended = false;
4989         atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
4990 }