drivers/gpu/drm/i915/display/i9xx_wm.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2023 Intel Corporation
   4  */
   5
   6 #include "i915_drv.h"
   7 #include "i915_reg.h"
   8 #include "i9xx_wm.h"
   9 #include "intel_atomic.h"
  10 #include "intel_display.h"
  11 #include "intel_display_trace.h"
  12 #include "intel_mchbar_regs.h"
  13 #include "intel_wm.h"
  14 #include "skl_watermark.h"
  15 #include "vlv_sideband.h"
  16
  17 /* used in computing the new watermarks state */
  18 struct intel_wm_config {
  19         unsigned int num_pipes_active;
  20         bool sprites_enabled;
  21         bool sprites_scaled;
  22 };
  23
  24 struct cxsr_latency {
  25         bool is_desktop : 1;
  26         bool is_ddr3 : 1;
  27         u16 fsb_freq;
  28         u16 mem_freq;
  29         u16 display_sr;
  30         u16 display_hpll_disable;
  31         u16 cursor_sr;
  32         u16 cursor_hpll_disable;
  33 };
  34
  35 static const struct cxsr_latency cxsr_latency_table[] = {
  36         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
  37         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
  38         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
  39         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
  40         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
  41
  42         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
  43         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
  44         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
  45         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
  46         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
  47
  48         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
  49         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
  50         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
  51         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
  52         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
  53
  54         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
  55         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
  56         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
  57         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
  58         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
  59
  60         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
  61         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
  62         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
  63         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
  64         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
  65
  66         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
  67         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
  68         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
  69         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
  70         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
  71 };
  72
  73 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
  74                                                          bool is_ddr3,
  75                                                          int fsb,
  76                                                          int mem)
  77 {
  78         const struct cxsr_latency *latency;
  79         int i;
  80
  81         if (fsb == 0 || mem == 0)
  82                 return NULL;
  83
  84         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
  85                 latency = &cxsr_latency_table[i];
  86                 if (is_desktop == latency->is_desktop &&
  87                     is_ddr3 == latency->is_ddr3 &&
  88                     fsb == latency->fsb_freq && mem == latency->mem_freq)
  89                         return latency;
  90         }
  91
  92         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
  93
  94         return NULL;
  95 }
  96
  97 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
  98 {
  99         u32 val;
 100
 101         vlv_punit_get(dev_priv);
 102
 103         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 104         if (enable)
 105                 val &= ~FORCE_DDR_HIGH_FREQ;
 106         else
 107                 val |= FORCE_DDR_HIGH_FREQ;
 108         val &= ~FORCE_DDR_LOW_FREQ;
 109         val |= FORCE_DDR_FREQ_REQ_ACK;
 110         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 111
 112         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 113                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 114                 drm_err(&dev_priv->drm,
 115                         "timed out waiting for Punit DDR DVFS request\n");
 116
 117         vlv_punit_put(dev_priv);
 118 }
 119
 120 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 121 {
 122         u32 val;
 123
 124         vlv_punit_get(dev_priv);
 125
 126         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 127         if (enable)
 128                 val |= DSP_MAXFIFO_PM5_ENABLE;
 129         else
 130                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 131         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 132
 133         vlv_punit_put(dev_priv);
 134 }
 135
 136 #define FW_WM(value, plane) \
 137         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 138
 139 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 140 {
 141         bool was_enabled;
 142         u32 val;
 143
 144         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 145                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 146                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 147                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
 148         } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
 149                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
 150                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 151                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
 152         } else if (IS_PINEVIEW(dev_priv)) {
 153                 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
 154                 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 155                 if (enable)
 156                         val |= PINEVIEW_SELF_REFRESH_EN;
 157                 else
 158                         val &= ~PINEVIEW_SELF_REFRESH_EN;
 159                 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
 160                 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
 161         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 162                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
 163                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 164                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 165                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
 166                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
 167         } else if (IS_I915GM(dev_priv)) {
 168                 /*
 169                  * FIXME can't find a bit like this for 915G, and
 170                  * yet it does have the related watermark in
 171                  * FW_BLC_SELF. What's going on?
 172                  */
 173                 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
 174                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 175                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 176                 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
 177                 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
 178         } else {
 179                 return false;
 180         }
 181
 182         trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
 183
 184         drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
 185                     str_enabled_disabled(enable),
 186                     str_enabled_disabled(was_enabled));
 187
 188         return was_enabled;
 189 }
 190
 191 /**
 192  * intel_set_memory_cxsr - Configure CxSR state
 193  * @dev_priv: i915 device
 194  * @enable: Allow vs. disallow CxSR
 195  *
 196  * Allow or disallow the system to enter a special CxSR
 197  * (C-state self refresh) state. What typically happens in CxSR mode
 198  * is that several display FIFOs may get combined into a single larger
 199  * FIFO for a particular plane (so called max FIFO mode) to allow the
 200  * system to defer memory fetches longer, and the memory will enter
 201  * self refresh.
 202  *
 203  * Note that enabling CxSR does not guarantee that the system enter
 204  * this special mode, nor does it guarantee that the system stays
 205  * in that mode once entered. So this just allows/disallows the system
 206  * to autonomously utilize the CxSR mode. Other factors such as core
 207  * C-states will affect when/if the system actually enters/exits the
 208  * CxSR mode.
 209  *
 210  * Note that on VLV/CHV this actually only controls the max FIFO mode,
 211  * and the system is free to enter/exit memory self refresh at any time
 212  * even when the use of CxSR has been disallowed.
 213  *
 214  * While the system is actually in the CxSR/max FIFO mode, some plane
 215  * control registers will not get latched on vblank. Thus in order to
 216  * guarantee the system will respond to changes in the plane registers
 217  * we must always disallow CxSR prior to making changes to those registers.
 218  * Unfortunately the system will re-evaluate the CxSR conditions at
 219  * frame start which happens after vblank start (which is when the plane
 220  * registers would get latched), so we can't proceed with the plane update
 221  * during the same frame where we disallowed CxSR.
 222  *
 223  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 224  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 225  * the hardware w.r.t. HPLL SR when writing to plane registers.
 226  * Disallowing just CxSR is sufficient.
 227  */
 228 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 229 {
 230         bool ret;
 231
 232         mutex_lock(&dev_priv->display.wm.wm_mutex);
 233         ret = _intel_set_memory_cxsr(dev_priv, enable);
 234         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 235                 dev_priv->display.wm.vlv.cxsr = enable;
 236         else if (IS_G4X(dev_priv))
 237                 dev_priv->display.wm.g4x.cxsr = enable;
 238         mutex_unlock(&dev_priv->display.wm.wm_mutex);
 239
 240         return ret;
 241 }
 242
 243 /*
 244  * Latency for FIFO fetches is dependent on several factors:
 245  *   - memory configuration (speed, channels)
 246  *   - chipset
 247  *   - current MCH state
 248  * It can be fairly high in some situations, so here we assume a fairly
 249  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 250  * set this value too high, the FIFO will fetch frequently to stay full)
 251  * and power consumption (set it too low to save power and we might see
 252  * FIFO underruns and display "flicker").
 253  *
 254  * A value of 5us seems to be a good balance; safe for very low end
 255  * platforms but not overly aggressive on lower latency configs.
 256  */
 257 static const int pessimal_latency_ns = 5000;
 258
 259 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 260         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 261
 262 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 263 {
 264         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 265         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 266         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 267         enum pipe pipe = crtc->pipe;
 268         int sprite0_start, sprite1_start;
 269         u32 dsparb, dsparb2, dsparb3;
 270
 271         switch (pipe) {
 272         case PIPE_A:
 273                 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 274                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 275                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 276                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 277                 break;
 278         case PIPE_B:
 279                 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 280                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 281                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 282                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 283                 break;
 284         case PIPE_C:
 285                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 286                 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
 287                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 288                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 289                 break;
 290         default:
 291                 MISSING_CASE(pipe);
 292                 return;
 293         }
 294
 295         fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 296         fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 297         fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 298         fifo_state->plane[PLANE_CURSOR] = 63;
 299 }
 300
 301 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
 302                               enum i9xx_plane_id i9xx_plane)
 303 {
 304         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 305         int size;
 306
 307         size = dsparb & 0x7f;
 308         if (i9xx_plane == PLANE_B)
 309                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 310
 311         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 312                     dsparb, plane_name(i9xx_plane), size);
 313
 314         return size;
 315 }
 316
 317 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
 318                               enum i9xx_plane_id i9xx_plane)
 319 {
 320         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 321         int size;
 322
 323         size = dsparb & 0x1ff;
 324         if (i9xx_plane == PLANE_B)
 325                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 326         size >>= 1; /* Convert to cachelines */
 327
 328         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 329                     dsparb, plane_name(i9xx_plane), size);
 330
 331         return size;
 332 }
 333
 334 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
 335                               enum i9xx_plane_id i9xx_plane)
 336 {
 337         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 338         int size;
 339
 340         size = dsparb & 0x7f;
 341         size >>= 2; /* Convert to cachelines */
 342
 343         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 344                     dsparb, plane_name(i9xx_plane), size);
 345
 346         return size;
 347 }
 348
 349 /* Pineview has different values for various configs */
 350 static const struct intel_watermark_params pnv_display_wm = {
 351         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 352         .max_wm = PINEVIEW_MAX_WM,
 353         .default_wm = PINEVIEW_DFT_WM,
 354         .guard_size = PINEVIEW_GUARD_WM,
 355         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 356 };
 357
 358 static const struct intel_watermark_params pnv_display_hplloff_wm = {
 359         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 360         .max_wm = PINEVIEW_MAX_WM,
 361         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 362         .guard_size = PINEVIEW_GUARD_WM,
 363         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 364 };
 365
 366 static const struct intel_watermark_params pnv_cursor_wm = {
 367         .fifo_size = PINEVIEW_CURSOR_FIFO,
 368         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 369         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 370         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 371         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 372 };
 373
 374 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
 375         .fifo_size = PINEVIEW_CURSOR_FIFO,
 376         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 377         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 378         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 379         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 380 };
 381
 382 static const struct intel_watermark_params i965_cursor_wm_info = {
 383         .fifo_size = I965_CURSOR_FIFO,
 384         .max_wm = I965_CURSOR_MAX_WM,
 385         .default_wm = I965_CURSOR_DFT_WM,
 386         .guard_size = 2,
 387         .cacheline_size = I915_FIFO_LINE_SIZE,
 388 };
 389
 390 static const struct intel_watermark_params i945_wm_info = {
 391         .fifo_size = I945_FIFO_SIZE,
 392         .max_wm = I915_MAX_WM,
 393         .default_wm = 1,
 394         .guard_size = 2,
 395         .cacheline_size = I915_FIFO_LINE_SIZE,
 396 };
 397
 398 static const struct intel_watermark_params i915_wm_info = {
 399         .fifo_size = I915_FIFO_SIZE,
 400         .max_wm = I915_MAX_WM,
 401         .default_wm = 1,
 402         .guard_size = 2,
 403         .cacheline_size = I915_FIFO_LINE_SIZE,
 404 };
 405
 406 static const struct intel_watermark_params i830_a_wm_info = {
 407         .fifo_size = I855GM_FIFO_SIZE,
 408         .max_wm = I915_MAX_WM,
 409         .default_wm = 1,
 410         .guard_size = 2,
 411         .cacheline_size = I830_FIFO_LINE_SIZE,
 412 };
 413
 414 static const struct intel_watermark_params i830_bc_wm_info = {
 415         .fifo_size = I855GM_FIFO_SIZE,
 416         .max_wm = I915_MAX_WM / 2,
 417         .default_wm = 1,
 418         .guard_size = 2,
 419         .cacheline_size = I830_FIFO_LINE_SIZE,
 420 };
 421
 422 static const struct intel_watermark_params i845_wm_info = {
 423         .fifo_size = I830_FIFO_SIZE,
 424         .max_wm = I915_MAX_WM,
 425         .default_wm = 1,
 426         .guard_size = 2,
 427         .cacheline_size = I830_FIFO_LINE_SIZE,
 428 };
 429
 430 /**
 431  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 432  * @pixel_rate: Pipe pixel rate in kHz
 433  * @cpp: Plane bytes per pixel
 434  * @latency: Memory wakeup latency in 0.1us units
 435  *
 436  * Compute the watermark using the method 1 or "small buffer"
 437  * formula. The caller may additonally add extra cachelines
 438  * to account for TLB misses and clock crossings.
 439  *
 440  * This method is concerned with the short term drain rate
 441  * of the FIFO, ie. it does not account for blanking periods
 442  * which would effectively reduce the average drain rate across
 443  * a longer period. The name "small" refers to the fact the
 444  * FIFO is relatively small compared to the amount of data
 445  * fetched.
 446  *
 447  * The FIFO level vs. time graph might look something like:
 448  *
 449  *   |\   |\
 450  *   | \  | \
 451  * __---__---__ (- plane active, _ blanking)
 452  * -> time
 453  *
 454  * or perhaps like this:
 455  *
 456  *   |\|\  |\|\
 457  * __----__----__ (- plane active, _ blanking)
 458  * -> time
 459  *
 460  * Returns:
 461  * The watermark in bytes
 462  */
 463 static unsigned int intel_wm_method1(unsigned int pixel_rate,
 464                                      unsigned int cpp,
 465                                      unsigned int latency)
 466 {
 467         u64 ret;
 468
 469         ret = mul_u32_u32(pixel_rate, cpp * latency);
 470         ret = DIV_ROUND_UP_ULL(ret, 10000);
 471
 472         return ret;
 473 }
 474
 475 /**
 476  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 477  * @pixel_rate: Pipe pixel rate in kHz
 478  * @htotal: Pipe horizontal total
 479  * @width: Plane width in pixels
 480  * @cpp: Plane bytes per pixel
 481  * @latency: Memory wakeup latency in 0.1us units
 482  *
 483  * Compute the watermark using the method 2 or "large buffer"
 484  * formula. The caller may additonally add extra cachelines
 485  * to account for TLB misses and clock crossings.
 486  *
 487  * This method is concerned with the long term drain rate
 488  * of the FIFO, ie. it does account for blanking periods
 489  * which effectively reduce the average drain rate across
 490  * a longer period. The name "large" refers to the fact the
 491  * FIFO is relatively large compared to the amount of data
 492  * fetched.
 493  *
 494  * The FIFO level vs. time graph might look something like:
 495  *
 496  *    |\___       |\___
 497  *    |    \___   |    \___
 498  *    |        \  |        \
 499  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 500  * -> time
 501  *
 502  * Returns:
 503  * The watermark in bytes
 504  */
 505 static unsigned int intel_wm_method2(unsigned int pixel_rate,
 506                                      unsigned int htotal,
 507                                      unsigned int width,
 508                                      unsigned int cpp,
 509                                      unsigned int latency)
 510 {
 511         unsigned int ret;
 512
 513         /*
 514          * FIXME remove once all users are computing
 515          * watermarks in the correct place.
 516          */
 517         if (WARN_ON_ONCE(htotal == 0))
 518                 htotal = 1;
 519
 520         ret = (latency * pixel_rate) / (htotal * 10000);
 521         ret = (ret + 1) * width * cpp;
 522
 523         return ret;
 524 }
 525
 526 /**
 527  * intel_calculate_wm - calculate watermark level
 528  * @pixel_rate: pixel clock
 529  * @wm: chip FIFO params
 530  * @fifo_size: size of the FIFO buffer
 531  * @cpp: bytes per pixel
 532  * @latency_ns: memory latency for the platform
 533  *
 534  * Calculate the watermark level (the level at which the display plane will
 535  * start fetching from memory again).  Each chip has a different display
 536  * FIFO size and allocation, so the caller needs to figure that out and pass
 537  * in the correct intel_watermark_params structure.
 538  *
 539  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 540  * on the pixel size.  When it reaches the watermark level, it'll start
 541  * fetching FIFO line sized based chunks from memory until the FIFO fills
 542  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 543  * will occur, and a display engine hang could result.
 544  */
 545 static unsigned int intel_calculate_wm(int pixel_rate,
 546                                        const struct intel_watermark_params *wm,
 547                                        int fifo_size, int cpp,
 548                                        unsigned int latency_ns)
 549 {
 550         int entries, wm_size;
 551
 552         /*
 553          * Note: we need to make sure we don't overflow for various clock &
 554          * latency values.
 555          * clocks go from a few thousand to several hundred thousand.
 556          * latency is usually a few thousand
 557          */
 558         entries = intel_wm_method1(pixel_rate, cpp,
 559                                    latency_ns / 100);
 560         entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 561                 wm->guard_size;
 562         DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
 563
 564         wm_size = fifo_size - entries;
 565         DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
 566
 567         /* Don't promote wm_size to unsigned... */
 568         if (wm_size > wm->max_wm)
 569                 wm_size = wm->max_wm;
 570         if (wm_size <= 0)
 571                 wm_size = wm->default_wm;
 572
 573         /*
 574          * Bspec seems to indicate that the value shouldn't be lower than
 575          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 576          * Lets go for 8 which is the burst size since certain platforms
 577          * already use a hardcoded 8 (which is what the spec says should be
 578          * done).
 579          */
 580         if (wm_size <= 8)
 581                 wm_size = 8;
 582
 583         return wm_size;
 584 }
 585
 586 static bool is_disabling(int old, int new, int threshold)
 587 {
 588         return old >= threshold && new < threshold;
 589 }
 590
 591 static bool is_enabling(int old, int new, int threshold)
 592 {
 593         return old < threshold && new >= threshold;
 594 }
 595
 596 static bool intel_crtc_active(struct intel_crtc *crtc)
 597 {
 598         /* Be paranoid as we can arrive here with only partial
 599          * state retrieved from the hardware during setup.
 600          *
 601          * We can ditch the adjusted_mode.crtc_clock check as soon
 602          * as Haswell has gained clock readout/fastboot support.
 603          *
 604          * We can ditch the crtc->primary->state->fb check as soon as we can
 605          * properly reconstruct framebuffers.
 606          *
 607          * FIXME: The intel_crtc->active here should be switched to
 608          * crtc->state->active once we have proper CRTC states wired up
 609          * for atomic.
 610          */
 611         return crtc && crtc->active && crtc->base.primary->state->fb &&
 612                 crtc->config->hw.adjusted_mode.crtc_clock;
 613 }
 614
 615 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 616 {
 617         struct intel_crtc *crtc, *enabled = NULL;
 618
 619         for_each_intel_crtc(&dev_priv->drm, crtc) {
 620                 if (intel_crtc_active(crtc)) {
 621                         if (enabled)
 622                                 return NULL;
 623                         enabled = crtc;
 624                 }
 625         }
 626
 627         return enabled;
 628 }
 629
 630 static void pnv_update_wm(struct drm_i915_private *dev_priv)
 631 {
 632         struct intel_crtc *crtc;
 633         const struct cxsr_latency *latency;
 634         u32 reg;
 635         unsigned int wm;
 636
 637         latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
 638                                          dev_priv->is_ddr3,
 639                                          dev_priv->fsb_freq,
 640                                          dev_priv->mem_freq);
 641         if (!latency) {
 642                 drm_dbg_kms(&dev_priv->drm,
 643                             "Unknown FSB/MEM found, disable CxSR\n");
 644                 intel_set_memory_cxsr(dev_priv, false);
 645                 return;
 646         }
 647
 648         crtc = single_enabled_crtc(dev_priv);
 649         if (crtc) {
 650                 const struct drm_framebuffer *fb =
 651                         crtc->base.primary->state->fb;
 652                 int pixel_rate = crtc->config->pixel_rate;
 653                 int cpp = fb->format->cpp[0];
 654
 655                 /* Display SR */
 656                 wm = intel_calculate_wm(pixel_rate, &pnv_display_wm,
 657                                         pnv_display_wm.fifo_size,
 658                                         cpp, latency->display_sr);
 659                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
 660                 reg &= ~DSPFW_SR_MASK;
 661                 reg |= FW_WM(wm, SR);
 662                 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
 663                 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
 664
 665                 /* cursor SR */
 666                 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_wm,
 667                                         pnv_display_wm.fifo_size,
 668                                         4, latency->cursor_sr);
 669                 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_CURSOR_SR_MASK,
 670                                  FW_WM(wm, CURSOR_SR));
 671
 672                 /* Display HPLL off SR */
 673                 wm = intel_calculate_wm(pixel_rate, &pnv_display_hplloff_wm,
 674                                         pnv_display_hplloff_wm.fifo_size,
 675                                         cpp, latency->display_hpll_disable);
 676                 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR));
 677
 678                 /* cursor HPLL off SR */
 679                 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_hplloff_wm,
 680                                         pnv_display_hplloff_wm.fifo_size,
 681                                         4, latency->cursor_hpll_disable);
 682                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
 683                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 684                 reg |= FW_WM(wm, HPLL_CURSOR);
 685                 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
 686                 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
 687
 688                 intel_set_memory_cxsr(dev_priv, true);
 689         } else {
 690                 intel_set_memory_cxsr(dev_priv, false);
 691         }
 692 }
 693
 694 /*
 695  * Documentation says:
 696  * "If the line size is small, the TLB fetches can get in the way of the
 697  *  data fetches, causing some lag in the pixel data return which is not
 698  *  accounted for in the above formulas. The following adjustment only
 699  *  needs to be applied if eight whole lines fit in the buffer at once.
 700  *  The WM is adjusted upwards by the difference between the FIFO size
 701  *  and the size of 8 whole lines. This adjustment is always performed
 702  *  in the actual pixel depth regardless of whether FBC is enabled or not."
 703  */
 704 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 705 {
 706         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 707
 708         return max(0, tlb_miss);
 709 }
 710
 711 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 712                                 const struct g4x_wm_values *wm)
 713 {
 714         enum pipe pipe;
 715
 716         for_each_pipe(dev_priv, pipe)
 717                 trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
 718
 719         intel_uncore_write(&dev_priv->uncore, DSPFW1,
 720                            FW_WM(wm->sr.plane, SR) |
 721                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 722                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 723                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 724         intel_uncore_write(&dev_priv->uncore, DSPFW2,
 725                            (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 726                            FW_WM(wm->sr.fbc, FBC_SR) |
 727                            FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
 728                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
 729                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 730                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 731         intel_uncore_write(&dev_priv->uncore, DSPFW3,
 732                            (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
 733                            FW_WM(wm->sr.cursor, CURSOR_SR) |
 734                            FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
 735                            FW_WM(wm->hpll.plane, HPLL_SR));
 736
 737         intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
 738 }
 739
 740 #define FW_WM_VLV(value, plane) \
 741         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
 742
 743 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
 744                                 const struct vlv_wm_values *wm)
 745 {
 746         enum pipe pipe;
 747
 748         for_each_pipe(dev_priv, pipe) {
 749                 trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
 750
 751                 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
 752                                    (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
 753                                    (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
 754                                    (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
 755                                    (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
 756         }
 757
 758         /*
 759          * Zero the (unused) WM1 watermarks, and also clear all the
 760          * high order bits so that there are no out of bounds values
 761          * present in the registers during the reprogramming.
 762          */
 763         intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
 764         intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
 765         intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
 766         intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
 767         intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
 768
 769         intel_uncore_write(&dev_priv->uncore, DSPFW1,
 770                            FW_WM(wm->sr.plane, SR) |
 771                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 772                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 773                            FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 774         intel_uncore_write(&dev_priv->uncore, DSPFW2,
 775                            FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
 776                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 777                            FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 778         intel_uncore_write(&dev_priv->uncore, DSPFW3,
 779                            FW_WM(wm->sr.cursor, CURSOR_SR));
 780
 781         if (IS_CHERRYVIEW(dev_priv)) {
 782                 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
 783                                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
 784                                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
 785                 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
 786                                    FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
 787                                    FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
 788                 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
 789                                    FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
 790                                    FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
 791                 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
 792                                    FW_WM(wm->sr.plane >> 9, SR_HI) |
 793                                    FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
 794                                    FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
 795                                    FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
 796                                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
 797                                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
 798                                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
 799                                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
 800                                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
 801                                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
 802         } else {
 803                 intel_uncore_write(&dev_priv->uncore, DSPFW7,
 804                                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
 805                                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
 806                 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
 807                                    FW_WM(wm->sr.plane >> 9, SR_HI) |
 808                                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
 809                                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
 810                                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
 811                                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
 812                                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
 813                                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
 814         }
 815
 816         intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
 817 }
 818
 819 #undef FW_WM_VLV
 820
 821 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
 822 {
 823         /* all latencies in usec */
 824         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
 825         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
 826         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
 827
 828         dev_priv->display.wm.num_levels = G4X_WM_LEVEL_HPLL + 1;
 829 }
 830
 831 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
 832 {
 833         /*
 834          * DSPCNTR[13] supposedly controls whether the
 835          * primary plane can use the FIFO space otherwise
 836          * reserved for the sprite plane. It's not 100% clear
 837          * what the actual FIFO size is, but it looks like we
 838          * can happily set both primary and sprite watermarks
 839          * up to 127 cachelines. So that would seem to mean
 840          * that either DSPCNTR[13] doesn't do anything, or that
 841          * the total FIFO is >= 256 cachelines in size. Either
 842          * way, we don't seem to have to worry about this
 843          * repartitioning as the maximum watermark value the
 844          * register can hold for each plane is lower than the
 845          * minimum FIFO size.
 846          */
 847         switch (plane_id) {
 848         case PLANE_CURSOR:
 849                 return 63;
 850         case PLANE_PRIMARY:
 851                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
 852         case PLANE_SPRITE0:
 853                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
 854         default:
 855                 MISSING_CASE(plane_id);
 856                 return 0;
 857         }
 858 }
 859
 860 static int g4x_fbc_fifo_size(int level)
 861 {
 862         switch (level) {
 863         case G4X_WM_LEVEL_SR:
 864                 return 7;
 865         case G4X_WM_LEVEL_HPLL:
 866                 return 15;
 867         default:
 868                 MISSING_CASE(level);
 869                 return 0;
 870         }
 871 }
 872
 873 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
 874                           const struct intel_plane_state *plane_state,
 875                           int level)
 876 {
 877         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 878         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 879         const struct drm_display_mode *pipe_mode =
 880                 &crtc_state->hw.pipe_mode;
 881         unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;
 882         unsigned int pixel_rate, htotal, cpp, width, wm;
 883
 884         if (latency == 0)
 885                 return USHRT_MAX;
 886
 887         if (!intel_wm_plane_visible(crtc_state, plane_state))
 888                 return 0;
 889
 890         cpp = plane_state->hw.fb->format->cpp[0];
 891
 892         /*
 893          * WaUse32BppForSRWM:ctg,elk
 894          *
 895          * The spec fails to list this restriction for the
 896          * HPLL watermark, which seems a little strange.
 897          * Let's use 32bpp for the HPLL watermark as well.
 898          */
 899         if (plane->id == PLANE_PRIMARY &&
 900             level != G4X_WM_LEVEL_NORMAL)
 901                 cpp = max(cpp, 4u);
 902
 903         pixel_rate = crtc_state->pixel_rate;
 904         htotal = pipe_mode->crtc_htotal;
 905         width = drm_rect_width(&plane_state->uapi.src) >> 16;
 906
 907         if (plane->id == PLANE_CURSOR) {
 908                 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
 909         } else if (plane->id == PLANE_PRIMARY &&
 910                    level == G4X_WM_LEVEL_NORMAL) {
 911                 wm = intel_wm_method1(pixel_rate, cpp, latency);
 912         } else {
 913                 unsigned int small, large;
 914
 915                 small = intel_wm_method1(pixel_rate, cpp, latency);
 916                 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
 917
 918                 wm = min(small, large);
 919         }
 920
 921         wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
 922                               width, cpp);
 923
 924         wm = DIV_ROUND_UP(wm, 64) + 2;
 925
 926         return min_t(unsigned int, wm, USHRT_MAX);
 927 }
 928
 929 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
 930                                  int level, enum plane_id plane_id, u16 value)
 931 {
 932         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 933         bool dirty = false;
 934
 935         for (; level < dev_priv->display.wm.num_levels; level++) {
 936                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
 937
 938                 dirty |= raw->plane[plane_id] != value;
 939                 raw->plane[plane_id] = value;
 940         }
 941
 942         return dirty;
 943 }
 944
 945 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
 946                                int level, u16 value)
 947 {
 948         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 949         bool dirty = false;
 950
 951         /* NORMAL level doesn't have an FBC watermark */
 952         level = max(level, G4X_WM_LEVEL_SR);
 953
 954         for (; level < dev_priv->display.wm.num_levels; level++) {
 955                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
 956
 957                 dirty |= raw->fbc != value;
 958                 raw->fbc = value;
 959         }
 960
 961         return dirty;
 962 }
 963
 964 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
 965                               const struct intel_plane_state *plane_state,
 966                               u32 pri_val);
 967
 968 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
 969                                      const struct intel_plane_state *plane_state)
 970 {
 971         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 972         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 973         enum plane_id plane_id = plane->id;
 974         bool dirty = false;
 975         int level;
 976
 977         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
 978                 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
 979                 if (plane_id == PLANE_PRIMARY)
 980                         dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
 981                 goto out;
 982         }
 983
 984         for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
 985                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
 986                 int wm, max_wm;
 987
 988                 wm = g4x_compute_wm(crtc_state, plane_state, level);
 989                 max_wm = g4x_plane_fifo_size(plane_id, level);
 990
 991                 if (wm > max_wm)
 992                         break;
 993
 994                 dirty |= raw->plane[plane_id] != wm;
 995                 raw->plane[plane_id] = wm;
 996
 997                 if (plane_id != PLANE_PRIMARY ||
 998                     level == G4X_WM_LEVEL_NORMAL)
 999                         continue;
1000
1001                 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1002                                         raw->plane[plane_id]);
1003                 max_wm = g4x_fbc_fifo_size(level);
1004
1005                 /*
1006                  * FBC wm is not mandatory as we
1007                  * can always just disable its use.
1008                  */
1009                 if (wm > max_wm)
1010                         wm = USHRT_MAX;
1011
1012                 dirty |= raw->fbc != wm;
1013                 raw->fbc = wm;
1014         }
1015
1016         /* mark watermarks as invalid */
1017         dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1018
1019         if (plane_id == PLANE_PRIMARY)
1020                 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1021
1022  out:
1023         if (dirty) {
1024                 drm_dbg_kms(&dev_priv->drm,
1025                             "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1026                             plane->base.name,
1027                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1028                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1029                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1030
1031                 if (plane_id == PLANE_PRIMARY)
1032                         drm_dbg_kms(&dev_priv->drm,
1033                                     "FBC watermarks: SR=%d, HPLL=%d\n",
1034                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1035                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1036         }
1037
1038         return dirty;
1039 }
1040
1041 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1042                                       enum plane_id plane_id, int level)
1043 {
1044         const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1045
1046         return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1047 }
1048
1049 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1050                                      int level)
1051 {
1052         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1053
1054         if (level >= dev_priv->display.wm.num_levels)
1055                 return false;
1056
1057         return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1058                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1059                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1060 }
1061
1062 /* mark all levels starting from 'level' as invalid */
1063 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1064                                struct g4x_wm_state *wm_state, int level)
1065 {
1066         if (level <= G4X_WM_LEVEL_NORMAL) {
1067                 enum plane_id plane_id;
1068
1069                 for_each_plane_id_on_crtc(crtc, plane_id)
1070                         wm_state->wm.plane[plane_id] = USHRT_MAX;
1071         }
1072
1073         if (level <= G4X_WM_LEVEL_SR) {
1074                 wm_state->cxsr = false;
1075                 wm_state->sr.cursor = USHRT_MAX;
1076                 wm_state->sr.plane = USHRT_MAX;
1077                 wm_state->sr.fbc = USHRT_MAX;
1078         }
1079
1080         if (level <= G4X_WM_LEVEL_HPLL) {
1081                 wm_state->hpll_en = false;
1082                 wm_state->hpll.cursor = USHRT_MAX;
1083                 wm_state->hpll.plane = USHRT_MAX;
1084                 wm_state->hpll.fbc = USHRT_MAX;
1085         }
1086 }
1087
1088 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1089                                int level)
1090 {
1091         if (level < G4X_WM_LEVEL_SR)
1092                 return false;
1093
1094         if (level >= G4X_WM_LEVEL_SR &&
1095             wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1096                 return false;
1097
1098         if (level >= G4X_WM_LEVEL_HPLL &&
1099             wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1100                 return false;
1101
1102         return true;
1103 }
1104
1105 static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1106 {
1107         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1108         struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1109         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1110         const struct g4x_pipe_wm *raw;
1111         enum plane_id plane_id;
1112         int level;
1113
1114         level = G4X_WM_LEVEL_NORMAL;
1115         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1116                 goto out;
1117
1118         raw = &crtc_state->wm.g4x.raw[level];
1119         for_each_plane_id_on_crtc(crtc, plane_id)
1120                 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1121
1122         level = G4X_WM_LEVEL_SR;
1123         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1124                 goto out;
1125
1126         raw = &crtc_state->wm.g4x.raw[level];
1127         wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1128         wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1129         wm_state->sr.fbc = raw->fbc;
1130
1131         wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1132
1133         level = G4X_WM_LEVEL_HPLL;
1134         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1135                 goto out;
1136
1137         raw = &crtc_state->wm.g4x.raw[level];
1138         wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1139         wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1140         wm_state->hpll.fbc = raw->fbc;
1141
1142         wm_state->hpll_en = wm_state->cxsr;
1143
1144         level++;
1145
1146  out:
1147         if (level == G4X_WM_LEVEL_NORMAL)
1148                 return -EINVAL;
1149
1150         /* invalidate the higher levels */
1151         g4x_invalidate_wms(crtc, wm_state, level);
1152
1153         /*
1154          * Determine if the FBC watermark(s) can be used. IF
1155          * this isn't the case we prefer to disable the FBC
1156          * watermark(s) rather than disable the SR/HPLL
1157          * level(s) entirely. 'level-1' is the highest valid
1158          * level here.
1159          */
1160         wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1161
1162         return 0;
1163 }
1164
1165 static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1166                                struct intel_crtc *crtc)
1167 {
1168         struct intel_crtc_state *crtc_state =
1169                 intel_atomic_get_new_crtc_state(state, crtc);
1170         const struct intel_plane_state *old_plane_state;
1171         const struct intel_plane_state *new_plane_state;
1172         struct intel_plane *plane;
1173         unsigned int dirty = 0;
1174         int i;
1175
1176         for_each_oldnew_intel_plane_in_state(state, plane,
1177                                              old_plane_state,
1178                                              new_plane_state, i) {
1179                 if (new_plane_state->hw.crtc != &crtc->base &&
1180                     old_plane_state->hw.crtc != &crtc->base)
1181                         continue;
1182
1183                 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1184                         dirty |= BIT(plane->id);
1185         }
1186
1187         if (!dirty)
1188                 return 0;
1189
1190         return _g4x_compute_pipe_wm(crtc_state);
1191 }
1192
1193 static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1194                                        struct intel_crtc *crtc)
1195 {
1196         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1197         struct intel_crtc_state *new_crtc_state =
1198                 intel_atomic_get_new_crtc_state(state, crtc);
1199         const struct intel_crtc_state *old_crtc_state =
1200                 intel_atomic_get_old_crtc_state(state, crtc);
1201         struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1202         const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1203         const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1204         enum plane_id plane_id;
1205
1206         if (!new_crtc_state->hw.active ||
1207             intel_crtc_needs_modeset(new_crtc_state)) {
1208                 *intermediate = *optimal;
1209
1210                 intermediate->cxsr = false;
1211                 intermediate->hpll_en = false;
1212                 goto out;
1213         }
1214
1215         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1216                 !new_crtc_state->disable_cxsr;
1217         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1218                 !new_crtc_state->disable_cxsr;
1219         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1220
1221         for_each_plane_id_on_crtc(crtc, plane_id) {
1222                 intermediate->wm.plane[plane_id] =
1223                         max(optimal->wm.plane[plane_id],
1224                             active->wm.plane[plane_id]);
1225
1226                 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1227                             g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1228         }
1229
1230         intermediate->sr.plane = max(optimal->sr.plane,
1231                                      active->sr.plane);
1232         intermediate->sr.cursor = max(optimal->sr.cursor,
1233                                       active->sr.cursor);
1234         intermediate->sr.fbc = max(optimal->sr.fbc,
1235                                    active->sr.fbc);
1236
1237         intermediate->hpll.plane = max(optimal->hpll.plane,
1238                                        active->hpll.plane);
1239         intermediate->hpll.cursor = max(optimal->hpll.cursor,
1240                                         active->hpll.cursor);
1241         intermediate->hpll.fbc = max(optimal->hpll.fbc,
1242                                      active->hpll.fbc);
1243
1244         drm_WARN_ON(&dev_priv->drm,
1245                     (intermediate->sr.plane >
1246                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1247                      intermediate->sr.cursor >
1248                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1249                     intermediate->cxsr);
1250         drm_WARN_ON(&dev_priv->drm,
1251                     (intermediate->sr.plane >
1252                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1253                      intermediate->sr.cursor >
1254                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1255                     intermediate->hpll_en);
1256
1257         drm_WARN_ON(&dev_priv->drm,
1258                     intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1259                     intermediate->fbc_en && intermediate->cxsr);
1260         drm_WARN_ON(&dev_priv->drm,
1261                     intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1262                     intermediate->fbc_en && intermediate->hpll_en);
1263
1264 out:
1265         /*
1266          * If our intermediate WM are identical to the final WM, then we can
1267          * omit the post-vblank programming; only update if it's different.
1268          */
1269         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1270                 new_crtc_state->wm.need_postvbl_update = true;
1271
1272         return 0;
1273 }
1274
1275 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1276                          struct g4x_wm_values *wm)
1277 {
1278         struct intel_crtc *crtc;
1279         int num_active_pipes = 0;
1280
1281         wm->cxsr = true;
1282         wm->hpll_en = true;
1283         wm->fbc_en = true;
1284
1285         for_each_intel_crtc(&dev_priv->drm, crtc) {
1286                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1287
1288                 if (!crtc->active)
1289                         continue;
1290
1291                 if (!wm_state->cxsr)
1292                         wm->cxsr = false;
1293                 if (!wm_state->hpll_en)
1294                         wm->hpll_en = false;
1295                 if (!wm_state->fbc_en)
1296                         wm->fbc_en = false;
1297
1298                 num_active_pipes++;
1299         }
1300
1301         if (num_active_pipes != 1) {
1302                 wm->cxsr = false;
1303                 wm->hpll_en = false;
1304                 wm->fbc_en = false;
1305         }
1306
1307         for_each_intel_crtc(&dev_priv->drm, crtc) {
1308                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1309                 enum pipe pipe = crtc->pipe;
1310
1311                 wm->pipe[pipe] = wm_state->wm;
1312                 if (crtc->active && wm->cxsr)
1313                         wm->sr = wm_state->sr;
1314                 if (crtc->active && wm->hpll_en)
1315                         wm->hpll = wm_state->hpll;
1316         }
1317 }
1318
1319 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1320 {
1321         struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;
1322         struct g4x_wm_values new_wm = {};
1323
1324         g4x_merge_wm(dev_priv, &new_wm);
1325
1326         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1327                 return;
1328
1329         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1330                 _intel_set_memory_cxsr(dev_priv, false);
1331
1332         g4x_write_wm_values(dev_priv, &new_wm);
1333
1334         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1335                 _intel_set_memory_cxsr(dev_priv, true);
1336
1337         *old_wm = new_wm;
1338 }
1339
1340 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1341                                    struct intel_crtc *crtc)
1342 {
1343         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1344         const struct intel_crtc_state *crtc_state =
1345                 intel_atomic_get_new_crtc_state(state, crtc);
1346
1347         mutex_lock(&dev_priv->display.wm.wm_mutex);
1348         crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1349         g4x_program_watermarks(dev_priv);
1350         mutex_unlock(&dev_priv->display.wm.wm_mutex);
1351 }
1352
1353 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1354                                     struct intel_crtc *crtc)
1355 {
1356         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1357         const struct intel_crtc_state *crtc_state =
1358                 intel_atomic_get_new_crtc_state(state, crtc);
1359
1360         if (!crtc_state->wm.need_postvbl_update)
1361                 return;
1362
1363         mutex_lock(&dev_priv->display.wm.wm_mutex);
1364         crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1365         g4x_program_watermarks(dev_priv);
1366         mutex_unlock(&dev_priv->display.wm.wm_mutex);
1367 }
1368
1369 /* latency must be in 0.1us units. */
1370 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1371                                    unsigned int htotal,
1372                                    unsigned int width,
1373                                    unsigned int cpp,
1374                                    unsigned int latency)
1375 {
1376         unsigned int ret;
1377
1378         ret = intel_wm_method2(pixel_rate, htotal,
1379                                width, cpp, latency);
1380         ret = DIV_ROUND_UP(ret, 64);
1381
1382         return ret;
1383 }
1384
1385 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1386 {
1387         /* all latencies in usec */
1388         dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1389
1390         dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM2 + 1;
1391
1392         if (IS_CHERRYVIEW(dev_priv)) {
1393                 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1394                 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1395
1396                 dev_priv->display.wm.num_levels = VLV_WM_LEVEL_DDR_DVFS + 1;
1397         }
1398 }
1399
1400 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1401                                 const struct intel_plane_state *plane_state,
1402                                 int level)
1403 {
1404         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1405         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1406         const struct drm_display_mode *pipe_mode =
1407                 &crtc_state->hw.pipe_mode;
1408         unsigned int pixel_rate, htotal, cpp, width, wm;
1409
1410         if (dev_priv->display.wm.pri_latency[level] == 0)
1411                 return USHRT_MAX;
1412
1413         if (!intel_wm_plane_visible(crtc_state, plane_state))
1414                 return 0;
1415
1416         cpp = plane_state->hw.fb->format->cpp[0];
1417         pixel_rate = crtc_state->pixel_rate;
1418         htotal = pipe_mode->crtc_htotal;
1419         width = drm_rect_width(&plane_state->uapi.src) >> 16;
1420
1421         if (plane->id == PLANE_CURSOR) {
1422                 /*
1423                  * FIXME the formula gives values that are
1424                  * too big for the cursor FIFO, and hence we
1425                  * would never be able to use cursors. For
1426                  * now just hardcode the watermark.
1427                  */
1428                 wm = 63;
1429         } else {
1430                 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1431                                     dev_priv->display.wm.pri_latency[level] * 10);
1432         }
1433
1434         return min_t(unsigned int, wm, USHRT_MAX);
1435 }
1436
1437 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1438 {
1439         return (active_planes & (BIT(PLANE_SPRITE0) |
1440                                  BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1441 }
1442
1443 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1444 {
1445         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1446         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1447         const struct g4x_pipe_wm *raw =
1448                 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1449         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1450         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1451         int num_active_planes = hweight8(active_planes);
1452         const int fifo_size = 511;
1453         int fifo_extra, fifo_left = fifo_size;
1454         int sprite0_fifo_extra = 0;
1455         unsigned int total_rate;
1456         enum plane_id plane_id;
1457
1458         /*
1459          * When enabling sprite0 after sprite1 has already been enabled
1460          * we tend to get an underrun unless sprite0 already has some
1461          * FIFO space allcoated. Hence we always allocate at least one
1462          * cacheline for sprite0 whenever sprite1 is enabled.
1463          *
1464          * All other plane enable sequences appear immune to this problem.
1465          */
1466         if (vlv_need_sprite0_fifo_workaround(active_planes))
1467                 sprite0_fifo_extra = 1;
1468
1469         total_rate = raw->plane[PLANE_PRIMARY] +
1470                 raw->plane[PLANE_SPRITE0] +
1471                 raw->plane[PLANE_SPRITE1] +
1472                 sprite0_fifo_extra;
1473
1474         if (total_rate > fifo_size)
1475                 return -EINVAL;
1476
1477         if (total_rate == 0)
1478                 total_rate = 1;
1479
1480         for_each_plane_id_on_crtc(crtc, plane_id) {
1481                 unsigned int rate;
1482
1483                 if ((active_planes & BIT(plane_id)) == 0) {
1484                         fifo_state->plane[plane_id] = 0;
1485                         continue;
1486                 }
1487
1488                 rate = raw->plane[plane_id];
1489                 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1490                 fifo_left -= fifo_state->plane[plane_id];
1491         }
1492
1493         fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1494         fifo_left -= sprite0_fifo_extra;
1495
1496         fifo_state->plane[PLANE_CURSOR] = 63;
1497
1498         fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1499
1500         /* spread the remainder evenly */
1501         for_each_plane_id_on_crtc(crtc, plane_id) {
1502                 int plane_extra;
1503
1504                 if (fifo_left == 0)
1505                         break;
1506
1507                 if ((active_planes & BIT(plane_id)) == 0)
1508                         continue;
1509
1510                 plane_extra = min(fifo_extra, fifo_left);
1511                 fifo_state->plane[plane_id] += plane_extra;
1512                 fifo_left -= plane_extra;
1513         }
1514
1515         drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1516
1517         /* give it all to the first plane if none are active */
1518         if (active_planes == 0) {
1519                 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1520                 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1521         }
1522
1523         return 0;
1524 }
1525
1526 /* mark all levels starting from 'level' as invalid */
1527 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1528                                struct vlv_wm_state *wm_state, int level)
1529 {
1530         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1531
1532         for (; level < dev_priv->display.wm.num_levels; level++) {
1533                 enum plane_id plane_id;
1534
1535                 for_each_plane_id_on_crtc(crtc, plane_id)
1536                         wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1537
1538                 wm_state->sr[level].cursor = USHRT_MAX;
1539                 wm_state->sr[level].plane = USHRT_MAX;
1540         }
1541 }
1542
1543 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1544 {
1545         if (wm > fifo_size)
1546                 return USHRT_MAX;
1547         else
1548                 return fifo_size - wm;
1549 }
1550
1551 /*
1552  * Starting from 'level' set all higher
1553  * levels to 'value' in the "raw" watermarks.
1554  */
1555 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1556                                  int level, enum plane_id plane_id, u16 value)
1557 {
1558         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1559         bool dirty = false;
1560
1561         for (; level < dev_priv->display.wm.num_levels; level++) {
1562                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1563
1564                 dirty |= raw->plane[plane_id] != value;
1565                 raw->plane[plane_id] = value;
1566         }
1567
1568         return dirty;
1569 }
1570
1571 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1572                                      const struct intel_plane_state *plane_state)
1573 {
1574         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1575         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1576         enum plane_id plane_id = plane->id;
1577         int level;
1578         bool dirty = false;
1579
1580         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1581                 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1582                 goto out;
1583         }
1584
1585         for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
1586                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1587                 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1588                 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1589
1590                 if (wm > max_wm)
1591                         break;
1592
1593                 dirty |= raw->plane[plane_id] != wm;
1594                 raw->plane[plane_id] = wm;
1595         }
1596
1597         /* mark all higher levels as invalid */
1598         dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1599
1600 out:
1601         if (dirty)
1602                 drm_dbg_kms(&dev_priv->drm,
1603                             "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1604                             plane->base.name,
1605                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1606                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1607                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1608
1609         return dirty;
1610 }
1611
1612 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1613                                       enum plane_id plane_id, int level)
1614 {
1615         const struct g4x_pipe_wm *raw =
1616                 &crtc_state->wm.vlv.raw[level];
1617         const struct vlv_fifo_state *fifo_state =
1618                 &crtc_state->wm.vlv.fifo_state;
1619
1620         return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1621 }
1622
1623 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1624 {
1625         return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1626                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1627                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1628                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1629 }
1630
1631 static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1632 {
1633         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1634         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1635         struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1636         const struct vlv_fifo_state *fifo_state =
1637                 &crtc_state->wm.vlv.fifo_state;
1638         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1639         int num_active_planes = hweight8(active_planes);
1640         enum plane_id plane_id;
1641         int level;
1642
1643         /* initially allow all levels */
1644         wm_state->num_levels = dev_priv->display.wm.num_levels;
1645         /*
1646          * Note that enabling cxsr with no primary/sprite planes
1647          * enabled can wedge the pipe. Hence we only allow cxsr
1648          * with exactly one enabled primary/sprite plane.
1649          */
1650         wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1651
1652         for (level = 0; level < wm_state->num_levels; level++) {
1653                 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1654                 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1655
1656                 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1657                         break;
1658
1659                 for_each_plane_id_on_crtc(crtc, plane_id) {
1660                         wm_state->wm[level].plane[plane_id] =
1661                                 vlv_invert_wm_value(raw->plane[plane_id],
1662                                                     fifo_state->plane[plane_id]);
1663                 }
1664
1665                 wm_state->sr[level].plane =
1666                         vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1667                                                  raw->plane[PLANE_SPRITE0],
1668                                                  raw->plane[PLANE_SPRITE1]),
1669                                             sr_fifo_size);
1670
1671                 wm_state->sr[level].cursor =
1672                         vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1673                                             63);
1674         }
1675
1676         if (level == 0)
1677                 return -EINVAL;
1678
1679         /* limit to only levels we can actually handle */
1680         wm_state->num_levels = level;
1681
1682         /* invalidate the higher levels */
1683         vlv_invalidate_wms(crtc, wm_state, level);
1684
1685         return 0;
1686 }
1687
1688 static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1689                                struct intel_crtc *crtc)
1690 {
1691         struct intel_crtc_state *crtc_state =
1692                 intel_atomic_get_new_crtc_state(state, crtc);
1693         const struct intel_plane_state *old_plane_state;
1694         const struct intel_plane_state *new_plane_state;
1695         struct intel_plane *plane;
1696         unsigned int dirty = 0;
1697         int i;
1698
1699         for_each_oldnew_intel_plane_in_state(state, plane,
1700                                              old_plane_state,
1701                                              new_plane_state, i) {
1702                 if (new_plane_state->hw.crtc != &crtc->base &&
1703                     old_plane_state->hw.crtc != &crtc->base)
1704                         continue;
1705
1706                 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1707                         dirty |= BIT(plane->id);
1708         }
1709
1710         /*
1711          * DSPARB registers may have been reset due to the
1712          * power well being turned off. Make sure we restore
1713          * them to a consistent state even if no primary/sprite
1714          * planes are initially active. We also force a FIFO
1715          * recomputation so that we are sure to sanitize the
1716          * FIFO setting we took over from the BIOS even if there
1717          * are no active planes on the crtc.
1718          */
1719         if (intel_crtc_needs_modeset(crtc_state))
1720                 dirty = ~0;
1721
1722         if (!dirty)
1723                 return 0;
1724
1725         /* cursor changes don't warrant a FIFO recompute */
1726         if (dirty & ~BIT(PLANE_CURSOR)) {
1727                 const struct intel_crtc_state *old_crtc_state =
1728                         intel_atomic_get_old_crtc_state(state, crtc);
1729                 const struct vlv_fifo_state *old_fifo_state =
1730                         &old_crtc_state->wm.vlv.fifo_state;
1731                 const struct vlv_fifo_state *new_fifo_state =
1732                         &crtc_state->wm.vlv.fifo_state;
1733                 int ret;
1734
1735                 ret = vlv_compute_fifo(crtc_state);
1736                 if (ret)
1737                         return ret;
1738
1739                 if (intel_crtc_needs_modeset(crtc_state) ||
1740                     memcmp(old_fifo_state, new_fifo_state,
1741                            sizeof(*new_fifo_state)) != 0)
1742                         crtc_state->fifo_changed = true;
1743         }
1744
1745         return _vlv_compute_pipe_wm(crtc_state);
1746 }
1747
1748 #define VLV_FIFO(plane, value) \
1749         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1750
1751 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1752                                    struct intel_crtc *crtc)
1753 {
1754         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1755         struct intel_uncore *uncore = &dev_priv->uncore;
1756         const struct intel_crtc_state *crtc_state =
1757                 intel_atomic_get_new_crtc_state(state, crtc);
1758         const struct vlv_fifo_state *fifo_state =
1759                 &crtc_state->wm.vlv.fifo_state;
1760         int sprite0_start, sprite1_start, fifo_size;
1761         u32 dsparb, dsparb2, dsparb3;
1762
1763         if (!crtc_state->fifo_changed)
1764                 return;
1765
1766         sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1767         sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1768         fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1769
1770         drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1771         drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
1772
1773         trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1774
1775         /*
1776          * uncore.lock serves a double purpose here. It allows us to
1777          * use the less expensive I915_{READ,WRITE}_FW() functions, and
1778          * it protects the DSPARB registers from getting clobbered by
1779          * parallel updates from multiple pipes.
1780          *
1781          * intel_pipe_update_start() has already disabled interrupts
1782          * for us, so a plain spin_lock() is sufficient here.
1783          */
1784         spin_lock(&uncore->lock);
1785
1786         switch (crtc->pipe) {
1787         case PIPE_A:
1788                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
1789                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1790
1791                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1792                             VLV_FIFO(SPRITEB, 0xff));
1793                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1794                            VLV_FIFO(SPRITEB, sprite1_start));
1795
1796                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1797                              VLV_FIFO(SPRITEB_HI, 0x1));
1798                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1799                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1800
1801                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
1802                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1803                 break;
1804         case PIPE_B:
1805                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
1806                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1807
1808                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1809                             VLV_FIFO(SPRITED, 0xff));
1810                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1811                            VLV_FIFO(SPRITED, sprite1_start));
1812
1813                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
1814                              VLV_FIFO(SPRITED_HI, 0xff));
1815                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
1816                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
1817
1818                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
1819                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1820                 break;
1821         case PIPE_C:
1822                 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
1823                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1824
1825                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
1826                              VLV_FIFO(SPRITEF, 0xff));
1827                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
1828                             VLV_FIFO(SPRITEF, sprite1_start));
1829
1830                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
1831                              VLV_FIFO(SPRITEF_HI, 0xff));
1832                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
1833                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
1834
1835                 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
1836                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1837                 break;
1838         default:
1839                 break;
1840         }
1841
1842         intel_uncore_posting_read_fw(uncore, DSPARB);
1843
1844         spin_unlock(&uncore->lock);
1845 }
1846
1847 #undef VLV_FIFO
1848
1849 static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
1850                                        struct intel_crtc *crtc)
1851 {
1852         struct intel_crtc_state *new_crtc_state =
1853                 intel_atomic_get_new_crtc_state(state, crtc);
1854         const struct intel_crtc_state *old_crtc_state =
1855                 intel_atomic_get_old_crtc_state(state, crtc);
1856         struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
1857         const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
1858         const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
1859         int level;
1860
1861         if (!new_crtc_state->hw.active ||
1862             intel_crtc_needs_modeset(new_crtc_state)) {
1863                 *intermediate = *optimal;
1864
1865                 intermediate->cxsr = false;
1866                 goto out;
1867         }
1868
1869         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
1870         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1871                 !new_crtc_state->disable_cxsr;
1872
1873         for (level = 0; level < intermediate->num_levels; level++) {
1874                 enum plane_id plane_id;
1875
1876                 for_each_plane_id_on_crtc(crtc, plane_id) {
1877                         intermediate->wm[level].plane[plane_id] =
1878                                 min(optimal->wm[level].plane[plane_id],
1879                                     active->wm[level].plane[plane_id]);
1880                 }
1881
1882                 intermediate->sr[level].plane = min(optimal->sr[level].plane,
1883                                                     active->sr[level].plane);
1884                 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
1885                                                      active->sr[level].cursor);
1886         }
1887
1888         vlv_invalidate_wms(crtc, intermediate, level);
1889
1890 out:
1891         /*
1892          * If our intermediate WM are identical to the final WM, then we can
1893          * omit the post-vblank programming; only update if it's different.
1894          */
1895         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1896                 new_crtc_state->wm.need_postvbl_update = true;
1897
1898         return 0;
1899 }
1900
1901 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
1902                          struct vlv_wm_values *wm)
1903 {
1904         struct intel_crtc *crtc;
1905         int num_active_pipes = 0;
1906
1907         wm->level = dev_priv->display.wm.num_levels - 1;
1908         wm->cxsr = true;
1909
1910         for_each_intel_crtc(&dev_priv->drm, crtc) {
1911                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
1912
1913                 if (!crtc->active)
1914                         continue;
1915
1916                 if (!wm_state->cxsr)
1917                         wm->cxsr = false;
1918
1919                 num_active_pipes++;
1920                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
1921         }
1922
1923         if (num_active_pipes != 1)
1924                 wm->cxsr = false;
1925
1926         if (num_active_pipes > 1)
1927                 wm->level = VLV_WM_LEVEL_PM2;
1928
1929         for_each_intel_crtc(&dev_priv->drm, crtc) {
1930                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
1931                 enum pipe pipe = crtc->pipe;
1932
1933                 wm->pipe[pipe] = wm_state->wm[wm->level];
1934                 if (crtc->active && wm->cxsr)
1935                         wm->sr = wm_state->sr[wm->level];
1936
1937                 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
1938                 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
1939                 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
1940                 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
1941         }
1942 }
1943
1944 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
1945 {
1946         struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;
1947         struct vlv_wm_values new_wm = {};
1948
1949         vlv_merge_wm(dev_priv, &new_wm);
1950
1951         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1952                 return;
1953
1954         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
1955                 chv_set_memory_dvfs(dev_priv, false);
1956
1957         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
1958                 chv_set_memory_pm5(dev_priv, false);
1959
1960         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1961                 _intel_set_memory_cxsr(dev_priv, false);
1962
1963         vlv_write_wm_values(dev_priv, &new_wm);
1964
1965         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1966                 _intel_set_memory_cxsr(dev_priv, true);
1967
1968         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
1969                 chv_set_memory_pm5(dev_priv, true);
1970
1971         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
1972                 chv_set_memory_dvfs(dev_priv, true);
1973
1974         *old_wm = new_wm;
1975 }
1976
1977 static void vlv_initial_watermarks(struct intel_atomic_state *state,
1978                                    struct intel_crtc *crtc)
1979 {
1980         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1981         const struct intel_crtc_state *crtc_state =
1982                 intel_atomic_get_new_crtc_state(state, crtc);
1983
1984         mutex_lock(&dev_priv->display.wm.wm_mutex);
1985         crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
1986         vlv_program_watermarks(dev_priv);
1987         mutex_unlock(&dev_priv->display.wm.wm_mutex);
1988 }
1989
1990 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
1991                                     struct intel_crtc *crtc)
1992 {
1993         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1994         const struct intel_crtc_state *crtc_state =
1995                 intel_atomic_get_new_crtc_state(state, crtc);
1996
1997         if (!crtc_state->wm.need_postvbl_update)
1998                 return;
1999
2000         mutex_lock(&dev_priv->display.wm.wm_mutex);
2001         crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2002         vlv_program_watermarks(dev_priv);
2003         mutex_unlock(&dev_priv->display.wm.wm_mutex);
2004 }
2005
2006 static void i965_update_wm(struct drm_i915_private *dev_priv)
2007 {
2008         struct intel_crtc *crtc;
2009         int srwm = 1;
2010         int cursor_sr = 16;
2011         bool cxsr_enabled;
2012
2013         /* Calc sr entries for one plane configs */
2014         crtc = single_enabled_crtc(dev_priv);
2015         if (crtc) {
2016                 /* self-refresh has much higher latency */
2017                 static const int sr_latency_ns = 12000;
2018                 const struct drm_display_mode *pipe_mode =
2019                         &crtc->config->hw.pipe_mode;
2020                 const struct drm_framebuffer *fb =
2021                         crtc->base.primary->state->fb;
2022                 int pixel_rate = crtc->config->pixel_rate;
2023                 int htotal = pipe_mode->crtc_htotal;
2024                 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2025                 int cpp = fb->format->cpp[0];
2026                 int entries;
2027
2028                 entries = intel_wm_method2(pixel_rate, htotal,
2029                                            width, cpp, sr_latency_ns / 100);
2030                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2031                 srwm = I965_FIFO_SIZE - entries;
2032                 if (srwm < 0)
2033                         srwm = 1;
2034                 srwm &= 0x1ff;
2035                 drm_dbg_kms(&dev_priv->drm,
2036                             "self-refresh entries: %d, wm: %d\n",
2037                             entries, srwm);
2038
2039                 entries = intel_wm_method2(pixel_rate, htotal,
2040                                            crtc->base.cursor->state->crtc_w, 4,
2041                                            sr_latency_ns / 100);
2042                 entries = DIV_ROUND_UP(entries,
2043                                        i965_cursor_wm_info.cacheline_size) +
2044                         i965_cursor_wm_info.guard_size;
2045
2046                 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2047                 if (cursor_sr > i965_cursor_wm_info.max_wm)
2048                         cursor_sr = i965_cursor_wm_info.max_wm;
2049
2050                 drm_dbg_kms(&dev_priv->drm,
2051                             "self-refresh watermark: display plane %d "
2052                             "cursor %d\n", srwm, cursor_sr);
2053
2054                 cxsr_enabled = true;
2055         } else {
2056                 cxsr_enabled = false;
2057                 /* Turn off self refresh if both pipes are enabled */
2058                 intel_set_memory_cxsr(dev_priv, false);
2059         }
2060
2061         drm_dbg_kms(&dev_priv->drm,
2062                     "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2063                     srwm);
2064
2065         /* 965 has limitations... */
2066         intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2067                    FW_WM(8, CURSORB) |
2068                    FW_WM(8, PLANEB) |
2069                    FW_WM(8, PLANEA));
2070         intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2071                    FW_WM(8, PLANEC_OLD));
2072         /* update cursor SR watermark */
2073         intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2074
2075         if (cxsr_enabled)
2076                 intel_set_memory_cxsr(dev_priv, true);
2077 }
2078
2079 #undef FW_WM
2080
2081 static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2082                                                enum i9xx_plane_id i9xx_plane)
2083 {
2084         struct intel_plane *plane;
2085
2086         for_each_intel_plane(&i915->drm, plane) {
2087                 if (plane->id == PLANE_PRIMARY &&
2088                     plane->i9xx_plane == i9xx_plane)
2089                         return intel_crtc_for_pipe(i915, plane->pipe);
2090         }
2091
2092         return NULL;
2093 }
2094
2095 static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2096 {
2097         const struct intel_watermark_params *wm_info;
2098         u32 fwater_lo;
2099         u32 fwater_hi;
2100         int cwm, srwm = 1;
2101         int fifo_size;
2102         int planea_wm, planeb_wm;
2103         struct intel_crtc *crtc;
2104
2105         if (IS_I945GM(dev_priv))
2106                 wm_info = &i945_wm_info;
2107         else if (DISPLAY_VER(dev_priv) != 2)
2108                 wm_info = &i915_wm_info;
2109         else
2110                 wm_info = &i830_a_wm_info;
2111
2112         if (DISPLAY_VER(dev_priv) == 2)
2113                 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2114         else
2115                 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2116         crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2117         if (intel_crtc_active(crtc)) {
2118                 const struct drm_framebuffer *fb =
2119                         crtc->base.primary->state->fb;
2120                 int cpp;
2121
2122                 if (DISPLAY_VER(dev_priv) == 2)
2123                         cpp = 4;
2124                 else
2125                         cpp = fb->format->cpp[0];
2126
2127                 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2128                                                wm_info, fifo_size, cpp,
2129                                                pessimal_latency_ns);
2130         } else {
2131                 planea_wm = fifo_size - wm_info->guard_size;
2132                 if (planea_wm > (long)wm_info->max_wm)
2133                         planea_wm = wm_info->max_wm;
2134         }
2135
2136         if (DISPLAY_VER(dev_priv) == 2)
2137                 wm_info = &i830_bc_wm_info;
2138
2139         if (DISPLAY_VER(dev_priv) == 2)
2140                 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2141         else
2142                 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2143         crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2144         if (intel_crtc_active(crtc)) {
2145                 const struct drm_framebuffer *fb =
2146                         crtc->base.primary->state->fb;
2147                 int cpp;
2148
2149                 if (DISPLAY_VER(dev_priv) == 2)
2150                         cpp = 4;
2151                 else
2152                         cpp = fb->format->cpp[0];
2153
2154                 planeb_wm = intel_calculate_wm(crtc->config->pixel_rate,
2155                                                wm_info, fifo_size, cpp,
2156                                                pessimal_latency_ns);
2157         } else {
2158                 planeb_wm = fifo_size - wm_info->guard_size;
2159                 if (planeb_wm > (long)wm_info->max_wm)
2160                         planeb_wm = wm_info->max_wm;
2161         }
2162
2163         drm_dbg_kms(&dev_priv->drm,
2164                     "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2165
2166         crtc = single_enabled_crtc(dev_priv);
2167         if (IS_I915GM(dev_priv) && crtc) {
2168                 struct drm_i915_gem_object *obj;
2169
2170                 obj = intel_fb_obj(crtc->base.primary->state->fb);
2171
2172                 /* self-refresh seems busted with untiled */
2173                 if (!i915_gem_object_is_tiled(obj))
2174                         crtc = NULL;
2175         }
2176
2177         /*
2178          * Overlay gets an aggressive default since video jitter is bad.
2179          */
2180         cwm = 2;
2181
2182         /* Play safe and disable self-refresh before adjusting watermarks. */
2183         intel_set_memory_cxsr(dev_priv, false);
2184
2185         /* Calc sr entries for one plane configs */
2186         if (HAS_FW_BLC(dev_priv) && crtc) {
2187                 /* self-refresh has much higher latency */
2188                 static const int sr_latency_ns = 6000;
2189                 const struct drm_display_mode *pipe_mode =
2190                         &crtc->config->hw.pipe_mode;
2191                 const struct drm_framebuffer *fb =
2192                         crtc->base.primary->state->fb;
2193                 int pixel_rate = crtc->config->pixel_rate;
2194                 int htotal = pipe_mode->crtc_htotal;
2195                 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2196                 int cpp;
2197                 int entries;
2198
2199                 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2200                         cpp = 4;
2201                 else
2202                         cpp = fb->format->cpp[0];
2203
2204                 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2205                                            sr_latency_ns / 100);
2206                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2207                 drm_dbg_kms(&dev_priv->drm,
2208                             "self-refresh entries: %d\n", entries);
2209                 srwm = wm_info->fifo_size - entries;
2210                 if (srwm < 0)
2211                         srwm = 1;
2212
2213                 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2214                         intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2215                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2216                 else
2217                         intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2218         }
2219
2220         drm_dbg_kms(&dev_priv->drm,
2221                     "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2222                      planea_wm, planeb_wm, cwm, srwm);
2223
2224         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2225         fwater_hi = (cwm & 0x1f);
2226
2227         /* Set request length to 8 cachelines per fetch */
2228         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2229         fwater_hi = fwater_hi | (1 << 8);
2230
2231         intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2232         intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2233
2234         if (crtc)
2235                 intel_set_memory_cxsr(dev_priv, true);
2236 }
2237
2238 static void i845_update_wm(struct drm_i915_private *dev_priv)
2239 {
2240         struct intel_crtc *crtc;
2241         u32 fwater_lo;
2242         int planea_wm;
2243
2244         crtc = single_enabled_crtc(dev_priv);
2245         if (crtc == NULL)
2246                 return;
2247
2248         planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2249                                        &i845_wm_info,
2250                                        i845_get_fifo_size(dev_priv, PLANE_A),
2251                                        4, pessimal_latency_ns);
2252         fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2253         fwater_lo |= (3<<8) | planea_wm;
2254
2255         drm_dbg_kms(&dev_priv->drm,
2256                     "Setting FIFO watermarks - A: %d\n", planea_wm);
2257
2258         intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2259 }
2260
2261 /* latency must be in 0.1us units. */
2262 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2263                                    unsigned int cpp,
2264                                    unsigned int latency)
2265 {
2266         unsigned int ret;
2267
2268         ret = intel_wm_method1(pixel_rate, cpp, latency);
2269         ret = DIV_ROUND_UP(ret, 64) + 2;
2270
2271         return ret;
2272 }
2273
2274 /* latency must be in 0.1us units. */
2275 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2276                                    unsigned int htotal,
2277                                    unsigned int width,
2278                                    unsigned int cpp,
2279                                    unsigned int latency)
2280 {
2281         unsigned int ret;
2282
2283         ret = intel_wm_method2(pixel_rate, htotal,
2284                                width, cpp, latency);
2285         ret = DIV_ROUND_UP(ret, 64) + 2;
2286
2287         return ret;
2288 }
2289
2290 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2291 {
2292         /*
2293          * Neither of these should be possible since this function shouldn't be
2294          * called if the CRTC is off or the plane is invisible.  But let's be
2295          * extra paranoid to avoid a potential divide-by-zero if we screw up
2296          * elsewhere in the driver.
2297          */
2298         if (WARN_ON(!cpp))
2299                 return 0;
2300         if (WARN_ON(!horiz_pixels))
2301                 return 0;
2302
2303         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2304 }
2305
2306 struct ilk_wm_maximums {
2307         u16 pri;
2308         u16 spr;
2309         u16 cur;
2310         u16 fbc;
2311 };
2312
2313 /*
2314  * For both WM_PIPE and WM_LP.
2315  * mem_value must be in 0.1us units.
2316  */
2317 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2318                               const struct intel_plane_state *plane_state,
2319                               u32 mem_value, bool is_lp)
2320 {
2321         u32 method1, method2;
2322         int cpp;
2323
2324         if (mem_value == 0)
2325                 return U32_MAX;
2326
2327         if (!intel_wm_plane_visible(crtc_state, plane_state))
2328                 return 0;
2329
2330         cpp = plane_state->hw.fb->format->cpp[0];
2331
2332         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2333
2334         if (!is_lp)
2335                 return method1;
2336
2337         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2338                                  crtc_state->hw.pipe_mode.crtc_htotal,
2339                                  drm_rect_width(&plane_state->uapi.src) >> 16,
2340                                  cpp, mem_value);
2341
2342         return min(method1, method2);
2343 }
2344
2345 /*
2346  * For both WM_PIPE and WM_LP.
2347  * mem_value must be in 0.1us units.
2348  */
2349 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2350                               const struct intel_plane_state *plane_state,
2351                               u32 mem_value)
2352 {
2353         u32 method1, method2;
2354         int cpp;
2355
2356         if (mem_value == 0)
2357                 return U32_MAX;
2358
2359         if (!intel_wm_plane_visible(crtc_state, plane_state))
2360                 return 0;
2361
2362         cpp = plane_state->hw.fb->format->cpp[0];
2363
2364         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2365         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2366                                  crtc_state->hw.pipe_mode.crtc_htotal,
2367                                  drm_rect_width(&plane_state->uapi.src) >> 16,
2368                                  cpp, mem_value);
2369         return min(method1, method2);
2370 }
2371
2372 /*
2373  * For both WM_PIPE and WM_LP.
2374  * mem_value must be in 0.1us units.
2375  */
2376 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2377                               const struct intel_plane_state *plane_state,
2378                               u32 mem_value)
2379 {
2380         int cpp;
2381
2382         if (mem_value == 0)
2383                 return U32_MAX;
2384
2385         if (!intel_wm_plane_visible(crtc_state, plane_state))
2386                 return 0;
2387
2388         cpp = plane_state->hw.fb->format->cpp[0];
2389
2390         return ilk_wm_method2(crtc_state->pixel_rate,
2391                               crtc_state->hw.pipe_mode.crtc_htotal,
2392                               drm_rect_width(&plane_state->uapi.src) >> 16,
2393                               cpp, mem_value);
2394 }
2395
2396 /* Only for WM_LP. */
2397 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2398                               const struct intel_plane_state *plane_state,
2399                               u32 pri_val)
2400 {
2401         int cpp;
2402
2403         if (!intel_wm_plane_visible(crtc_state, plane_state))
2404                 return 0;
2405
2406         cpp = plane_state->hw.fb->format->cpp[0];
2407
2408         return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2409                           cpp);
2410 }
2411
2412 static unsigned int
2413 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2414 {
2415         if (DISPLAY_VER(dev_priv) >= 8)
2416                 return 3072;
2417         else if (DISPLAY_VER(dev_priv) >= 7)
2418                 return 768;
2419         else
2420                 return 512;
2421 }
2422
2423 static unsigned int
2424 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2425                      int level, bool is_sprite)
2426 {
2427         if (DISPLAY_VER(dev_priv) >= 8)
2428                 /* BDW primary/sprite plane watermarks */
2429                 return level == 0 ? 255 : 2047;
2430         else if (DISPLAY_VER(dev_priv) >= 7)
2431                 /* IVB/HSW primary/sprite plane watermarks */
2432                 return level == 0 ? 127 : 1023;
2433         else if (!is_sprite)
2434                 /* ILK/SNB primary plane watermarks */
2435                 return level == 0 ? 127 : 511;
2436         else
2437                 /* ILK/SNB sprite plane watermarks */
2438                 return level == 0 ? 63 : 255;
2439 }
2440
2441 static unsigned int
2442 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2443 {
2444         if (DISPLAY_VER(dev_priv) >= 7)
2445                 return level == 0 ? 63 : 255;
2446         else
2447                 return level == 0 ? 31 : 63;
2448 }
2449
2450 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2451 {
2452         if (DISPLAY_VER(dev_priv) >= 8)
2453                 return 31;
2454         else
2455                 return 15;
2456 }
2457
2458 /* Calculate the maximum primary/sprite plane watermark */
2459 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2460                                      int level,
2461                                      const struct intel_wm_config *config,
2462                                      enum intel_ddb_partitioning ddb_partitioning,
2463                                      bool is_sprite)
2464 {
2465         unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2466
2467         /* if sprites aren't enabled, sprites get nothing */
2468         if (is_sprite && !config->sprites_enabled)
2469                 return 0;
2470
2471         /* HSW allows LP1+ watermarks even with multiple pipes */
2472         if (level == 0 || config->num_pipes_active > 1) {
2473                 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2474
2475                 /*
2476                  * For some reason the non self refresh
2477                  * FIFO size is only half of the self
2478                  * refresh FIFO size on ILK/SNB.
2479                  */
2480                 if (DISPLAY_VER(dev_priv) <= 6)
2481                         fifo_size /= 2;
2482         }
2483
2484         if (config->sprites_enabled) {
2485                 /* level 0 is always calculated with 1:1 split */
2486                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2487                         if (is_sprite)
2488                                 fifo_size *= 5;
2489                         fifo_size /= 6;
2490                 } else {
2491                         fifo_size /= 2;
2492                 }
2493         }
2494
2495         /* clamp to max that the registers can hold */
2496         return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2497 }
2498
2499 /* Calculate the maximum cursor plane watermark */
2500 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2501                                       int level,
2502                                       const struct intel_wm_config *config)
2503 {
2504         /* HSW LP1+ watermarks w/ multiple pipes */
2505         if (level > 0 && config->num_pipes_active > 1)
2506                 return 64;
2507
2508         /* otherwise just report max that registers can hold */
2509         return ilk_cursor_wm_reg_max(dev_priv, level);
2510 }
2511
2512 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2513                                     int level,
2514                                     const struct intel_wm_config *config,
2515                                     enum intel_ddb_partitioning ddb_partitioning,
2516                                     struct ilk_wm_maximums *max)
2517 {
2518         max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2519         max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2520         max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2521         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2522 }
2523
2524 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2525                                         int level,
2526                                         struct ilk_wm_maximums *max)
2527 {
2528         max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2529         max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2530         max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2531         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2532 }
2533
2534 static bool ilk_validate_wm_level(int level,
2535                                   const struct ilk_wm_maximums *max,
2536                                   struct intel_wm_level *result)
2537 {
2538         bool ret;
2539
2540         /* already determined to be invalid? */
2541         if (!result->enable)
2542                 return false;
2543
2544         result->enable = result->pri_val <= max->pri &&
2545                          result->spr_val <= max->spr &&
2546                          result->cur_val <= max->cur;
2547
2548         ret = result->enable;
2549
2550         /*
2551          * HACK until we can pre-compute everything,
2552          * and thus fail gracefully if LP0 watermarks
2553          * are exceeded...
2554          */
2555         if (level == 0 && !result->enable) {
2556                 if (result->pri_val > max->pri)
2557                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2558                                       level, result->pri_val, max->pri);
2559                 if (result->spr_val > max->spr)
2560                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2561                                       level, result->spr_val, max->spr);
2562                 if (result->cur_val > max->cur)
2563                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2564                                       level, result->cur_val, max->cur);
2565
2566                 result->pri_val = min_t(u32, result->pri_val, max->pri);
2567                 result->spr_val = min_t(u32, result->spr_val, max->spr);
2568                 result->cur_val = min_t(u32, result->cur_val, max->cur);
2569                 result->enable = true;
2570         }
2571
2572         return ret;
2573 }
2574
2575 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2576                                  const struct intel_crtc *crtc,
2577                                  int level,
2578                                  struct intel_crtc_state *crtc_state,
2579                                  const struct intel_plane_state *pristate,
2580                                  const struct intel_plane_state *sprstate,
2581                                  const struct intel_plane_state *curstate,
2582                                  struct intel_wm_level *result)
2583 {
2584         u16 pri_latency = dev_priv->display.wm.pri_latency[level];
2585         u16 spr_latency = dev_priv->display.wm.spr_latency[level];
2586         u16 cur_latency = dev_priv->display.wm.cur_latency[level];
2587
2588         /* WM1+ latency values stored in 0.5us units */
2589         if (level > 0) {
2590                 pri_latency *= 5;
2591                 spr_latency *= 5;
2592                 cur_latency *= 5;
2593         }
2594
2595         if (pristate) {
2596                 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2597                                                      pri_latency, level);
2598                 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2599         }
2600
2601         if (sprstate)
2602                 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2603
2604         if (curstate)
2605                 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2606
2607         result->enable = true;
2608 }
2609
2610 static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2611 {
2612         u64 sskpd;
2613
2614         i915->display.wm.num_levels = 5;
2615
2616         sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);
2617
2618         wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2619         if (wm[0] == 0)
2620                 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2621         wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2622         wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2623         wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2624         wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2625 }
2626
2627 static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2628 {
2629         u32 sskpd;
2630
2631         i915->display.wm.num_levels = 4;
2632
2633         sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);
2634
2635         wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2636         wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2637         wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2638         wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2639 }
2640
2641 static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2642 {
2643         u32 mltr;
2644
2645         i915->display.wm.num_levels = 3;
2646
2647         mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);
2648
2649         /* ILK primary LP0 latency is 700 ns */
2650         wm[0] = 7;
2651         wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2652         wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2653 }
2654
2655 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2656                                        u16 wm[5])
2657 {
2658         /* ILK sprite LP0 latency is 1300 ns */
2659         if (DISPLAY_VER(dev_priv) == 5)
2660                 wm[0] = 13;
2661 }
2662
2663 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2664                                        u16 wm[5])
2665 {
2666         /* ILK cursor LP0 latency is 1300 ns */
2667         if (DISPLAY_VER(dev_priv) == 5)
2668                 wm[0] = 13;
2669 }
2670
2671 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2672                                     u16 wm[5], u16 min)
2673 {
2674         int level;
2675
2676         if (wm[0] >= min)
2677                 return false;
2678
2679         wm[0] = max(wm[0], min);
2680         for (level = 1; level < dev_priv->display.wm.num_levels; level++)
2681                 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2682
2683         return true;
2684 }
2685
2686 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2687 {
2688         bool changed;
2689
2690         /*
2691          * The BIOS provided WM memory latency values are often
2692          * inadequate for high resolution displays. Adjust them.
2693          */
2694         changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);
2695         changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);
2696         changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);
2697
2698         if (!changed)
2699                 return;
2700
2701         drm_dbg_kms(&dev_priv->drm,
2702                     "WM latency values increased to avoid potential underruns\n");
2703         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2704         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2705         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2706 }
2707
2708 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
2709 {
2710         /*
2711          * On some SNB machines (Thinkpad X220 Tablet at least)
2712          * LP3 usage can cause vblank interrupts to be lost.
2713          * The DEIIR bit will go high but it looks like the CPU
2714          * never gets interrupted.
2715          *
2716          * It's not clear whether other interrupt source could
2717          * be affected or if this is somehow limited to vblank
2718          * interrupts only. To play it safe we disable LP3
2719          * watermarks entirely.
2720          */
2721         if (dev_priv->display.wm.pri_latency[3] == 0 &&
2722             dev_priv->display.wm.spr_latency[3] == 0 &&
2723             dev_priv->display.wm.cur_latency[3] == 0)
2724                 return;
2725
2726         dev_priv->display.wm.pri_latency[3] = 0;
2727         dev_priv->display.wm.spr_latency[3] = 0;
2728         dev_priv->display.wm.cur_latency[3] = 0;
2729
2730         drm_dbg_kms(&dev_priv->drm,
2731                     "LP3 watermarks disabled due to potential for lost interrupts\n");
2732         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2733         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2734         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2735 }
2736
2737 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
2738 {
2739         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2740                 hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2741         else if (DISPLAY_VER(dev_priv) >= 6)
2742                 snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2743         else
2744                 ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2745
2746         memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,
2747                sizeof(dev_priv->display.wm.pri_latency));
2748         memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,
2749                sizeof(dev_priv->display.wm.pri_latency));
2750
2751         intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);
2752         intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);
2753
2754         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2755         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2756         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2757
2758         if (DISPLAY_VER(dev_priv) == 6) {
2759                 snb_wm_latency_quirk(dev_priv);
2760                 snb_wm_lp3_irq_quirk(dev_priv);
2761         }
2762 }
2763
2764 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
2765                                  struct intel_pipe_wm *pipe_wm)
2766 {
2767         /* LP0 watermark maximums depend on this pipe alone */
2768         const struct intel_wm_config config = {
2769                 .num_pipes_active = 1,
2770                 .sprites_enabled = pipe_wm->sprites_enabled,
2771                 .sprites_scaled = pipe_wm->sprites_scaled,
2772         };
2773         struct ilk_wm_maximums max;
2774
2775         /* LP0 watermarks always use 1/2 DDB partitioning */
2776         ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
2777
2778         /* At least LP0 must be valid */
2779         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
2780                 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
2781                 return false;
2782         }
2783
2784         return true;
2785 }
2786
2787 /* Compute new watermarks for the pipe */
2788 static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
2789                                struct intel_crtc *crtc)
2790 {
2791         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2792         struct intel_crtc_state *crtc_state =
2793                 intel_atomic_get_new_crtc_state(state, crtc);
2794         struct intel_pipe_wm *pipe_wm;
2795         struct intel_plane *plane;
2796         const struct intel_plane_state *plane_state;
2797         const struct intel_plane_state *pristate = NULL;
2798         const struct intel_plane_state *sprstate = NULL;
2799         const struct intel_plane_state *curstate = NULL;
2800         struct ilk_wm_maximums max;
2801         int level, usable_level;
2802
2803         pipe_wm = &crtc_state->wm.ilk.optimal;
2804
2805         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
2806                 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
2807                         pristate = plane_state;
2808                 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
2809                         sprstate = plane_state;
2810                 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
2811                         curstate = plane_state;
2812         }
2813
2814         pipe_wm->pipe_enabled = crtc_state->hw.active;
2815         pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
2816         pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
2817
2818         usable_level = dev_priv->display.wm.num_levels - 1;
2819
2820         /* ILK/SNB: LP2+ watermarks only w/o sprites */
2821         if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
2822                 usable_level = 1;
2823
2824         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
2825         if (pipe_wm->sprites_scaled)
2826                 usable_level = 0;
2827
2828         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
2829         ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
2830                              pristate, sprstate, curstate, &pipe_wm->wm[0]);
2831
2832         if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
2833                 return -EINVAL;
2834
2835         ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
2836
2837         for (level = 1; level <= usable_level; level++) {
2838                 struct intel_wm_level *wm = &pipe_wm->wm[level];
2839
2840                 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
2841                                      pristate, sprstate, curstate, wm);
2842
2843                 /*
2844                  * Disable any watermark level that exceeds the
2845                  * register maximums since such watermarks are
2846                  * always invalid.
2847                  */
2848                 if (!ilk_validate_wm_level(level, &max, wm)) {
2849                         memset(wm, 0, sizeof(*wm));
2850                         break;
2851                 }
2852         }
2853
2854         return 0;
2855 }
2856
2857 /*
2858  * Build a set of 'intermediate' watermark values that satisfy both the old
2859  * state and the new state.  These can be programmed to the hardware
2860  * immediately.
2861  */
2862 static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
2863                                        struct intel_crtc *crtc)
2864 {
2865         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2866         struct intel_crtc_state *new_crtc_state =
2867                 intel_atomic_get_new_crtc_state(state, crtc);
2868         const struct intel_crtc_state *old_crtc_state =
2869                 intel_atomic_get_old_crtc_state(state, crtc);
2870         struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
2871         const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
2872         int level;
2873
2874         /*
2875          * Start with the final, target watermarks, then combine with the
2876          * currently active watermarks to get values that are safe both before
2877          * and after the vblank.
2878          */
2879         *a = new_crtc_state->wm.ilk.optimal;
2880         if (!new_crtc_state->hw.active ||
2881             intel_crtc_needs_modeset(new_crtc_state) ||
2882             state->skip_intermediate_wm)
2883                 return 0;
2884
2885         a->pipe_enabled |= b->pipe_enabled;
2886         a->sprites_enabled |= b->sprites_enabled;
2887         a->sprites_scaled |= b->sprites_scaled;
2888
2889         for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
2890                 struct intel_wm_level *a_wm = &a->wm[level];
2891                 const struct intel_wm_level *b_wm = &b->wm[level];
2892
2893                 a_wm->enable &= b_wm->enable;
2894                 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
2895                 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
2896                 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
2897                 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
2898         }
2899
2900         /*
2901          * We need to make sure that these merged watermark values are
2902          * actually a valid configuration themselves.  If they're not,
2903          * there's no safe way to transition from the old state to
2904          * the new state, so we need to fail the atomic transaction.
2905          */
2906         if (!ilk_validate_pipe_wm(dev_priv, a))
2907                 return -EINVAL;
2908
2909         /*
2910          * If our intermediate WM are identical to the final WM, then we can
2911          * omit the post-vblank programming; only update if it's different.
2912          */
2913         if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
2914                 new_crtc_state->wm.need_postvbl_update = true;
2915
2916         return 0;
2917 }
2918
2919 /*
2920  * Merge the watermarks from all active pipes for a specific level.
2921  */
2922 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
2923                                int level,
2924                                struct intel_wm_level *ret_wm)
2925 {
2926         const struct intel_crtc *crtc;
2927
2928         ret_wm->enable = true;
2929
2930         for_each_intel_crtc(&dev_priv->drm, crtc) {
2931                 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
2932                 const struct intel_wm_level *wm = &active->wm[level];
2933
2934                 if (!active->pipe_enabled)
2935                         continue;
2936
2937                 /*
2938                  * The watermark values may have been used in the past,
2939                  * so we must maintain them in the registers for some
2940                  * time even if the level is now disabled.
2941                  */
2942                 if (!wm->enable)
2943                         ret_wm->enable = false;
2944
2945                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
2946                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
2947                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
2948                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
2949         }
2950 }
2951
2952 /*
2953  * Merge all low power watermarks for all active pipes.
2954  */
2955 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
2956                          const struct intel_wm_config *config,
2957                          const struct ilk_wm_maximums *max,
2958                          struct intel_pipe_wm *merged)
2959 {
2960         int level, num_levels = dev_priv->display.wm.num_levels;
2961         int last_enabled_level = num_levels - 1;
2962
2963         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
2964         if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
2965             config->num_pipes_active > 1)
2966                 last_enabled_level = 0;
2967
2968         /* ILK: FBC WM must be disabled always */
2969         merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
2970
2971         /* merge each WM1+ level */
2972         for (level = 1; level < num_levels; level++) {
2973                 struct intel_wm_level *wm = &merged->wm[level];
2974
2975                 ilk_merge_wm_level(dev_priv, level, wm);
2976
2977                 if (level > last_enabled_level)
2978                         wm->enable = false;
2979                 else if (!ilk_validate_wm_level(level, max, wm))
2980                         /* make sure all following levels get disabled */
2981                         last_enabled_level = level - 1;
2982
2983                 /*
2984                  * The spec says it is preferred to disable
2985                  * FBC WMs instead of disabling a WM level.
2986                  */
2987                 if (wm->fbc_val > max->fbc) {
2988                         if (wm->enable)
2989                                 merged->fbc_wm_enabled = false;
2990                         wm->fbc_val = 0;
2991                 }
2992         }
2993
2994         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
2995         if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
2996             dev_priv->params.enable_fbc && !merged->fbc_wm_enabled) {
2997                 for (level = 2; level < num_levels; level++) {
2998                         struct intel_wm_level *wm = &merged->wm[level];
2999
3000                         wm->enable = false;
3001                 }
3002         }
3003 }
3004
3005 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3006 {
3007         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3008         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3009 }
3010
3011 /* The value we need to program into the WM_LPx latency field */
3012 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3013                                       int level)
3014 {
3015         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3016                 return 2 * level;
3017         else
3018                 return dev_priv->display.wm.pri_latency[level];
3019 }
3020
3021 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3022                                    const struct intel_pipe_wm *merged,
3023                                    enum intel_ddb_partitioning partitioning,
3024                                    struct ilk_wm_values *results)
3025 {
3026         struct intel_crtc *crtc;
3027         int level, wm_lp;
3028
3029         results->enable_fbc_wm = merged->fbc_wm_enabled;
3030         results->partitioning = partitioning;
3031
3032         /* LP1+ register values */
3033         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3034                 const struct intel_wm_level *r;
3035
3036                 level = ilk_wm_lp_to_level(wm_lp, merged);
3037
3038                 r = &merged->wm[level];
3039
3040                 /*
3041                  * Maintain the watermark values even if the level is
3042                  * disabled. Doing otherwise could cause underruns.
3043                  */
3044                 results->wm_lp[wm_lp - 1] =
3045                         WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3046                         WM_LP_PRIMARY(r->pri_val) |
3047                         WM_LP_CURSOR(r->cur_val);
3048
3049                 if (r->enable)
3050                         results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3051
3052                 if (DISPLAY_VER(dev_priv) >= 8)
3053                         results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3054                 else
3055                         results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3056
3057                 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3058
3059                 /*
3060                  * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3061                  * level is disabled. Doing otherwise could cause underruns.
3062                  */
3063                 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3064                         drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3065                         results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3066                 }
3067         }
3068
3069         /* LP0 register values */
3070         for_each_intel_crtc(&dev_priv->drm, crtc) {
3071                 enum pipe pipe = crtc->pipe;
3072                 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3073                 const struct intel_wm_level *r = &pipe_wm->wm[0];
3074
3075                 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3076                         continue;
3077
3078                 results->wm_pipe[pipe] =
3079                         WM0_PIPE_PRIMARY(r->pri_val) |
3080                         WM0_PIPE_SPRITE(r->spr_val) |
3081                         WM0_PIPE_CURSOR(r->cur_val);
3082         }
3083 }
3084
3085 /*
3086  * Find the result with the highest level enabled. Check for enable_fbc_wm in
3087  * case both are at the same level. Prefer r1 in case they're the same.
3088  */
3089 static struct intel_pipe_wm *
3090 ilk_find_best_result(struct drm_i915_private *dev_priv,
3091                      struct intel_pipe_wm *r1,
3092                      struct intel_pipe_wm *r2)
3093 {
3094         int level, level1 = 0, level2 = 0;
3095
3096         for (level = 1; level < dev_priv->display.wm.num_levels; level++) {
3097                 if (r1->wm[level].enable)
3098                         level1 = level;
3099                 if (r2->wm[level].enable)
3100                         level2 = level;
3101         }
3102
3103         if (level1 == level2) {
3104                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3105                         return r2;
3106                 else
3107                         return r1;
3108         } else if (level1 > level2) {
3109                 return r1;
3110         } else {
3111                 return r2;
3112         }
3113 }
3114
3115 /* dirty bits used to track which watermarks need changes */
3116 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3117 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3118 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3119 #define WM_DIRTY_FBC (1 << 24)
3120 #define WM_DIRTY_DDB (1 << 25)
3121
3122 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3123                                          const struct ilk_wm_values *old,
3124                                          const struct ilk_wm_values *new)
3125 {
3126         unsigned int dirty = 0;
3127         enum pipe pipe;
3128         int wm_lp;
3129
3130         for_each_pipe(dev_priv, pipe) {
3131                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3132                         dirty |= WM_DIRTY_PIPE(pipe);
3133                         /* Must disable LP1+ watermarks too */
3134                         dirty |= WM_DIRTY_LP_ALL;
3135                 }
3136         }
3137
3138         if (old->enable_fbc_wm != new->enable_fbc_wm) {
3139                 dirty |= WM_DIRTY_FBC;
3140                 /* Must disable LP1+ watermarks too */
3141                 dirty |= WM_DIRTY_LP_ALL;
3142         }
3143
3144         if (old->partitioning != new->partitioning) {
3145                 dirty |= WM_DIRTY_DDB;
3146                 /* Must disable LP1+ watermarks too */
3147                 dirty |= WM_DIRTY_LP_ALL;
3148         }
3149
3150         /* LP1+ watermarks already deemed dirty, no need to continue */
3151         if (dirty & WM_DIRTY_LP_ALL)
3152                 return dirty;
3153
3154         /* Find the lowest numbered LP1+ watermark in need of an update... */
3155         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3156                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3157                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3158                         break;
3159         }
3160
3161         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3162         for (; wm_lp <= 3; wm_lp++)
3163                 dirty |= WM_DIRTY_LP(wm_lp);
3164
3165         return dirty;
3166 }
3167
3168 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3169                                unsigned int dirty)
3170 {
3171         struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3172         bool changed = false;
3173
3174         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3175                 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3176                 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3177                 changed = true;
3178         }
3179         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3180                 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3181                 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3182                 changed = true;
3183         }
3184         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3185                 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3186                 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3187                 changed = true;
3188         }
3189
3190         /*
3191          * Don't touch WM_LP_SPRITE_ENABLE here.
3192          * Doing so could cause underruns.
3193          */
3194
3195         return changed;
3196 }
3197
3198 /*
3199  * The spec says we shouldn't write when we don't need, because every write
3200  * causes WMs to be re-evaluated, expending some power.
3201  */
3202 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3203                                 struct ilk_wm_values *results)
3204 {
3205         struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3206         unsigned int dirty;
3207
3208         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3209         if (!dirty)
3210                 return;
3211
3212         _ilk_disable_lp_wm(dev_priv, dirty);
3213
3214         if (dirty & WM_DIRTY_PIPE(PIPE_A))
3215                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3216         if (dirty & WM_DIRTY_PIPE(PIPE_B))
3217                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3218         if (dirty & WM_DIRTY_PIPE(PIPE_C))
3219                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3220
3221         if (dirty & WM_DIRTY_DDB) {
3222                 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3223                         intel_uncore_rmw(&dev_priv->uncore, WM_MISC, WM_MISC_DATA_PARTITION_5_6,
3224                                          results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3225                                          WM_MISC_DATA_PARTITION_5_6);
3226                 else
3227                         intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,
3228                                          results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3229                                          DISP_DATA_PARTITION_5_6);
3230         }
3231
3232         if (dirty & WM_DIRTY_FBC)
3233                 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, DISP_FBC_WM_DIS,
3234                                  results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);
3235
3236         if (dirty & WM_DIRTY_LP(1) &&
3237             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3238                 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3239
3240         if (DISPLAY_VER(dev_priv) >= 7) {
3241                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3242                         intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3243                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3244                         intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3245         }
3246
3247         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3248                 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3249         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3250                 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3251         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3252                 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3253
3254         dev_priv->display.wm.hw = *results;
3255 }
3256
3257 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3258 {
3259         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3260 }
3261
3262 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
3263                                   struct intel_wm_config *config)
3264 {
3265         struct intel_crtc *crtc;
3266
3267         /* Compute the currently _active_ config */
3268         for_each_intel_crtc(&dev_priv->drm, crtc) {
3269                 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3270
3271                 if (!wm->pipe_enabled)
3272                         continue;
3273
3274                 config->sprites_enabled |= wm->sprites_enabled;
3275                 config->sprites_scaled |= wm->sprites_scaled;
3276                 config->num_pipes_active++;
3277         }
3278 }
3279
3280 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
3281 {
3282         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3283         struct ilk_wm_maximums max;
3284         struct intel_wm_config config = {};
3285         struct ilk_wm_values results = {};
3286         enum intel_ddb_partitioning partitioning;
3287
3288         ilk_compute_wm_config(dev_priv, &config);
3289
3290         ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
3291         ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
3292
3293         /* 5/6 split only in single pipe config on IVB+ */
3294         if (DISPLAY_VER(dev_priv) >= 7 &&
3295             config.num_pipes_active == 1 && config.sprites_enabled) {
3296                 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
3297                 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
3298
3299                 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
3300         } else {
3301                 best_lp_wm = &lp_wm_1_2;
3302         }
3303
3304         partitioning = (best_lp_wm == &lp_wm_1_2) ?
3305                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3306
3307         ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
3308
3309         ilk_write_wm_values(dev_priv, &results);
3310 }
3311
3312 static void ilk_initial_watermarks(struct intel_atomic_state *state,
3313                                    struct intel_crtc *crtc)
3314 {
3315         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3316         const struct intel_crtc_state *crtc_state =
3317                 intel_atomic_get_new_crtc_state(state, crtc);
3318
3319         mutex_lock(&dev_priv->display.wm.wm_mutex);
3320         crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
3321         ilk_program_watermarks(dev_priv);
3322         mutex_unlock(&dev_priv->display.wm.wm_mutex);
3323 }
3324
3325 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
3326                                     struct intel_crtc *crtc)
3327 {
3328         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3329         const struct intel_crtc_state *crtc_state =
3330                 intel_atomic_get_new_crtc_state(state, crtc);
3331
3332         if (!crtc_state->wm.need_postvbl_update)
3333                 return;
3334
3335         mutex_lock(&dev_priv->display.wm.wm_mutex);
3336         crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
3337         ilk_program_watermarks(dev_priv);
3338         mutex_unlock(&dev_priv->display.wm.wm_mutex);
3339 }
3340
3341 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
3342 {
3343         struct drm_device *dev = crtc->base.dev;
3344         struct drm_i915_private *dev_priv = to_i915(dev);
3345         struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3346         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
3347         struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
3348         enum pipe pipe = crtc->pipe;
3349
3350         hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
3351
3352         memset(active, 0, sizeof(*active));
3353
3354         active->pipe_enabled = crtc->active;
3355
3356         if (active->pipe_enabled) {
3357                 u32 tmp = hw->wm_pipe[pipe];
3358
3359                 /*
3360                  * For active pipes LP0 watermark is marked as
3361                  * enabled, and LP1+ watermaks as disabled since
3362                  * we can't really reverse compute them in case
3363                  * multiple pipes are active.
3364                  */
3365                 active->wm[0].enable = true;
3366                 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
3367                 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
3368                 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
3369         } else {
3370                 int level;
3371
3372                 /*
3373                  * For inactive pipes, all watermark levels
3374                  * should be marked as enabled but zeroed,
3375                  * which is what we'd compute them to.
3376                  */
3377                 for (level = 0; level < dev_priv->display.wm.num_levels; level++)
3378                         active->wm[level].enable = true;
3379         }
3380
3381         crtc->wm.active.ilk = *active;
3382 }
3383
3384 static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_state *state)
3385 {
3386         struct drm_plane *plane;
3387         struct intel_crtc *crtc;
3388
3389         for_each_intel_crtc(state->dev, crtc) {
3390                 struct intel_crtc_state *crtc_state;
3391
3392                 crtc_state = intel_atomic_get_crtc_state(state, crtc);
3393                 if (IS_ERR(crtc_state))
3394                         return PTR_ERR(crtc_state);
3395
3396                 if (crtc_state->hw.active) {
3397                         /*
3398                          * Preserve the inherited flag to avoid
3399                          * taking the full modeset path.
3400                          */
3401                         crtc_state->inherited = true;
3402                 }
3403         }
3404
3405         drm_for_each_plane(plane, state->dev) {
3406                 struct drm_plane_state *plane_state;
3407
3408                 plane_state = drm_atomic_get_plane_state(state, plane);
3409                 if (IS_ERR(plane_state))
3410                         return PTR_ERR(plane_state);
3411         }
3412
3413         return 0;
3414 }
3415
3416 /*
3417  * Calculate what we think the watermarks should be for the state we've read
3418  * out of the hardware and then immediately program those watermarks so that
3419  * we ensure the hardware settings match our internal state.
3420  *
3421  * We can calculate what we think WM's should be by creating a duplicate of the
3422  * current state (which was constructed during hardware readout) and running it
3423  * through the atomic check code to calculate new watermark values in the
3424  * state object.
3425  */
3426 void ilk_wm_sanitize(struct drm_i915_private *dev_priv)
3427 {
3428         struct drm_atomic_state *state;
3429         struct intel_atomic_state *intel_state;
3430         struct intel_crtc *crtc;
3431         struct intel_crtc_state *crtc_state;
3432         struct drm_modeset_acquire_ctx ctx;
3433         int ret;
3434         int i;
3435
3436         /* Only supported on platforms that use atomic watermark design */
3437         if (!dev_priv->display.funcs.wm->optimize_watermarks)
3438                 return;
3439
3440         if (drm_WARN_ON(&dev_priv->drm, DISPLAY_VER(dev_priv) >= 9))
3441                 return;
3442
3443         state = drm_atomic_state_alloc(&dev_priv->drm);
3444         if (drm_WARN_ON(&dev_priv->drm, !state))
3445                 return;
3446
3447         intel_state = to_intel_atomic_state(state);
3448
3449         drm_modeset_acquire_init(&ctx, 0);
3450
3451         state->acquire_ctx = &ctx;
3452         to_intel_atomic_state(state)->internal = true;
3453
3454 retry:
3455         /*
3456          * Hardware readout is the only time we don't want to calculate
3457          * intermediate watermarks (since we don't trust the current
3458          * watermarks).
3459          */
3460         if (!HAS_GMCH(dev_priv))
3461                 intel_state->skip_intermediate_wm = true;
3462
3463         ret = ilk_sanitize_watermarks_add_affected(state);
3464         if (ret)
3465                 goto fail;
3466
3467         ret = intel_atomic_check(&dev_priv->drm, state);
3468         if (ret)
3469                 goto fail;
3470
3471         /* Write calculated watermark values back */
3472         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
3473                 crtc_state->wm.need_postvbl_update = true;
3474                 intel_optimize_watermarks(intel_state, crtc);
3475
3476                 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
3477         }
3478
3479 fail:
3480         if (ret == -EDEADLK) {
3481                 drm_atomic_state_clear(state);
3482                 drm_modeset_backoff(&ctx);
3483                 goto retry;
3484         }
3485
3486         /*
3487          * If we fail here, it means that the hardware appears to be
3488          * programmed in a way that shouldn't be possible, given our
3489          * understanding of watermark requirements.  This might mean a
3490          * mistake in the hardware readout code or a mistake in the
3491          * watermark calculations for a given platform.  Raise a WARN
3492          * so that this is noticeable.
3493          *
3494          * If this actually happens, we'll have to just leave the
3495          * BIOS-programmed watermarks untouched and hope for the best.
3496          */
3497         drm_WARN(&dev_priv->drm, ret,
3498                  "Could not determine valid watermarks for inherited state\n");
3499
3500         drm_atomic_state_put(state);
3501
3502         drm_modeset_drop_locks(&ctx);
3503         drm_modeset_acquire_fini(&ctx);
3504 }
3505
3506 #define _FW_WM(value, plane) \
3507         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3508 #define _FW_WM_VLV(value, plane) \
3509         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3510
3511 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
3512                                struct g4x_wm_values *wm)
3513 {
3514         u32 tmp;
3515
3516         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3517         wm->sr.plane = _FW_WM(tmp, SR);
3518         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3519         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
3520         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
3521
3522         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3523         wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
3524         wm->sr.fbc = _FW_WM(tmp, FBC_SR);
3525         wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
3526         wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
3527         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3528         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
3529
3530         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3531         wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
3532         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3533         wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
3534         wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
3535 }
3536
3537 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
3538                                struct vlv_wm_values *wm)
3539 {
3540         enum pipe pipe;
3541         u32 tmp;
3542
3543         for_each_pipe(dev_priv, pipe) {
3544                 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
3545
3546                 wm->ddl[pipe].plane[PLANE_PRIMARY] =
3547                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3548                 wm->ddl[pipe].plane[PLANE_CURSOR] =
3549                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3550                 wm->ddl[pipe].plane[PLANE_SPRITE0] =
3551                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3552                 wm->ddl[pipe].plane[PLANE_SPRITE1] =
3553                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3554         }
3555
3556         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3557         wm->sr.plane = _FW_WM(tmp, SR);
3558         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3559         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
3560         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
3561
3562         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3563         wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
3564         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3565         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
3566
3567         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3568         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3569
3570         if (IS_CHERRYVIEW(dev_priv)) {
3571                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
3572                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3573                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3574
3575                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
3576                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
3577                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
3578
3579                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
3580                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
3581                 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
3582
3583                 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3584                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3585                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3586                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3587                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
3588                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3589                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3590                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3591                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3592                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3593                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3594         } else {
3595                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
3596                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3597                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3598
3599                 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3600                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3601                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3602                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3603                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3604                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3605                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3606                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3607         }
3608 }
3609
3610 #undef _FW_WM
3611 #undef _FW_WM_VLV
3612
3613 static void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
3614 {
3615         struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;
3616         struct intel_crtc *crtc;
3617
3618         g4x_read_wm_values(dev_priv, wm);
3619
3620         wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
3621
3622         for_each_intel_crtc(&dev_priv->drm, crtc) {
3623                 struct intel_crtc_state *crtc_state =
3624                         to_intel_crtc_state(crtc->base.state);
3625                 struct g4x_wm_state *active = &crtc->wm.active.g4x;
3626                 struct g4x_pipe_wm *raw;
3627                 enum pipe pipe = crtc->pipe;
3628                 enum plane_id plane_id;
3629                 int level, max_level;
3630
3631                 active->cxsr = wm->cxsr;
3632                 active->hpll_en = wm->hpll_en;
3633                 active->fbc_en = wm->fbc_en;
3634
3635                 active->sr = wm->sr;
3636                 active->hpll = wm->hpll;
3637
3638                 for_each_plane_id_on_crtc(crtc, plane_id) {
3639                         active->wm.plane[plane_id] =
3640                                 wm->pipe[pipe].plane[plane_id];
3641                 }
3642
3643                 if (wm->cxsr && wm->hpll_en)
3644                         max_level = G4X_WM_LEVEL_HPLL;
3645                 else if (wm->cxsr)
3646                         max_level = G4X_WM_LEVEL_SR;
3647                 else
3648                         max_level = G4X_WM_LEVEL_NORMAL;
3649
3650                 level = G4X_WM_LEVEL_NORMAL;
3651                 raw = &crtc_state->wm.g4x.raw[level];
3652                 for_each_plane_id_on_crtc(crtc, plane_id)
3653                         raw->plane[plane_id] = active->wm.plane[plane_id];
3654
3655                 level = G4X_WM_LEVEL_SR;
3656                 if (level > max_level)
3657                         goto out;
3658
3659                 raw = &crtc_state->wm.g4x.raw[level];
3660                 raw->plane[PLANE_PRIMARY] = active->sr.plane;
3661                 raw->plane[PLANE_CURSOR] = active->sr.cursor;
3662                 raw->plane[PLANE_SPRITE0] = 0;
3663                 raw->fbc = active->sr.fbc;
3664
3665                 level = G4X_WM_LEVEL_HPLL;
3666                 if (level > max_level)
3667                         goto out;
3668
3669                 raw = &crtc_state->wm.g4x.raw[level];
3670                 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
3671                 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
3672                 raw->plane[PLANE_SPRITE0] = 0;
3673                 raw->fbc = active->hpll.fbc;
3674
3675                 level++;
3676         out:
3677                 for_each_plane_id_on_crtc(crtc, plane_id)
3678                         g4x_raw_plane_wm_set(crtc_state, level,
3679                                              plane_id, USHRT_MAX);
3680                 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
3681
3682                 g4x_invalidate_wms(crtc, active, level);
3683
3684                 crtc_state->wm.g4x.optimal = *active;
3685                 crtc_state->wm.g4x.intermediate = *active;
3686
3687                 drm_dbg_kms(&dev_priv->drm,
3688                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
3689                             pipe_name(pipe),
3690                             wm->pipe[pipe].plane[PLANE_PRIMARY],
3691                             wm->pipe[pipe].plane[PLANE_CURSOR],
3692                             wm->pipe[pipe].plane[PLANE_SPRITE0]);
3693         }
3694
3695         drm_dbg_kms(&dev_priv->drm,
3696                     "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
3697                     wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
3698         drm_dbg_kms(&dev_priv->drm,
3699                     "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
3700                     wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
3701         drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
3702                     str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
3703                     str_yes_no(wm->fbc_en));
3704 }
3705
3706 static void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
3707 {
3708         struct intel_plane *plane;
3709         struct intel_crtc *crtc;
3710
3711         mutex_lock(&dev_priv->display.wm.wm_mutex);
3712
3713         for_each_intel_plane(&dev_priv->drm, plane) {
3714                 struct intel_crtc *crtc =
3715                         intel_crtc_for_pipe(dev_priv, plane->pipe);
3716                 struct intel_crtc_state *crtc_state =
3717                         to_intel_crtc_state(crtc->base.state);
3718                 struct intel_plane_state *plane_state =
3719                         to_intel_plane_state(plane->base.state);
3720                 enum plane_id plane_id = plane->id;
3721                 int level;
3722
3723                 if (plane_state->uapi.visible)
3724                         continue;
3725
3726                 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
3727                         struct g4x_pipe_wm *raw =
3728                                 &crtc_state->wm.g4x.raw[level];
3729
3730                         raw->plane[plane_id] = 0;
3731
3732                         if (plane_id == PLANE_PRIMARY)
3733                                 raw->fbc = 0;
3734                 }
3735         }
3736
3737         for_each_intel_crtc(&dev_priv->drm, crtc) {
3738                 struct intel_crtc_state *crtc_state =
3739                         to_intel_crtc_state(crtc->base.state);
3740                 int ret;
3741
3742                 ret = _g4x_compute_pipe_wm(crtc_state);
3743                 drm_WARN_ON(&dev_priv->drm, ret);
3744
3745                 crtc_state->wm.g4x.intermediate =
3746                         crtc_state->wm.g4x.optimal;
3747                 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
3748         }
3749
3750         g4x_program_watermarks(dev_priv);
3751
3752         mutex_unlock(&dev_priv->display.wm.wm_mutex);
3753 }
3754
3755 static void g4x_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3756 {
3757         g4x_wm_get_hw_state(i915);
3758         g4x_wm_sanitize(i915);
3759 }
3760
3761 static void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
3762 {
3763         struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;
3764         struct intel_crtc *crtc;
3765         u32 val;
3766
3767         vlv_read_wm_values(dev_priv, wm);
3768
3769         wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3770         wm->level = VLV_WM_LEVEL_PM2;
3771
3772         if (IS_CHERRYVIEW(dev_priv)) {
3773                 vlv_punit_get(dev_priv);
3774
3775                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
3776                 if (val & DSP_MAXFIFO_PM5_ENABLE)
3777                         wm->level = VLV_WM_LEVEL_PM5;
3778
3779                 /*
3780                  * If DDR DVFS is disabled in the BIOS, Punit
3781                  * will never ack the request. So if that happens
3782                  * assume we don't have to enable/disable DDR DVFS
3783                  * dynamically. To test that just set the REQ_ACK
3784                  * bit to poke the Punit, but don't change the
3785                  * HIGH/LOW bits so that we don't actually change
3786                  * the current state.
3787                  */
3788                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3789                 val |= FORCE_DDR_FREQ_REQ_ACK;
3790                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
3791
3792                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
3793                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
3794                         drm_dbg_kms(&dev_priv->drm,
3795                                     "Punit not acking DDR DVFS request, "
3796                                     "assuming DDR DVFS is disabled\n");
3797                         dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM5 + 1;
3798                 } else {
3799                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3800                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
3801                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
3802                 }
3803
3804                 vlv_punit_put(dev_priv);
3805         }
3806
3807         for_each_intel_crtc(&dev_priv->drm, crtc) {
3808                 struct intel_crtc_state *crtc_state =
3809                         to_intel_crtc_state(crtc->base.state);
3810                 struct vlv_wm_state *active = &crtc->wm.active.vlv;
3811                 const struct vlv_fifo_state *fifo_state =
3812                         &crtc_state->wm.vlv.fifo_state;
3813                 enum pipe pipe = crtc->pipe;
3814                 enum plane_id plane_id;
3815                 int level;
3816
3817                 vlv_get_fifo_size(crtc_state);
3818
3819                 active->num_levels = wm->level + 1;
3820                 active->cxsr = wm->cxsr;
3821
3822                 for (level = 0; level < active->num_levels; level++) {
3823                         struct g4x_pipe_wm *raw =
3824                                 &crtc_state->wm.vlv.raw[level];
3825
3826                         active->sr[level].plane = wm->sr.plane;
3827                         active->sr[level].cursor = wm->sr.cursor;
3828
3829                         for_each_plane_id_on_crtc(crtc, plane_id) {
3830                                 active->wm[level].plane[plane_id] =
3831                                         wm->pipe[pipe].plane[plane_id];
3832
3833                                 raw->plane[plane_id] =
3834                                         vlv_invert_wm_value(active->wm[level].plane[plane_id],
3835                                                             fifo_state->plane[plane_id]);
3836                         }
3837                 }
3838
3839                 for_each_plane_id_on_crtc(crtc, plane_id)
3840                         vlv_raw_plane_wm_set(crtc_state, level,
3841                                              plane_id, USHRT_MAX);
3842                 vlv_invalidate_wms(crtc, active, level);
3843
3844                 crtc_state->wm.vlv.optimal = *active;
3845                 crtc_state->wm.vlv.intermediate = *active;
3846
3847                 drm_dbg_kms(&dev_priv->drm,
3848                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
3849                             pipe_name(pipe),
3850                             wm->pipe[pipe].plane[PLANE_PRIMARY],
3851                             wm->pipe[pipe].plane[PLANE_CURSOR],
3852                             wm->pipe[pipe].plane[PLANE_SPRITE0],
3853                             wm->pipe[pipe].plane[PLANE_SPRITE1]);
3854         }
3855
3856         drm_dbg_kms(&dev_priv->drm,
3857                     "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
3858                     wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
3859 }
3860
3861 static void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
3862 {
3863         struct intel_plane *plane;
3864         struct intel_crtc *crtc;
3865
3866         mutex_lock(&dev_priv->display.wm.wm_mutex);
3867
3868         for_each_intel_plane(&dev_priv->drm, plane) {
3869                 struct intel_crtc *crtc =
3870                         intel_crtc_for_pipe(dev_priv, plane->pipe);
3871                 struct intel_crtc_state *crtc_state =
3872                         to_intel_crtc_state(crtc->base.state);
3873                 struct intel_plane_state *plane_state =
3874                         to_intel_plane_state(plane->base.state);
3875                 enum plane_id plane_id = plane->id;
3876                 int level;
3877
3878                 if (plane_state->uapi.visible)
3879                         continue;
3880
3881                 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
3882                         struct g4x_pipe_wm *raw =
3883                                 &crtc_state->wm.vlv.raw[level];
3884
3885                         raw->plane[plane_id] = 0;
3886                 }
3887         }
3888
3889         for_each_intel_crtc(&dev_priv->drm, crtc) {
3890                 struct intel_crtc_state *crtc_state =
3891                         to_intel_crtc_state(crtc->base.state);
3892                 int ret;
3893
3894                 ret = _vlv_compute_pipe_wm(crtc_state);
3895                 drm_WARN_ON(&dev_priv->drm, ret);
3896
3897                 crtc_state->wm.vlv.intermediate =
3898                         crtc_state->wm.vlv.optimal;
3899                 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
3900         }
3901
3902         vlv_program_watermarks(dev_priv);
3903
3904         mutex_unlock(&dev_priv->display.wm.wm_mutex);
3905 }
3906
3907 static void vlv_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3908 {
3909         vlv_wm_get_hw_state(i915);
3910         vlv_wm_sanitize(i915);
3911 }
3912
3913 /*
3914  * FIXME should probably kill this and improve
3915  * the real watermark readout/sanitation instead
3916  */
3917 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
3918 {
3919         intel_uncore_rmw(&dev_priv->uncore, WM3_LP_ILK, WM_LP_ENABLE, 0);
3920         intel_uncore_rmw(&dev_priv->uncore, WM2_LP_ILK, WM_LP_ENABLE, 0);
3921         intel_uncore_rmw(&dev_priv->uncore, WM1_LP_ILK, WM_LP_ENABLE, 0);
3922
3923         /*
3924          * Don't touch WM_LP_SPRITE_ENABLE here.
3925          * Doing so could cause underruns.
3926          */
3927 }
3928
3929 static void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
3930 {
3931         struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3932         struct intel_crtc *crtc;
3933
3934         ilk_init_lp_watermarks(dev_priv);
3935
3936         for_each_intel_crtc(&dev_priv->drm, crtc)
3937                 ilk_pipe_wm_get_hw_state(crtc);
3938
3939         hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
3940         hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
3941         hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
3942
3943         hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
3944         if (DISPLAY_VER(dev_priv) >= 7) {
3945                 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
3946                 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
3947         }
3948
3949         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3950                 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) &
3951                                     WM_MISC_DATA_PARTITION_5_6) ?
3952                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3953         else if (IS_IVYBRIDGE(dev_priv))
3954                 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) &
3955                                     DISP_DATA_PARTITION_5_6) ?
3956                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3957
3958         hw->enable_fbc_wm =
3959                 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
3960 }
3961
3962 static const struct intel_wm_funcs ilk_wm_funcs = {
3963         .compute_pipe_wm = ilk_compute_pipe_wm,
3964         .compute_intermediate_wm = ilk_compute_intermediate_wm,
3965         .initial_watermarks = ilk_initial_watermarks,
3966         .optimize_watermarks = ilk_optimize_watermarks,
3967         .get_hw_state = ilk_wm_get_hw_state,
3968 };
3969
3970 static const struct intel_wm_funcs vlv_wm_funcs = {
3971         .compute_pipe_wm = vlv_compute_pipe_wm,
3972         .compute_intermediate_wm = vlv_compute_intermediate_wm,
3973         .initial_watermarks = vlv_initial_watermarks,
3974         .optimize_watermarks = vlv_optimize_watermarks,
3975         .atomic_update_watermarks = vlv_atomic_update_fifo,
3976         .get_hw_state = vlv_wm_get_hw_state_and_sanitize,
3977 };
3978
3979 static const struct intel_wm_funcs g4x_wm_funcs = {
3980         .compute_pipe_wm = g4x_compute_pipe_wm,
3981         .compute_intermediate_wm = g4x_compute_intermediate_wm,
3982         .initial_watermarks = g4x_initial_watermarks,
3983         .optimize_watermarks = g4x_optimize_watermarks,
3984         .get_hw_state = g4x_wm_get_hw_state_and_sanitize,
3985 };
3986
3987 static const struct intel_wm_funcs pnv_wm_funcs = {
3988         .update_wm = pnv_update_wm,
3989 };
3990
3991 static const struct intel_wm_funcs i965_wm_funcs = {
3992         .update_wm = i965_update_wm,
3993 };
3994
3995 static const struct intel_wm_funcs i9xx_wm_funcs = {
3996         .update_wm = i9xx_update_wm,
3997 };
3998
3999 static const struct intel_wm_funcs i845_wm_funcs = {
4000         .update_wm = i845_update_wm,
4001 };
4002
4003 static const struct intel_wm_funcs nop_funcs = {
4004 };
4005
4006 void i9xx_wm_init(struct drm_i915_private *dev_priv)
4007 {
4008         /* For FIFO watermark updates */
4009         if (HAS_PCH_SPLIT(dev_priv)) {
4010                 ilk_setup_wm_latency(dev_priv);
4011                 dev_priv->display.funcs.wm = &ilk_wm_funcs;
4012         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
4013                 vlv_setup_wm_latency(dev_priv);
4014                 dev_priv->display.funcs.wm = &vlv_wm_funcs;
4015         } else if (IS_G4X(dev_priv)) {
4016                 g4x_setup_wm_latency(dev_priv);
4017                 dev_priv->display.funcs.wm = &g4x_wm_funcs;
4018         } else if (IS_PINEVIEW(dev_priv)) {
4019                 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
4020                                             dev_priv->is_ddr3,
4021                                             dev_priv->fsb_freq,
4022                                             dev_priv->mem_freq)) {
4023                         drm_info(&dev_priv->drm,
4024                                  "failed to find known CxSR latency "
4025                                  "(found ddr%s fsb freq %d, mem freq %d), "
4026                                  "disabling CxSR\n",
4027                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
4028                                  dev_priv->fsb_freq, dev_priv->mem_freq);
4029                         /* Disable CxSR and never update its watermark again */
4030                         intel_set_memory_cxsr(dev_priv, false);
4031                         dev_priv->display.funcs.wm = &nop_funcs;
4032                 } else {
4033                         dev_priv->display.funcs.wm = &pnv_wm_funcs;
4034                 }
4035         } else if (DISPLAY_VER(dev_priv) == 4) {
4036                 dev_priv->display.funcs.wm = &i965_wm_funcs;
4037         } else if (DISPLAY_VER(dev_priv) == 3) {
4038                 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4039         } else if (DISPLAY_VER(dev_priv) == 2) {
4040                 if (INTEL_NUM_PIPES(dev_priv) == 1)
4041                         dev_priv->display.funcs.wm = &i845_wm_funcs;
4042                 else
4043                         dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4044         } else {
4045                 drm_err(&dev_priv->drm,
4046                         "unexpected fall-through in %s\n", __func__);
4047                 dev_priv->display.funcs.wm = &nop_funcs;
4048         }
4049 }