drivers/gpu/drm/i915/display/i9xx_wm.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2023 Intel Corporation
   4  */
   5
   6 #include "i915_drv.h"
   7 #include "i915_reg.h"
   8 #include "i9xx_wm.h"
   9 #include "i9xx_wm_regs.h"
  10 #include "intel_atomic.h"
  11 #include "intel_bo.h"
  12 #include "intel_display.h"
  13 #include "intel_display_trace.h"
  14 #include "intel_fb.h"
  15 #include "intel_mchbar_regs.h"
  16 #include "intel_wm.h"
  17 #include "skl_watermark.h"
  18 #include "vlv_sideband.h"
  19
  20 struct intel_watermark_params {
  21         u16 fifo_size;
  22         u16 max_wm;
  23         u8 default_wm;
  24         u8 guard_size;
  25         u8 cacheline_size;
  26 };
  27
  28 /* used in computing the new watermarks state */
  29 struct intel_wm_config {
  30         unsigned int num_pipes_active;
  31         bool sprites_enabled;
  32         bool sprites_scaled;
  33 };
  34
  35 struct cxsr_latency {
  36         bool is_desktop : 1;
  37         bool is_ddr3 : 1;
  38         u16 fsb_freq;
  39         u16 mem_freq;
  40         u16 display_sr;
  41         u16 display_hpll_disable;
  42         u16 cursor_sr;
  43         u16 cursor_hpll_disable;
  44 };
  45
  46 static const struct cxsr_latency cxsr_latency_table[] = {
  47         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
  48         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
  49         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
  50         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
  51         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
  52
  53         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
  54         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
  55         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
  56         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
  57         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
  58
  59         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
  60         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
  61         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
  62         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
  63         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
  64
  65         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
  66         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
  67         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
  68         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
  69         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
  70
  71         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
  72         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
  73         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
  74         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
  75         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
  76
  77         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
  78         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
  79         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
  80         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
  81         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
  82 };
  83
  84 static const struct cxsr_latency *pnv_get_cxsr_latency(struct drm_i915_private *i915)
  85 {
  86         int i;
  87
  88         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
  89                 const struct cxsr_latency *latency = &cxsr_latency_table[i];
  90                 bool is_desktop = !IS_MOBILE(i915);
  91
  92                 if (is_desktop == latency->is_desktop &&
  93                     i915->is_ddr3 == latency->is_ddr3 &&
  94                     DIV_ROUND_CLOSEST(i915->fsb_freq, 1000) == latency->fsb_freq &&
  95                     DIV_ROUND_CLOSEST(i915->mem_freq, 1000) == latency->mem_freq)
  96                         return latency;
  97         }
  98
  99         drm_dbg_kms(&i915->drm,
 100                     "Could not find CxSR latency for DDR%s, FSB %u kHz, MEM %u kHz\n",
 101                     i915->is_ddr3 ? "3" : "2", i915->fsb_freq, i915->mem_freq);
 102
 103         return NULL;
 104 }
 105
 106 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 107 {
 108         u32 val;
 109
 110         vlv_punit_get(dev_priv);
 111
 112         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 113         if (enable)
 114                 val &= ~FORCE_DDR_HIGH_FREQ;
 115         else
 116                 val |= FORCE_DDR_HIGH_FREQ;
 117         val &= ~FORCE_DDR_LOW_FREQ;
 118         val |= FORCE_DDR_FREQ_REQ_ACK;
 119         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 120
 121         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 122                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 123                 drm_err(&dev_priv->drm,
 124                         "timed out waiting for Punit DDR DVFS request\n");
 125
 126         vlv_punit_put(dev_priv);
 127 }
 128
 129 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 130 {
 131         u32 val;
 132
 133         vlv_punit_get(dev_priv);
 134
 135         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 136         if (enable)
 137                 val |= DSP_MAXFIFO_PM5_ENABLE;
 138         else
 139                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 140         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 141
 142         vlv_punit_put(dev_priv);
 143 }
 144
 145 #define FW_WM(value, plane) \
 146         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 147
 148 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 149 {
 150         struct intel_display *display = &dev_priv->display;
 151         bool was_enabled;
 152         u32 val;
 153
 154         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 155                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 156                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 157                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
 158         } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
 159                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
 160                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 161                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
 162         } else if (IS_PINEVIEW(dev_priv)) {
 163                 val = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));
 164                 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 165                 if (enable)
 166                         val |= PINEVIEW_SELF_REFRESH_EN;
 167                 else
 168                         val &= ~PINEVIEW_SELF_REFRESH_EN;
 169                 intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv), val);
 170                 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3(dev_priv));
 171         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 172                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
 173                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 174                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 175                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
 176                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
 177         } else if (IS_I915GM(dev_priv)) {
 178                 /*
 179                  * FIXME can't find a bit like this for 915G, and
 180                  * yet it does have the related watermark in
 181                  * FW_BLC_SELF. What's going on?
 182                  */
 183                 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
 184                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 185                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 186                 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
 187                 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
 188         } else {
 189                 return false;
 190         }
 191
 192         trace_intel_memory_cxsr(display, was_enabled, enable);
 193
 194         drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
 195                     str_enabled_disabled(enable),
 196                     str_enabled_disabled(was_enabled));
 197
 198         return was_enabled;
 199 }
 200
 201 /**
 202  * intel_set_memory_cxsr - Configure CxSR state
 203  * @dev_priv: i915 device
 204  * @enable: Allow vs. disallow CxSR
 205  *
 206  * Allow or disallow the system to enter a special CxSR
 207  * (C-state self refresh) state. What typically happens in CxSR mode
 208  * is that several display FIFOs may get combined into a single larger
 209  * FIFO for a particular plane (so called max FIFO mode) to allow the
 210  * system to defer memory fetches longer, and the memory will enter
 211  * self refresh.
 212  *
 213  * Note that enabling CxSR does not guarantee that the system enter
 214  * this special mode, nor does it guarantee that the system stays
 215  * in that mode once entered. So this just allows/disallows the system
 216  * to autonomously utilize the CxSR mode. Other factors such as core
 217  * C-states will affect when/if the system actually enters/exits the
 218  * CxSR mode.
 219  *
 220  * Note that on VLV/CHV this actually only controls the max FIFO mode,
 221  * and the system is free to enter/exit memory self refresh at any time
 222  * even when the use of CxSR has been disallowed.
 223  *
 224  * While the system is actually in the CxSR/max FIFO mode, some plane
 225  * control registers will not get latched on vblank. Thus in order to
 226  * guarantee the system will respond to changes in the plane registers
 227  * we must always disallow CxSR prior to making changes to those registers.
 228  * Unfortunately the system will re-evaluate the CxSR conditions at
 229  * frame start which happens after vblank start (which is when the plane
 230  * registers would get latched), so we can't proceed with the plane update
 231  * during the same frame where we disallowed CxSR.
 232  *
 233  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 234  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 235  * the hardware w.r.t. HPLL SR when writing to plane registers.
 236  * Disallowing just CxSR is sufficient.
 237  */
 238 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 239 {
 240         bool ret;
 241
 242         mutex_lock(&dev_priv->display.wm.wm_mutex);
 243         ret = _intel_set_memory_cxsr(dev_priv, enable);
 244         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 245                 dev_priv->display.wm.vlv.cxsr = enable;
 246         else if (IS_G4X(dev_priv))
 247                 dev_priv->display.wm.g4x.cxsr = enable;
 248         mutex_unlock(&dev_priv->display.wm.wm_mutex);
 249
 250         return ret;
 251 }
 252
 253 /*
 254  * Latency for FIFO fetches is dependent on several factors:
 255  *   - memory configuration (speed, channels)
 256  *   - chipset
 257  *   - current MCH state
 258  * It can be fairly high in some situations, so here we assume a fairly
 259  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 260  * set this value too high, the FIFO will fetch frequently to stay full)
 261  * and power consumption (set it too low to save power and we might see
 262  * FIFO underruns and display "flicker").
 263  *
 264  * A value of 5us seems to be a good balance; safe for very low end
 265  * platforms but not overly aggressive on lower latency configs.
 266  */
 267 static const int pessimal_latency_ns = 5000;
 268
 269 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 270         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 271
 272 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 273 {
 274         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 275         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 276         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 277         enum pipe pipe = crtc->pipe;
 278         int sprite0_start, sprite1_start;
 279         u32 dsparb, dsparb2, dsparb3;
 280
 281         switch (pipe) {
 282         case PIPE_A:
 283                 dsparb = intel_uncore_read(&dev_priv->uncore,
 284                                            DSPARB(dev_priv));
 285                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 286                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 287                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 288                 break;
 289         case PIPE_B:
 290                 dsparb = intel_uncore_read(&dev_priv->uncore,
 291                                            DSPARB(dev_priv));
 292                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 293                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 294                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 295                 break;
 296         case PIPE_C:
 297                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 298                 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
 299                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 300                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 301                 break;
 302         default:
 303                 MISSING_CASE(pipe);
 304                 return;
 305         }
 306
 307         fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 308         fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 309         fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 310         fifo_state->plane[PLANE_CURSOR] = 63;
 311 }
 312
 313 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
 314                               enum i9xx_plane_id i9xx_plane)
 315 {
 316         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB(dev_priv));
 317         int size;
 318
 319         size = dsparb & 0x7f;
 320         if (i9xx_plane == PLANE_B)
 321                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 322
 323         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 324                     dsparb, plane_name(i9xx_plane), size);
 325
 326         return size;
 327 }
 328
 329 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
 330                               enum i9xx_plane_id i9xx_plane)
 331 {
 332         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB(dev_priv));
 333         int size;
 334
 335         size = dsparb & 0x1ff;
 336         if (i9xx_plane == PLANE_B)
 337                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 338         size >>= 1; /* Convert to cachelines */
 339
 340         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 341                     dsparb, plane_name(i9xx_plane), size);
 342
 343         return size;
 344 }
 345
 346 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
 347                               enum i9xx_plane_id i9xx_plane)
 348 {
 349         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB(dev_priv));
 350         int size;
 351
 352         size = dsparb & 0x7f;
 353         size >>= 2; /* Convert to cachelines */
 354
 355         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 356                     dsparb, plane_name(i9xx_plane), size);
 357
 358         return size;
 359 }
 360
 361 /* Pineview has different values for various configs */
 362 static const struct intel_watermark_params pnv_display_wm = {
 363         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 364         .max_wm = PINEVIEW_MAX_WM,
 365         .default_wm = PINEVIEW_DFT_WM,
 366         .guard_size = PINEVIEW_GUARD_WM,
 367         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 368 };
 369
 370 static const struct intel_watermark_params pnv_display_hplloff_wm = {
 371         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 372         .max_wm = PINEVIEW_MAX_WM,
 373         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 374         .guard_size = PINEVIEW_GUARD_WM,
 375         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 376 };
 377
 378 static const struct intel_watermark_params pnv_cursor_wm = {
 379         .fifo_size = PINEVIEW_CURSOR_FIFO,
 380         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 381         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 382         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 383         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 384 };
 385
 386 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
 387         .fifo_size = PINEVIEW_CURSOR_FIFO,
 388         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 389         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 390         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 391         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 392 };
 393
 394 static const struct intel_watermark_params i965_cursor_wm_info = {
 395         .fifo_size = I965_CURSOR_FIFO,
 396         .max_wm = I965_CURSOR_MAX_WM,
 397         .default_wm = I965_CURSOR_DFT_WM,
 398         .guard_size = 2,
 399         .cacheline_size = I915_FIFO_LINE_SIZE,
 400 };
 401
 402 static const struct intel_watermark_params i945_wm_info = {
 403         .fifo_size = I945_FIFO_SIZE,
 404         .max_wm = I915_MAX_WM,
 405         .default_wm = 1,
 406         .guard_size = 2,
 407         .cacheline_size = I915_FIFO_LINE_SIZE,
 408 };
 409
 410 static const struct intel_watermark_params i915_wm_info = {
 411         .fifo_size = I915_FIFO_SIZE,
 412         .max_wm = I915_MAX_WM,
 413         .default_wm = 1,
 414         .guard_size = 2,
 415         .cacheline_size = I915_FIFO_LINE_SIZE,
 416 };
 417
 418 static const struct intel_watermark_params i830_a_wm_info = {
 419         .fifo_size = I855GM_FIFO_SIZE,
 420         .max_wm = I915_MAX_WM,
 421         .default_wm = 1,
 422         .guard_size = 2,
 423         .cacheline_size = I830_FIFO_LINE_SIZE,
 424 };
 425
 426 static const struct intel_watermark_params i830_bc_wm_info = {
 427         .fifo_size = I855GM_FIFO_SIZE,
 428         .max_wm = I915_MAX_WM / 2,
 429         .default_wm = 1,
 430         .guard_size = 2,
 431         .cacheline_size = I830_FIFO_LINE_SIZE,
 432 };
 433
 434 static const struct intel_watermark_params i845_wm_info = {
 435         .fifo_size = I830_FIFO_SIZE,
 436         .max_wm = I915_MAX_WM,
 437         .default_wm = 1,
 438         .guard_size = 2,
 439         .cacheline_size = I830_FIFO_LINE_SIZE,
 440 };
 441
 442 /**
 443  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 444  * @pixel_rate: Pipe pixel rate in kHz
 445  * @cpp: Plane bytes per pixel
 446  * @latency: Memory wakeup latency in 0.1us units
 447  *
 448  * Compute the watermark using the method 1 or "small buffer"
 449  * formula. The caller may additonally add extra cachelines
 450  * to account for TLB misses and clock crossings.
 451  *
 452  * This method is concerned with the short term drain rate
 453  * of the FIFO, ie. it does not account for blanking periods
 454  * which would effectively reduce the average drain rate across
 455  * a longer period. The name "small" refers to the fact the
 456  * FIFO is relatively small compared to the amount of data
 457  * fetched.
 458  *
 459  * The FIFO level vs. time graph might look something like:
 460  *
 461  *   |\   |\
 462  *   | \  | \
 463  * __---__---__ (- plane active, _ blanking)
 464  * -> time
 465  *
 466  * or perhaps like this:
 467  *
 468  *   |\|\  |\|\
 469  * __----__----__ (- plane active, _ blanking)
 470  * -> time
 471  *
 472  * Returns:
 473  * The watermark in bytes
 474  */
 475 static unsigned int intel_wm_method1(unsigned int pixel_rate,
 476                                      unsigned int cpp,
 477                                      unsigned int latency)
 478 {
 479         u64 ret;
 480
 481         ret = mul_u32_u32(pixel_rate, cpp * latency);
 482         ret = DIV_ROUND_UP_ULL(ret, 10000);
 483
 484         return ret;
 485 }
 486
 487 /**
 488  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 489  * @pixel_rate: Pipe pixel rate in kHz
 490  * @htotal: Pipe horizontal total
 491  * @width: Plane width in pixels
 492  * @cpp: Plane bytes per pixel
 493  * @latency: Memory wakeup latency in 0.1us units
 494  *
 495  * Compute the watermark using the method 2 or "large buffer"
 496  * formula. The caller may additonally add extra cachelines
 497  * to account for TLB misses and clock crossings.
 498  *
 499  * This method is concerned with the long term drain rate
 500  * of the FIFO, ie. it does account for blanking periods
 501  * which effectively reduce the average drain rate across
 502  * a longer period. The name "large" refers to the fact the
 503  * FIFO is relatively large compared to the amount of data
 504  * fetched.
 505  *
 506  * The FIFO level vs. time graph might look something like:
 507  *
 508  *    |\___       |\___
 509  *    |    \___   |    \___
 510  *    |        \  |        \
 511  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 512  * -> time
 513  *
 514  * Returns:
 515  * The watermark in bytes
 516  */
 517 static unsigned int intel_wm_method2(unsigned int pixel_rate,
 518                                      unsigned int htotal,
 519                                      unsigned int width,
 520                                      unsigned int cpp,
 521                                      unsigned int latency)
 522 {
 523         unsigned int ret;
 524
 525         /*
 526          * FIXME remove once all users are computing
 527          * watermarks in the correct place.
 528          */
 529         if (WARN_ON_ONCE(htotal == 0))
 530                 htotal = 1;
 531
 532         ret = (latency * pixel_rate) / (htotal * 10000);
 533         ret = (ret + 1) * width * cpp;
 534
 535         return ret;
 536 }
 537
 538 /**
 539  * intel_calculate_wm - calculate watermark level
 540  * @i915: the device
 541  * @pixel_rate: pixel clock
 542  * @wm: chip FIFO params
 543  * @fifo_size: size of the FIFO buffer
 544  * @cpp: bytes per pixel
 545  * @latency_ns: memory latency for the platform
 546  *
 547  * Calculate the watermark level (the level at which the display plane will
 548  * start fetching from memory again).  Each chip has a different display
 549  * FIFO size and allocation, so the caller needs to figure that out and pass
 550  * in the correct intel_watermark_params structure.
 551  *
 552  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 553  * on the pixel size.  When it reaches the watermark level, it'll start
 554  * fetching FIFO line sized based chunks from memory until the FIFO fills
 555  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 556  * will occur, and a display engine hang could result.
 557  */
 558 static unsigned int intel_calculate_wm(struct drm_i915_private *i915,
 559                                        int pixel_rate,
 560                                        const struct intel_watermark_params *wm,
 561                                        int fifo_size, int cpp,
 562                                        unsigned int latency_ns)
 563 {
 564         int entries, wm_size;
 565
 566         /*
 567          * Note: we need to make sure we don't overflow for various clock &
 568          * latency values.
 569          * clocks go from a few thousand to several hundred thousand.
 570          * latency is usually a few thousand
 571          */
 572         entries = intel_wm_method1(pixel_rate, cpp,
 573                                    latency_ns / 100);
 574         entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 575                 wm->guard_size;
 576         drm_dbg_kms(&i915->drm, "FIFO entries required for mode: %d\n", entries);
 577
 578         wm_size = fifo_size - entries;
 579         drm_dbg_kms(&i915->drm, "FIFO watermark level: %d\n", wm_size);
 580
 581         /* Don't promote wm_size to unsigned... */
 582         if (wm_size > wm->max_wm)
 583                 wm_size = wm->max_wm;
 584         if (wm_size <= 0)
 585                 wm_size = wm->default_wm;
 586
 587         /*
 588          * Bspec seems to indicate that the value shouldn't be lower than
 589          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 590          * Lets go for 8 which is the burst size since certain platforms
 591          * already use a hardcoded 8 (which is what the spec says should be
 592          * done).
 593          */
 594         if (wm_size <= 8)
 595                 wm_size = 8;
 596
 597         return wm_size;
 598 }
 599
 600 static bool is_disabling(int old, int new, int threshold)
 601 {
 602         return old >= threshold && new < threshold;
 603 }
 604
 605 static bool is_enabling(int old, int new, int threshold)
 606 {
 607         return old < threshold && new >= threshold;
 608 }
 609
 610 static bool intel_crtc_active(struct intel_crtc *crtc)
 611 {
 612         /* Be paranoid as we can arrive here with only partial
 613          * state retrieved from the hardware during setup.
 614          *
 615          * We can ditch the adjusted_mode.crtc_clock check as soon
 616          * as Haswell has gained clock readout/fastboot support.
 617          *
 618          * We can ditch the crtc->primary->state->fb check as soon as we can
 619          * properly reconstruct framebuffers.
 620          *
 621          * FIXME: The intel_crtc->active here should be switched to
 622          * crtc->state->active once we have proper CRTC states wired up
 623          * for atomic.
 624          */
 625         return crtc->active && crtc->base.primary->state->fb &&
 626                 crtc->config->hw.adjusted_mode.crtc_clock;
 627 }
 628
 629 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 630 {
 631         struct intel_crtc *crtc, *enabled = NULL;
 632
 633         for_each_intel_crtc(&dev_priv->drm, crtc) {
 634                 if (intel_crtc_active(crtc)) {
 635                         if (enabled)
 636                                 return NULL;
 637                         enabled = crtc;
 638                 }
 639         }
 640
 641         return enabled;
 642 }
 643
 644 static void pnv_update_wm(struct drm_i915_private *dev_priv)
 645 {
 646         struct intel_crtc *crtc;
 647         const struct cxsr_latency *latency;
 648         u32 reg;
 649         unsigned int wm;
 650
 651         latency = pnv_get_cxsr_latency(dev_priv);
 652         if (!latency) {
 653                 drm_dbg_kms(&dev_priv->drm, "Unknown FSB/MEM, disabling CxSR\n");
 654                 intel_set_memory_cxsr(dev_priv, false);
 655                 return;
 656         }
 657
 658         crtc = single_enabled_crtc(dev_priv);
 659         if (crtc) {
 660                 const struct drm_framebuffer *fb =
 661                         crtc->base.primary->state->fb;
 662                 int pixel_rate = crtc->config->pixel_rate;
 663                 int cpp = fb->format->cpp[0];
 664
 665                 /* Display SR */
 666                 wm = intel_calculate_wm(dev_priv, pixel_rate,
 667                                         &pnv_display_wm,
 668                                         pnv_display_wm.fifo_size,
 669                                         cpp, latency->display_sr);
 670                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1(dev_priv));
 671                 reg &= ~DSPFW_SR_MASK;
 672                 reg |= FW_WM(wm, SR);
 673                 intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv), reg);
 674                 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
 675
 676                 /* cursor SR */
 677                 wm = intel_calculate_wm(dev_priv, pixel_rate,
 678                                         &pnv_cursor_wm,
 679                                         pnv_display_wm.fifo_size,
 680                                         4, latency->cursor_sr);
 681                 intel_uncore_rmw(&dev_priv->uncore, DSPFW3(dev_priv),
 682                                  DSPFW_CURSOR_SR_MASK,
 683                                  FW_WM(wm, CURSOR_SR));
 684
 685                 /* Display HPLL off SR */
 686                 wm = intel_calculate_wm(dev_priv, pixel_rate,
 687                                         &pnv_display_hplloff_wm,
 688                                         pnv_display_hplloff_wm.fifo_size,
 689                                         cpp, latency->display_hpll_disable);
 690                 intel_uncore_rmw(&dev_priv->uncore, DSPFW3(dev_priv),
 691                                  DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR));
 692
 693                 /* cursor HPLL off SR */
 694                 wm = intel_calculate_wm(dev_priv, pixel_rate,
 695                                         &pnv_cursor_hplloff_wm,
 696                                         pnv_display_hplloff_wm.fifo_size,
 697                                         4, latency->cursor_hpll_disable);
 698                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));
 699                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 700                 reg |= FW_WM(wm, HPLL_CURSOR);
 701                 intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv), reg);
 702                 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
 703
 704                 intel_set_memory_cxsr(dev_priv, true);
 705         } else {
 706                 intel_set_memory_cxsr(dev_priv, false);
 707         }
 708 }
 709
 710 static bool i9xx_wm_need_update(const struct intel_plane_state *old_plane_state,
 711                                 const struct intel_plane_state *new_plane_state)
 712 {
 713         /* Update watermarks on tiling or size changes. */
 714         if (old_plane_state->uapi.visible != new_plane_state->uapi.visible)
 715                 return true;
 716
 717         if (!old_plane_state->hw.fb || !new_plane_state->hw.fb)
 718                 return false;
 719
 720         if (old_plane_state->hw.fb->modifier != new_plane_state->hw.fb->modifier ||
 721             old_plane_state->hw.rotation != new_plane_state->hw.rotation ||
 722             drm_rect_width(&old_plane_state->uapi.src) != drm_rect_width(&new_plane_state->uapi.src) ||
 723             drm_rect_height(&old_plane_state->uapi.src) != drm_rect_height(&new_plane_state->uapi.src) ||
 724             drm_rect_width(&old_plane_state->uapi.dst) != drm_rect_width(&new_plane_state->uapi.dst) ||
 725             drm_rect_height(&old_plane_state->uapi.dst) != drm_rect_height(&new_plane_state->uapi.dst))
 726                 return true;
 727
 728         return false;
 729 }
 730
 731 static void i9xx_wm_compute(struct intel_crtc_state *new_crtc_state,
 732                             const struct intel_plane_state *old_plane_state,
 733                             const struct intel_plane_state *new_plane_state)
 734 {
 735         bool turn_off, turn_on, visible, was_visible, mode_changed;
 736
 737         mode_changed = intel_crtc_needs_modeset(new_crtc_state);
 738         was_visible = old_plane_state->uapi.visible;
 739         visible = new_plane_state->uapi.visible;
 740
 741         if (!was_visible && !visible)
 742                 return;
 743
 744         turn_off = was_visible && (!visible || mode_changed);
 745         turn_on = visible && (!was_visible || mode_changed);
 746
 747         /* FIXME nuke when all wm code is atomic */
 748         if (turn_on) {
 749                 new_crtc_state->update_wm_pre = true;
 750         } else if (turn_off) {
 751                 new_crtc_state->update_wm_post = true;
 752         } else if (i9xx_wm_need_update(old_plane_state, new_plane_state)) {
 753                 /* FIXME bollocks */
 754                 new_crtc_state->update_wm_pre = true;
 755                 new_crtc_state->update_wm_post = true;
 756         }
 757 }
 758
 759 static int i9xx_compute_watermarks(struct intel_atomic_state *state,
 760                                    struct intel_crtc *crtc)
 761 {
 762         struct intel_crtc_state *new_crtc_state =
 763                 intel_atomic_get_new_crtc_state(state, crtc);
 764         const struct intel_plane_state *old_plane_state;
 765         const struct intel_plane_state *new_plane_state;
 766         struct intel_plane *plane;
 767         int i;
 768
 769         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
 770                                              new_plane_state, i) {
 771                 if (plane->pipe != crtc->pipe)
 772                         continue;
 773
 774                 i9xx_wm_compute(new_crtc_state, old_plane_state, new_plane_state);
 775         }
 776
 777         return 0;
 778 }
 779
 780 /*
 781  * Documentation says:
 782  * "If the line size is small, the TLB fetches can get in the way of the
 783  *  data fetches, causing some lag in the pixel data return which is not
 784  *  accounted for in the above formulas. The following adjustment only
 785  *  needs to be applied if eight whole lines fit in the buffer at once.
 786  *  The WM is adjusted upwards by the difference between the FIFO size
 787  *  and the size of 8 whole lines. This adjustment is always performed
 788  *  in the actual pixel depth regardless of whether FBC is enabled or not."
 789  */
 790 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 791 {
 792         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 793
 794         return max(0, tlb_miss);
 795 }
 796
 797 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 798                                 const struct g4x_wm_values *wm)
 799 {
 800         struct intel_display *display = &dev_priv->display;
 801         enum pipe pipe;
 802
 803         for_each_pipe(dev_priv, pipe)
 804                 trace_g4x_wm(intel_crtc_for_pipe(display, pipe), wm);
 805
 806         intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv),
 807                            FW_WM(wm->sr.plane, SR) |
 808                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 809                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 810                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 811         intel_uncore_write(&dev_priv->uncore, DSPFW2(dev_priv),
 812                            (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 813                            FW_WM(wm->sr.fbc, FBC_SR) |
 814                            FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
 815                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
 816                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 817                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 818         intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv),
 819                            (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
 820                            FW_WM(wm->sr.cursor, CURSOR_SR) |
 821                            FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
 822                            FW_WM(wm->hpll.plane, HPLL_SR));
 823
 824         intel_uncore_posting_read(&dev_priv->uncore, DSPFW1(dev_priv));
 825 }
 826
 827 #define FW_WM_VLV(value, plane) \
 828         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
 829
 830 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
 831                                 const struct vlv_wm_values *wm)
 832 {
 833         struct intel_display *display = &dev_priv->display;
 834         enum pipe pipe;
 835
 836         for_each_pipe(dev_priv, pipe) {
 837                 trace_vlv_wm(intel_crtc_for_pipe(display, pipe), wm);
 838
 839                 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
 840                                    (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
 841                                    (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
 842                                    (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
 843                                    (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
 844         }
 845
 846         /*
 847          * Zero the (unused) WM1 watermarks, and also clear all the
 848          * high order bits so that there are no out of bounds values
 849          * present in the registers during the reprogramming.
 850          */
 851         intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
 852         intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
 853         intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
 854         intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
 855         intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
 856
 857         intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv),
 858                            FW_WM(wm->sr.plane, SR) |
 859                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 860                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 861                            FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 862         intel_uncore_write(&dev_priv->uncore, DSPFW2(dev_priv),
 863                            FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
 864                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 865                            FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 866         intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv),
 867                            FW_WM(wm->sr.cursor, CURSOR_SR));
 868
 869         if (IS_CHERRYVIEW(dev_priv)) {
 870                 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
 871                                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
 872                                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
 873                 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
 874                                    FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
 875                                    FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
 876                 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
 877                                    FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
 878                                    FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
 879                 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
 880                                    FW_WM(wm->sr.plane >> 9, SR_HI) |
 881                                    FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
 882                                    FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
 883                                    FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
 884                                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
 885                                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
 886                                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
 887                                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
 888                                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
 889                                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
 890         } else {
 891                 intel_uncore_write(&dev_priv->uncore, DSPFW7,
 892                                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
 893                                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
 894                 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
 895                                    FW_WM(wm->sr.plane >> 9, SR_HI) |
 896                                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
 897                                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
 898                                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
 899                                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
 900                                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
 901                                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
 902         }
 903
 904         intel_uncore_posting_read(&dev_priv->uncore, DSPFW1(dev_priv));
 905 }
 906
 907 #undef FW_WM_VLV
 908
 909 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
 910 {
 911         /* all latencies in usec */
 912         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
 913         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
 914         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
 915
 916         dev_priv->display.wm.num_levels = G4X_WM_LEVEL_HPLL + 1;
 917 }
 918
 919 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
 920 {
 921         /*
 922          * DSPCNTR[13] supposedly controls whether the
 923          * primary plane can use the FIFO space otherwise
 924          * reserved for the sprite plane. It's not 100% clear
 925          * what the actual FIFO size is, but it looks like we
 926          * can happily set both primary and sprite watermarks
 927          * up to 127 cachelines. So that would seem to mean
 928          * that either DSPCNTR[13] doesn't do anything, or that
 929          * the total FIFO is >= 256 cachelines in size. Either
 930          * way, we don't seem to have to worry about this
 931          * repartitioning as the maximum watermark value the
 932          * register can hold for each plane is lower than the
 933          * minimum FIFO size.
 934          */
 935         switch (plane_id) {
 936         case PLANE_CURSOR:
 937                 return 63;
 938         case PLANE_PRIMARY:
 939                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
 940         case PLANE_SPRITE0:
 941                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
 942         default:
 943                 MISSING_CASE(plane_id);
 944                 return 0;
 945         }
 946 }
 947
 948 static int g4x_fbc_fifo_size(int level)
 949 {
 950         switch (level) {
 951         case G4X_WM_LEVEL_SR:
 952                 return 7;
 953         case G4X_WM_LEVEL_HPLL:
 954                 return 15;
 955         default:
 956                 MISSING_CASE(level);
 957                 return 0;
 958         }
 959 }
 960
 961 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
 962                           const struct intel_plane_state *plane_state,
 963                           int level)
 964 {
 965         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 966         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 967         const struct drm_display_mode *pipe_mode =
 968                 &crtc_state->hw.pipe_mode;
 969         unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;
 970         unsigned int pixel_rate, htotal, cpp, width, wm;
 971
 972         if (latency == 0)
 973                 return USHRT_MAX;
 974
 975         if (!intel_wm_plane_visible(crtc_state, plane_state))
 976                 return 0;
 977
 978         cpp = plane_state->hw.fb->format->cpp[0];
 979
 980         /*
 981          * WaUse32BppForSRWM:ctg,elk
 982          *
 983          * The spec fails to list this restriction for the
 984          * HPLL watermark, which seems a little strange.
 985          * Let's use 32bpp for the HPLL watermark as well.
 986          */
 987         if (plane->id == PLANE_PRIMARY &&
 988             level != G4X_WM_LEVEL_NORMAL)
 989                 cpp = max(cpp, 4u);
 990
 991         pixel_rate = crtc_state->pixel_rate;
 992         htotal = pipe_mode->crtc_htotal;
 993         width = drm_rect_width(&plane_state->uapi.src) >> 16;
 994
 995         if (plane->id == PLANE_CURSOR) {
 996                 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
 997         } else if (plane->id == PLANE_PRIMARY &&
 998                    level == G4X_WM_LEVEL_NORMAL) {
 999                 wm = intel_wm_method1(pixel_rate, cpp, latency);
1000         } else {
1001                 unsigned int small, large;
1002
1003                 small = intel_wm_method1(pixel_rate, cpp, latency);
1004                 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1005
1006                 wm = min(small, large);
1007         }
1008
1009         wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1010                               width, cpp);
1011
1012         wm = DIV_ROUND_UP(wm, 64) + 2;
1013
1014         return min_t(unsigned int, wm, USHRT_MAX);
1015 }
1016
1017 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1018                                  int level, enum plane_id plane_id, u16 value)
1019 {
1020         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1021         bool dirty = false;
1022
1023         for (; level < dev_priv->display.wm.num_levels; level++) {
1024                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1025
1026                 dirty |= raw->plane[plane_id] != value;
1027                 raw->plane[plane_id] = value;
1028         }
1029
1030         return dirty;
1031 }
1032
1033 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1034                                int level, u16 value)
1035 {
1036         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1037         bool dirty = false;
1038
1039         /* NORMAL level doesn't have an FBC watermark */
1040         level = max(level, G4X_WM_LEVEL_SR);
1041
1042         for (; level < dev_priv->display.wm.num_levels; level++) {
1043                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1044
1045                 dirty |= raw->fbc != value;
1046                 raw->fbc = value;
1047         }
1048
1049         return dirty;
1050 }
1051
1052 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1053                               const struct intel_plane_state *plane_state,
1054                               u32 pri_val);
1055
1056 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1057                                      const struct intel_plane_state *plane_state)
1058 {
1059         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1060         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1061         enum plane_id plane_id = plane->id;
1062         bool dirty = false;
1063         int level;
1064
1065         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1066                 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1067                 if (plane_id == PLANE_PRIMARY)
1068                         dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1069                 goto out;
1070         }
1071
1072         for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
1073                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1074                 int wm, max_wm;
1075
1076                 wm = g4x_compute_wm(crtc_state, plane_state, level);
1077                 max_wm = g4x_plane_fifo_size(plane_id, level);
1078
1079                 if (wm > max_wm)
1080                         break;
1081
1082                 dirty |= raw->plane[plane_id] != wm;
1083                 raw->plane[plane_id] = wm;
1084
1085                 if (plane_id != PLANE_PRIMARY ||
1086                     level == G4X_WM_LEVEL_NORMAL)
1087                         continue;
1088
1089                 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1090                                         raw->plane[plane_id]);
1091                 max_wm = g4x_fbc_fifo_size(level);
1092
1093                 /*
1094                  * FBC wm is not mandatory as we
1095                  * can always just disable its use.
1096                  */
1097                 if (wm > max_wm)
1098                         wm = USHRT_MAX;
1099
1100                 dirty |= raw->fbc != wm;
1101                 raw->fbc = wm;
1102         }
1103
1104         /* mark watermarks as invalid */
1105         dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1106
1107         if (plane_id == PLANE_PRIMARY)
1108                 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1109
1110  out:
1111         if (dirty) {
1112                 drm_dbg_kms(&dev_priv->drm,
1113                             "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1114                             plane->base.name,
1115                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1116                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1117                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1118
1119                 if (plane_id == PLANE_PRIMARY)
1120                         drm_dbg_kms(&dev_priv->drm,
1121                                     "FBC watermarks: SR=%d, HPLL=%d\n",
1122                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1123                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1124         }
1125
1126         return dirty;
1127 }
1128
1129 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1130                                       enum plane_id plane_id, int level)
1131 {
1132         const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1133
1134         return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1135 }
1136
1137 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1138                                      int level)
1139 {
1140         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1141
1142         if (level >= dev_priv->display.wm.num_levels)
1143                 return false;
1144
1145         return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1146                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1147                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1148 }
1149
1150 /* mark all levels starting from 'level' as invalid */
1151 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1152                                struct g4x_wm_state *wm_state, int level)
1153 {
1154         if (level <= G4X_WM_LEVEL_NORMAL) {
1155                 enum plane_id plane_id;
1156
1157                 for_each_plane_id_on_crtc(crtc, plane_id)
1158                         wm_state->wm.plane[plane_id] = USHRT_MAX;
1159         }
1160
1161         if (level <= G4X_WM_LEVEL_SR) {
1162                 wm_state->cxsr = false;
1163                 wm_state->sr.cursor = USHRT_MAX;
1164                 wm_state->sr.plane = USHRT_MAX;
1165                 wm_state->sr.fbc = USHRT_MAX;
1166         }
1167
1168         if (level <= G4X_WM_LEVEL_HPLL) {
1169                 wm_state->hpll_en = false;
1170                 wm_state->hpll.cursor = USHRT_MAX;
1171                 wm_state->hpll.plane = USHRT_MAX;
1172                 wm_state->hpll.fbc = USHRT_MAX;
1173         }
1174 }
1175
1176 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1177                                int level)
1178 {
1179         if (level < G4X_WM_LEVEL_SR)
1180                 return false;
1181
1182         if (level >= G4X_WM_LEVEL_SR &&
1183             wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1184                 return false;
1185
1186         if (level >= G4X_WM_LEVEL_HPLL &&
1187             wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1188                 return false;
1189
1190         return true;
1191 }
1192
1193 static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1194 {
1195         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1196         struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1197         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1198         const struct g4x_pipe_wm *raw;
1199         enum plane_id plane_id;
1200         int level;
1201
1202         level = G4X_WM_LEVEL_NORMAL;
1203         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1204                 goto out;
1205
1206         raw = &crtc_state->wm.g4x.raw[level];
1207         for_each_plane_id_on_crtc(crtc, plane_id)
1208                 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1209
1210         level = G4X_WM_LEVEL_SR;
1211         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1212                 goto out;
1213
1214         raw = &crtc_state->wm.g4x.raw[level];
1215         wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1216         wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1217         wm_state->sr.fbc = raw->fbc;
1218
1219         wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1220
1221         level = G4X_WM_LEVEL_HPLL;
1222         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1223                 goto out;
1224
1225         raw = &crtc_state->wm.g4x.raw[level];
1226         wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1227         wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1228         wm_state->hpll.fbc = raw->fbc;
1229
1230         wm_state->hpll_en = wm_state->cxsr;
1231
1232         level++;
1233
1234  out:
1235         if (level == G4X_WM_LEVEL_NORMAL)
1236                 return -EINVAL;
1237
1238         /* invalidate the higher levels */
1239         g4x_invalidate_wms(crtc, wm_state, level);
1240
1241         /*
1242          * Determine if the FBC watermark(s) can be used. IF
1243          * this isn't the case we prefer to disable the FBC
1244          * watermark(s) rather than disable the SR/HPLL
1245          * level(s) entirely. 'level-1' is the highest valid
1246          * level here.
1247          */
1248         wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1249
1250         return 0;
1251 }
1252
1253 static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1254                                struct intel_crtc *crtc)
1255 {
1256         struct intel_crtc_state *crtc_state =
1257                 intel_atomic_get_new_crtc_state(state, crtc);
1258         const struct intel_plane_state *old_plane_state;
1259         const struct intel_plane_state *new_plane_state;
1260         struct intel_plane *plane;
1261         unsigned int dirty = 0;
1262         int i;
1263
1264         for_each_oldnew_intel_plane_in_state(state, plane,
1265                                              old_plane_state,
1266                                              new_plane_state, i) {
1267                 if (new_plane_state->hw.crtc != &crtc->base &&
1268                     old_plane_state->hw.crtc != &crtc->base)
1269                         continue;
1270
1271                 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1272                         dirty |= BIT(plane->id);
1273         }
1274
1275         if (!dirty)
1276                 return 0;
1277
1278         return _g4x_compute_pipe_wm(crtc_state);
1279 }
1280
1281 static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1282                                        struct intel_crtc *crtc)
1283 {
1284         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1285         struct intel_crtc_state *new_crtc_state =
1286                 intel_atomic_get_new_crtc_state(state, crtc);
1287         const struct intel_crtc_state *old_crtc_state =
1288                 intel_atomic_get_old_crtc_state(state, crtc);
1289         struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1290         const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1291         const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1292         enum plane_id plane_id;
1293
1294         if (!new_crtc_state->hw.active ||
1295             intel_crtc_needs_modeset(new_crtc_state)) {
1296                 *intermediate = *optimal;
1297
1298                 intermediate->cxsr = false;
1299                 intermediate->hpll_en = false;
1300                 goto out;
1301         }
1302
1303         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1304                 !new_crtc_state->disable_cxsr;
1305         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1306                 !new_crtc_state->disable_cxsr;
1307         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1308
1309         for_each_plane_id_on_crtc(crtc, plane_id) {
1310                 intermediate->wm.plane[plane_id] =
1311                         max(optimal->wm.plane[plane_id],
1312                             active->wm.plane[plane_id]);
1313
1314                 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1315                             g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1316         }
1317
1318         intermediate->sr.plane = max(optimal->sr.plane,
1319                                      active->sr.plane);
1320         intermediate->sr.cursor = max(optimal->sr.cursor,
1321                                       active->sr.cursor);
1322         intermediate->sr.fbc = max(optimal->sr.fbc,
1323                                    active->sr.fbc);
1324
1325         intermediate->hpll.plane = max(optimal->hpll.plane,
1326                                        active->hpll.plane);
1327         intermediate->hpll.cursor = max(optimal->hpll.cursor,
1328                                         active->hpll.cursor);
1329         intermediate->hpll.fbc = max(optimal->hpll.fbc,
1330                                      active->hpll.fbc);
1331
1332         drm_WARN_ON(&dev_priv->drm,
1333                     (intermediate->sr.plane >
1334                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1335                      intermediate->sr.cursor >
1336                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1337                     intermediate->cxsr);
1338         drm_WARN_ON(&dev_priv->drm,
1339                     (intermediate->sr.plane >
1340                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1341                      intermediate->sr.cursor >
1342                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1343                     intermediate->hpll_en);
1344
1345         drm_WARN_ON(&dev_priv->drm,
1346                     intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1347                     intermediate->fbc_en && intermediate->cxsr);
1348         drm_WARN_ON(&dev_priv->drm,
1349                     intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1350                     intermediate->fbc_en && intermediate->hpll_en);
1351
1352 out:
1353         /*
1354          * If our intermediate WM are identical to the final WM, then we can
1355          * omit the post-vblank programming; only update if it's different.
1356          */
1357         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1358                 new_crtc_state->wm.need_postvbl_update = true;
1359
1360         return 0;
1361 }
1362
1363 static int g4x_compute_watermarks(struct intel_atomic_state *state,
1364                                   struct intel_crtc *crtc)
1365 {
1366         int ret;
1367
1368         ret = g4x_compute_pipe_wm(state, crtc);
1369         if (ret)
1370                 return ret;
1371
1372         ret = g4x_compute_intermediate_wm(state, crtc);
1373         if (ret)
1374                 return ret;
1375
1376         return 0;
1377 }
1378
1379 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1380                          struct g4x_wm_values *wm)
1381 {
1382         struct intel_crtc *crtc;
1383         int num_active_pipes = 0;
1384
1385         wm->cxsr = true;
1386         wm->hpll_en = true;
1387         wm->fbc_en = true;
1388
1389         for_each_intel_crtc(&dev_priv->drm, crtc) {
1390                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1391
1392                 if (!crtc->active)
1393                         continue;
1394
1395                 if (!wm_state->cxsr)
1396                         wm->cxsr = false;
1397                 if (!wm_state->hpll_en)
1398                         wm->hpll_en = false;
1399                 if (!wm_state->fbc_en)
1400                         wm->fbc_en = false;
1401
1402                 num_active_pipes++;
1403         }
1404
1405         if (num_active_pipes != 1) {
1406                 wm->cxsr = false;
1407                 wm->hpll_en = false;
1408                 wm->fbc_en = false;
1409         }
1410
1411         for_each_intel_crtc(&dev_priv->drm, crtc) {
1412                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1413                 enum pipe pipe = crtc->pipe;
1414
1415                 wm->pipe[pipe] = wm_state->wm;
1416                 if (crtc->active && wm->cxsr)
1417                         wm->sr = wm_state->sr;
1418                 if (crtc->active && wm->hpll_en)
1419                         wm->hpll = wm_state->hpll;
1420         }
1421 }
1422
1423 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1424 {
1425         struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;
1426         struct g4x_wm_values new_wm = {};
1427
1428         g4x_merge_wm(dev_priv, &new_wm);
1429
1430         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1431                 return;
1432
1433         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1434                 _intel_set_memory_cxsr(dev_priv, false);
1435
1436         g4x_write_wm_values(dev_priv, &new_wm);
1437
1438         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1439                 _intel_set_memory_cxsr(dev_priv, true);
1440
1441         *old_wm = new_wm;
1442 }
1443
1444 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1445                                    struct intel_crtc *crtc)
1446 {
1447         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1448         const struct intel_crtc_state *crtc_state =
1449                 intel_atomic_get_new_crtc_state(state, crtc);
1450
1451         mutex_lock(&dev_priv->display.wm.wm_mutex);
1452         crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1453         g4x_program_watermarks(dev_priv);
1454         mutex_unlock(&dev_priv->display.wm.wm_mutex);
1455 }
1456
1457 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1458                                     struct intel_crtc *crtc)
1459 {
1460         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1461         const struct intel_crtc_state *crtc_state =
1462                 intel_atomic_get_new_crtc_state(state, crtc);
1463
1464         if (!crtc_state->wm.need_postvbl_update)
1465                 return;
1466
1467         mutex_lock(&dev_priv->display.wm.wm_mutex);
1468         crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1469         g4x_program_watermarks(dev_priv);
1470         mutex_unlock(&dev_priv->display.wm.wm_mutex);
1471 }
1472
1473 /* latency must be in 0.1us units. */
1474 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1475                                    unsigned int htotal,
1476                                    unsigned int width,
1477                                    unsigned int cpp,
1478                                    unsigned int latency)
1479 {
1480         unsigned int ret;
1481
1482         ret = intel_wm_method2(pixel_rate, htotal,
1483                                width, cpp, latency);
1484         ret = DIV_ROUND_UP(ret, 64);
1485
1486         return ret;
1487 }
1488
1489 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1490 {
1491         /* all latencies in usec */
1492         dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1493
1494         dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM2 + 1;
1495
1496         if (IS_CHERRYVIEW(dev_priv)) {
1497                 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1498                 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1499
1500                 dev_priv->display.wm.num_levels = VLV_WM_LEVEL_DDR_DVFS + 1;
1501         }
1502 }
1503
1504 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1505                                 const struct intel_plane_state *plane_state,
1506                                 int level)
1507 {
1508         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1509         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1510         const struct drm_display_mode *pipe_mode =
1511                 &crtc_state->hw.pipe_mode;
1512         unsigned int pixel_rate, htotal, cpp, width, wm;
1513
1514         if (dev_priv->display.wm.pri_latency[level] == 0)
1515                 return USHRT_MAX;
1516
1517         if (!intel_wm_plane_visible(crtc_state, plane_state))
1518                 return 0;
1519
1520         cpp = plane_state->hw.fb->format->cpp[0];
1521         pixel_rate = crtc_state->pixel_rate;
1522         htotal = pipe_mode->crtc_htotal;
1523         width = drm_rect_width(&plane_state->uapi.src) >> 16;
1524
1525         if (plane->id == PLANE_CURSOR) {
1526                 /*
1527                  * FIXME the formula gives values that are
1528                  * too big for the cursor FIFO, and hence we
1529                  * would never be able to use cursors. For
1530                  * now just hardcode the watermark.
1531                  */
1532                 wm = 63;
1533         } else {
1534                 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1535                                     dev_priv->display.wm.pri_latency[level] * 10);
1536         }
1537
1538         return min_t(unsigned int, wm, USHRT_MAX);
1539 }
1540
1541 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1542 {
1543         return (active_planes & (BIT(PLANE_SPRITE0) |
1544                                  BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1545 }
1546
1547 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1548 {
1549         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1550         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1551         const struct g4x_pipe_wm *raw =
1552                 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1553         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1554         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1555         int num_active_planes = hweight8(active_planes);
1556         const int fifo_size = 511;
1557         int fifo_extra, fifo_left = fifo_size;
1558         int sprite0_fifo_extra = 0;
1559         unsigned int total_rate;
1560         enum plane_id plane_id;
1561
1562         /*
1563          * When enabling sprite0 after sprite1 has already been enabled
1564          * we tend to get an underrun unless sprite0 already has some
1565          * FIFO space allcoated. Hence we always allocate at least one
1566          * cacheline for sprite0 whenever sprite1 is enabled.
1567          *
1568          * All other plane enable sequences appear immune to this problem.
1569          */
1570         if (vlv_need_sprite0_fifo_workaround(active_planes))
1571                 sprite0_fifo_extra = 1;
1572
1573         total_rate = raw->plane[PLANE_PRIMARY] +
1574                 raw->plane[PLANE_SPRITE0] +
1575                 raw->plane[PLANE_SPRITE1] +
1576                 sprite0_fifo_extra;
1577
1578         if (total_rate > fifo_size)
1579                 return -EINVAL;
1580
1581         if (total_rate == 0)
1582                 total_rate = 1;
1583
1584         for_each_plane_id_on_crtc(crtc, plane_id) {
1585                 unsigned int rate;
1586
1587                 if ((active_planes & BIT(plane_id)) == 0) {
1588                         fifo_state->plane[plane_id] = 0;
1589                         continue;
1590                 }
1591
1592                 rate = raw->plane[plane_id];
1593                 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1594                 fifo_left -= fifo_state->plane[plane_id];
1595         }
1596
1597         fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1598         fifo_left -= sprite0_fifo_extra;
1599
1600         fifo_state->plane[PLANE_CURSOR] = 63;
1601
1602         fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1603
1604         /* spread the remainder evenly */
1605         for_each_plane_id_on_crtc(crtc, plane_id) {
1606                 int plane_extra;
1607
1608                 if (fifo_left == 0)
1609                         break;
1610
1611                 if ((active_planes & BIT(plane_id)) == 0)
1612                         continue;
1613
1614                 plane_extra = min(fifo_extra, fifo_left);
1615                 fifo_state->plane[plane_id] += plane_extra;
1616                 fifo_left -= plane_extra;
1617         }
1618
1619         drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1620
1621         /* give it all to the first plane if none are active */
1622         if (active_planes == 0) {
1623                 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1624                 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1625         }
1626
1627         return 0;
1628 }
1629
1630 /* mark all levels starting from 'level' as invalid */
1631 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1632                                struct vlv_wm_state *wm_state, int level)
1633 {
1634         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1635
1636         for (; level < dev_priv->display.wm.num_levels; level++) {
1637                 enum plane_id plane_id;
1638
1639                 for_each_plane_id_on_crtc(crtc, plane_id)
1640                         wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1641
1642                 wm_state->sr[level].cursor = USHRT_MAX;
1643                 wm_state->sr[level].plane = USHRT_MAX;
1644         }
1645 }
1646
1647 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1648 {
1649         if (wm > fifo_size)
1650                 return USHRT_MAX;
1651         else
1652                 return fifo_size - wm;
1653 }
1654
1655 /*
1656  * Starting from 'level' set all higher
1657  * levels to 'value' in the "raw" watermarks.
1658  */
1659 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1660                                  int level, enum plane_id plane_id, u16 value)
1661 {
1662         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1663         bool dirty = false;
1664
1665         for (; level < dev_priv->display.wm.num_levels; level++) {
1666                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1667
1668                 dirty |= raw->plane[plane_id] != value;
1669                 raw->plane[plane_id] = value;
1670         }
1671
1672         return dirty;
1673 }
1674
1675 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1676                                      const struct intel_plane_state *plane_state)
1677 {
1678         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1679         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1680         enum plane_id plane_id = plane->id;
1681         int level;
1682         bool dirty = false;
1683
1684         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1685                 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1686                 goto out;
1687         }
1688
1689         for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
1690                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1691                 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1692                 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1693
1694                 if (wm > max_wm)
1695                         break;
1696
1697                 dirty |= raw->plane[plane_id] != wm;
1698                 raw->plane[plane_id] = wm;
1699         }
1700
1701         /* mark all higher levels as invalid */
1702         dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1703
1704 out:
1705         if (dirty)
1706                 drm_dbg_kms(&dev_priv->drm,
1707                             "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1708                             plane->base.name,
1709                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1710                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1711                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1712
1713         return dirty;
1714 }
1715
1716 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1717                                       enum plane_id plane_id, int level)
1718 {
1719         const struct g4x_pipe_wm *raw =
1720                 &crtc_state->wm.vlv.raw[level];
1721         const struct vlv_fifo_state *fifo_state =
1722                 &crtc_state->wm.vlv.fifo_state;
1723
1724         return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1725 }
1726
1727 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1728 {
1729         return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1730                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1731                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1732                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1733 }
1734
1735 static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1736 {
1737         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1738         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1739         struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1740         const struct vlv_fifo_state *fifo_state =
1741                 &crtc_state->wm.vlv.fifo_state;
1742         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1743         int num_active_planes = hweight8(active_planes);
1744         enum plane_id plane_id;
1745         int level;
1746
1747         /* initially allow all levels */
1748         wm_state->num_levels = dev_priv->display.wm.num_levels;
1749         /*
1750          * Note that enabling cxsr with no primary/sprite planes
1751          * enabled can wedge the pipe. Hence we only allow cxsr
1752          * with exactly one enabled primary/sprite plane.
1753          */
1754         wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1755
1756         for (level = 0; level < wm_state->num_levels; level++) {
1757                 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1758                 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1759
1760                 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1761                         break;
1762
1763                 for_each_plane_id_on_crtc(crtc, plane_id) {
1764                         wm_state->wm[level].plane[plane_id] =
1765                                 vlv_invert_wm_value(raw->plane[plane_id],
1766                                                     fifo_state->plane[plane_id]);
1767                 }
1768
1769                 wm_state->sr[level].plane =
1770                         vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1771                                                  raw->plane[PLANE_SPRITE0],
1772                                                  raw->plane[PLANE_SPRITE1]),
1773                                             sr_fifo_size);
1774
1775                 wm_state->sr[level].cursor =
1776                         vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1777                                             63);
1778         }
1779
1780         if (level == 0)
1781                 return -EINVAL;
1782
1783         /* limit to only levels we can actually handle */
1784         wm_state->num_levels = level;
1785
1786         /* invalidate the higher levels */
1787         vlv_invalidate_wms(crtc, wm_state, level);
1788
1789         return 0;
1790 }
1791
1792 static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1793                                struct intel_crtc *crtc)
1794 {
1795         struct intel_crtc_state *crtc_state =
1796                 intel_atomic_get_new_crtc_state(state, crtc);
1797         const struct intel_plane_state *old_plane_state;
1798         const struct intel_plane_state *new_plane_state;
1799         struct intel_plane *plane;
1800         unsigned int dirty = 0;
1801         int i;
1802
1803         for_each_oldnew_intel_plane_in_state(state, plane,
1804                                              old_plane_state,
1805                                              new_plane_state, i) {
1806                 if (new_plane_state->hw.crtc != &crtc->base &&
1807                     old_plane_state->hw.crtc != &crtc->base)
1808                         continue;
1809
1810                 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1811                         dirty |= BIT(plane->id);
1812         }
1813
1814         /*
1815          * DSPARB registers may have been reset due to the
1816          * power well being turned off. Make sure we restore
1817          * them to a consistent state even if no primary/sprite
1818          * planes are initially active. We also force a FIFO
1819          * recomputation so that we are sure to sanitize the
1820          * FIFO setting we took over from the BIOS even if there
1821          * are no active planes on the crtc.
1822          */
1823         if (intel_crtc_needs_modeset(crtc_state))
1824                 dirty = ~0;
1825
1826         if (!dirty)
1827                 return 0;
1828
1829         /* cursor changes don't warrant a FIFO recompute */
1830         if (dirty & ~BIT(PLANE_CURSOR)) {
1831                 const struct intel_crtc_state *old_crtc_state =
1832                         intel_atomic_get_old_crtc_state(state, crtc);
1833                 const struct vlv_fifo_state *old_fifo_state =
1834                         &old_crtc_state->wm.vlv.fifo_state;
1835                 const struct vlv_fifo_state *new_fifo_state =
1836                         &crtc_state->wm.vlv.fifo_state;
1837                 int ret;
1838
1839                 ret = vlv_compute_fifo(crtc_state);
1840                 if (ret)
1841                         return ret;
1842
1843                 if (intel_crtc_needs_modeset(crtc_state) ||
1844                     memcmp(old_fifo_state, new_fifo_state,
1845                            sizeof(*new_fifo_state)) != 0)
1846                         crtc_state->fifo_changed = true;
1847         }
1848
1849         return _vlv_compute_pipe_wm(crtc_state);
1850 }
1851
1852 #define VLV_FIFO(plane, value) \
1853         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1854
1855 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1856                                    struct intel_crtc *crtc)
1857 {
1858         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1859         struct intel_uncore *uncore = &dev_priv->uncore;
1860         const struct intel_crtc_state *crtc_state =
1861                 intel_atomic_get_new_crtc_state(state, crtc);
1862         const struct vlv_fifo_state *fifo_state =
1863                 &crtc_state->wm.vlv.fifo_state;
1864         int sprite0_start, sprite1_start, fifo_size;
1865         u32 dsparb, dsparb2, dsparb3;
1866
1867         if (!crtc_state->fifo_changed)
1868                 return;
1869
1870         sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1871         sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1872         fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1873
1874         drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1875         drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
1876
1877         trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1878
1879         /*
1880          * uncore.lock serves a double purpose here. It allows us to
1881          * use the less expensive I915_{READ,WRITE}_FW() functions, and
1882          * it protects the DSPARB registers from getting clobbered by
1883          * parallel updates from multiple pipes.
1884          *
1885          * intel_pipe_update_start() has already disabled interrupts
1886          * for us, so a plain spin_lock() is sufficient here.
1887          */
1888         spin_lock(&uncore->lock);
1889
1890         switch (crtc->pipe) {
1891         case PIPE_A:
1892                 dsparb = intel_uncore_read_fw(uncore, DSPARB(dev_priv));
1893                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1894
1895                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1896                             VLV_FIFO(SPRITEB, 0xff));
1897                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1898                            VLV_FIFO(SPRITEB, sprite1_start));
1899
1900                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1901                              VLV_FIFO(SPRITEB_HI, 0x1));
1902                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1903                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1904
1905                 intel_uncore_write_fw(uncore, DSPARB(dev_priv), dsparb);
1906                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1907                 break;
1908         case PIPE_B:
1909                 dsparb = intel_uncore_read_fw(uncore, DSPARB(dev_priv));
1910                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1911
1912                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1913                             VLV_FIFO(SPRITED, 0xff));
1914                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1915                            VLV_FIFO(SPRITED, sprite1_start));
1916
1917                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
1918                              VLV_FIFO(SPRITED_HI, 0xff));
1919                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
1920                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
1921
1922                 intel_uncore_write_fw(uncore, DSPARB(dev_priv), dsparb);
1923                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1924                 break;
1925         case PIPE_C:
1926                 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
1927                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1928
1929                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
1930                              VLV_FIFO(SPRITEF, 0xff));
1931                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
1932                             VLV_FIFO(SPRITEF, sprite1_start));
1933
1934                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
1935                              VLV_FIFO(SPRITEF_HI, 0xff));
1936                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
1937                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
1938
1939                 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
1940                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1941                 break;
1942         default:
1943                 break;
1944         }
1945
1946         intel_uncore_posting_read_fw(uncore, DSPARB(dev_priv));
1947
1948         spin_unlock(&uncore->lock);
1949 }
1950
1951 #undef VLV_FIFO
1952
1953 static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
1954                                        struct intel_crtc *crtc)
1955 {
1956         struct intel_crtc_state *new_crtc_state =
1957                 intel_atomic_get_new_crtc_state(state, crtc);
1958         const struct intel_crtc_state *old_crtc_state =
1959                 intel_atomic_get_old_crtc_state(state, crtc);
1960         struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
1961         const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
1962         const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
1963         int level;
1964
1965         if (!new_crtc_state->hw.active ||
1966             intel_crtc_needs_modeset(new_crtc_state)) {
1967                 *intermediate = *optimal;
1968
1969                 intermediate->cxsr = false;
1970                 goto out;
1971         }
1972
1973         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
1974         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1975                 !new_crtc_state->disable_cxsr;
1976
1977         for (level = 0; level < intermediate->num_levels; level++) {
1978                 enum plane_id plane_id;
1979
1980                 for_each_plane_id_on_crtc(crtc, plane_id) {
1981                         intermediate->wm[level].plane[plane_id] =
1982                                 min(optimal->wm[level].plane[plane_id],
1983                                     active->wm[level].plane[plane_id]);
1984                 }
1985
1986                 intermediate->sr[level].plane = min(optimal->sr[level].plane,
1987                                                     active->sr[level].plane);
1988                 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
1989                                                      active->sr[level].cursor);
1990         }
1991
1992         vlv_invalidate_wms(crtc, intermediate, level);
1993
1994 out:
1995         /*
1996          * If our intermediate WM are identical to the final WM, then we can
1997          * omit the post-vblank programming; only update if it's different.
1998          */
1999         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2000                 new_crtc_state->wm.need_postvbl_update = true;
2001
2002         return 0;
2003 }
2004
2005 static int vlv_compute_watermarks(struct intel_atomic_state *state,
2006                                   struct intel_crtc *crtc)
2007 {
2008         int ret;
2009
2010         ret = vlv_compute_pipe_wm(state, crtc);
2011         if (ret)
2012                 return ret;
2013
2014         ret = vlv_compute_intermediate_wm(state, crtc);
2015         if (ret)
2016                 return ret;
2017
2018         return 0;
2019 }
2020
2021 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2022                          struct vlv_wm_values *wm)
2023 {
2024         struct intel_crtc *crtc;
2025         int num_active_pipes = 0;
2026
2027         wm->level = dev_priv->display.wm.num_levels - 1;
2028         wm->cxsr = true;
2029
2030         for_each_intel_crtc(&dev_priv->drm, crtc) {
2031                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2032
2033                 if (!crtc->active)
2034                         continue;
2035
2036                 if (!wm_state->cxsr)
2037                         wm->cxsr = false;
2038
2039                 num_active_pipes++;
2040                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2041         }
2042
2043         if (num_active_pipes != 1)
2044                 wm->cxsr = false;
2045
2046         if (num_active_pipes > 1)
2047                 wm->level = VLV_WM_LEVEL_PM2;
2048
2049         for_each_intel_crtc(&dev_priv->drm, crtc) {
2050                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2051                 enum pipe pipe = crtc->pipe;
2052
2053                 wm->pipe[pipe] = wm_state->wm[wm->level];
2054                 if (crtc->active && wm->cxsr)
2055                         wm->sr = wm_state->sr[wm->level];
2056
2057                 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2058                 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2059                 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2060                 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2061         }
2062 }
2063
2064 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2065 {
2066         struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;
2067         struct vlv_wm_values new_wm = {};
2068
2069         vlv_merge_wm(dev_priv, &new_wm);
2070
2071         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2072                 return;
2073
2074         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2075                 chv_set_memory_dvfs(dev_priv, false);
2076
2077         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2078                 chv_set_memory_pm5(dev_priv, false);
2079
2080         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2081                 _intel_set_memory_cxsr(dev_priv, false);
2082
2083         vlv_write_wm_values(dev_priv, &new_wm);
2084
2085         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2086                 _intel_set_memory_cxsr(dev_priv, true);
2087
2088         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2089                 chv_set_memory_pm5(dev_priv, true);
2090
2091         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2092                 chv_set_memory_dvfs(dev_priv, true);
2093
2094         *old_wm = new_wm;
2095 }
2096
2097 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2098                                    struct intel_crtc *crtc)
2099 {
2100         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2101         const struct intel_crtc_state *crtc_state =
2102                 intel_atomic_get_new_crtc_state(state, crtc);
2103
2104         mutex_lock(&dev_priv->display.wm.wm_mutex);
2105         crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2106         vlv_program_watermarks(dev_priv);
2107         mutex_unlock(&dev_priv->display.wm.wm_mutex);
2108 }
2109
2110 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2111                                     struct intel_crtc *crtc)
2112 {
2113         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2114         const struct intel_crtc_state *crtc_state =
2115                 intel_atomic_get_new_crtc_state(state, crtc);
2116
2117         if (!crtc_state->wm.need_postvbl_update)
2118                 return;
2119
2120         mutex_lock(&dev_priv->display.wm.wm_mutex);
2121         crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2122         vlv_program_watermarks(dev_priv);
2123         mutex_unlock(&dev_priv->display.wm.wm_mutex);
2124 }
2125
2126 static void i965_update_wm(struct drm_i915_private *dev_priv)
2127 {
2128         struct intel_crtc *crtc;
2129         int srwm = 1;
2130         int cursor_sr = 16;
2131         bool cxsr_enabled;
2132
2133         /* Calc sr entries for one plane configs */
2134         crtc = single_enabled_crtc(dev_priv);
2135         if (crtc) {
2136                 /* self-refresh has much higher latency */
2137                 static const int sr_latency_ns = 12000;
2138                 const struct drm_display_mode *pipe_mode =
2139                         &crtc->config->hw.pipe_mode;
2140                 const struct drm_framebuffer *fb =
2141                         crtc->base.primary->state->fb;
2142                 int pixel_rate = crtc->config->pixel_rate;
2143                 int htotal = pipe_mode->crtc_htotal;
2144                 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2145                 int cpp = fb->format->cpp[0];
2146                 int entries;
2147
2148                 entries = intel_wm_method2(pixel_rate, htotal,
2149                                            width, cpp, sr_latency_ns / 100);
2150                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2151                 srwm = I965_FIFO_SIZE - entries;
2152                 if (srwm < 0)
2153                         srwm = 1;
2154                 srwm &= 0x1ff;
2155                 drm_dbg_kms(&dev_priv->drm,
2156                             "self-refresh entries: %d, wm: %d\n",
2157                             entries, srwm);
2158
2159                 entries = intel_wm_method2(pixel_rate, htotal,
2160                                            crtc->base.cursor->state->crtc_w, 4,
2161                                            sr_latency_ns / 100);
2162                 entries = DIV_ROUND_UP(entries,
2163                                        i965_cursor_wm_info.cacheline_size) +
2164                         i965_cursor_wm_info.guard_size;
2165
2166                 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2167                 if (cursor_sr > i965_cursor_wm_info.max_wm)
2168                         cursor_sr = i965_cursor_wm_info.max_wm;
2169
2170                 drm_dbg_kms(&dev_priv->drm,
2171                             "self-refresh watermark: display plane %d "
2172                             "cursor %d\n", srwm, cursor_sr);
2173
2174                 cxsr_enabled = true;
2175         } else {
2176                 cxsr_enabled = false;
2177                 /* Turn off self refresh if both pipes are enabled */
2178                 intel_set_memory_cxsr(dev_priv, false);
2179         }
2180
2181         drm_dbg_kms(&dev_priv->drm,
2182                     "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2183                     srwm);
2184
2185         /* 965 has limitations... */
2186         intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv),
2187                            FW_WM(srwm, SR) |
2188                            FW_WM(8, CURSORB) |
2189                            FW_WM(8, PLANEB) |
2190                            FW_WM(8, PLANEA));
2191         intel_uncore_write(&dev_priv->uncore, DSPFW2(dev_priv),
2192                            FW_WM(8, CURSORA) |
2193                            FW_WM(8, PLANEC_OLD));
2194         /* update cursor SR watermark */
2195         intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv),
2196                            FW_WM(cursor_sr, CURSOR_SR));
2197
2198         if (cxsr_enabled)
2199                 intel_set_memory_cxsr(dev_priv, true);
2200 }
2201
2202 #undef FW_WM
2203
2204 static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2205                                                enum i9xx_plane_id i9xx_plane)
2206 {
2207         struct intel_display *display = &i915->display;
2208         struct intel_plane *plane;
2209
2210         for_each_intel_plane(&i915->drm, plane) {
2211                 if (plane->id == PLANE_PRIMARY &&
2212                     plane->i9xx_plane == i9xx_plane)
2213                         return intel_crtc_for_pipe(display, plane->pipe);
2214         }
2215
2216         return NULL;
2217 }
2218
2219 static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2220 {
2221         const struct intel_watermark_params *wm_info;
2222         u32 fwater_lo;
2223         u32 fwater_hi;
2224         int cwm, srwm = 1;
2225         int fifo_size;
2226         int planea_wm, planeb_wm;
2227         struct intel_crtc *crtc;
2228
2229         if (IS_I945GM(dev_priv))
2230                 wm_info = &i945_wm_info;
2231         else if (DISPLAY_VER(dev_priv) != 2)
2232                 wm_info = &i915_wm_info;
2233         else
2234                 wm_info = &i830_a_wm_info;
2235
2236         if (DISPLAY_VER(dev_priv) == 2)
2237                 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2238         else
2239                 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2240         crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2241         if (intel_crtc_active(crtc)) {
2242                 const struct drm_framebuffer *fb =
2243                         crtc->base.primary->state->fb;
2244                 int cpp;
2245
2246                 if (DISPLAY_VER(dev_priv) == 2)
2247                         cpp = 4;
2248                 else
2249                         cpp = fb->format->cpp[0];
2250
2251                 planea_wm = intel_calculate_wm(dev_priv, crtc->config->pixel_rate,
2252                                                wm_info, fifo_size, cpp,
2253                                                pessimal_latency_ns);
2254         } else {
2255                 planea_wm = fifo_size - wm_info->guard_size;
2256                 if (planea_wm > (long)wm_info->max_wm)
2257                         planea_wm = wm_info->max_wm;
2258         }
2259
2260         if (DISPLAY_VER(dev_priv) == 2)
2261                 wm_info = &i830_bc_wm_info;
2262
2263         if (DISPLAY_VER(dev_priv) == 2)
2264                 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2265         else
2266                 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2267         crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2268         if (intel_crtc_active(crtc)) {
2269                 const struct drm_framebuffer *fb =
2270                         crtc->base.primary->state->fb;
2271                 int cpp;
2272
2273                 if (DISPLAY_VER(dev_priv) == 2)
2274                         cpp = 4;
2275                 else
2276                         cpp = fb->format->cpp[0];
2277
2278                 planeb_wm = intel_calculate_wm(dev_priv, crtc->config->pixel_rate,
2279                                                wm_info, fifo_size, cpp,
2280                                                pessimal_latency_ns);
2281         } else {
2282                 planeb_wm = fifo_size - wm_info->guard_size;
2283                 if (planeb_wm > (long)wm_info->max_wm)
2284                         planeb_wm = wm_info->max_wm;
2285         }
2286
2287         drm_dbg_kms(&dev_priv->drm,
2288                     "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2289
2290         crtc = single_enabled_crtc(dev_priv);
2291         if (IS_I915GM(dev_priv) && crtc) {
2292                 struct drm_gem_object *obj;
2293
2294                 obj = intel_fb_bo(crtc->base.primary->state->fb);
2295
2296                 /* self-refresh seems busted with untiled */
2297                 if (!intel_bo_is_tiled(obj))
2298                         crtc = NULL;
2299         }
2300
2301         /*
2302          * Overlay gets an aggressive default since video jitter is bad.
2303          */
2304         cwm = 2;
2305
2306         /* Play safe and disable self-refresh before adjusting watermarks. */
2307         intel_set_memory_cxsr(dev_priv, false);
2308
2309         /* Calc sr entries for one plane configs */
2310         if (HAS_FW_BLC(dev_priv) && crtc) {
2311                 /* self-refresh has much higher latency */
2312                 static const int sr_latency_ns = 6000;
2313                 const struct drm_display_mode *pipe_mode =
2314                         &crtc->config->hw.pipe_mode;
2315                 const struct drm_framebuffer *fb =
2316                         crtc->base.primary->state->fb;
2317                 int pixel_rate = crtc->config->pixel_rate;
2318                 int htotal = pipe_mode->crtc_htotal;
2319                 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2320                 int cpp;
2321                 int entries;
2322
2323                 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2324                         cpp = 4;
2325                 else
2326                         cpp = fb->format->cpp[0];
2327
2328                 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2329                                            sr_latency_ns / 100);
2330                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2331                 drm_dbg_kms(&dev_priv->drm,
2332                             "self-refresh entries: %d\n", entries);
2333                 srwm = wm_info->fifo_size - entries;
2334                 if (srwm < 0)
2335                         srwm = 1;
2336
2337                 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2338                         intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2339                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2340                 else
2341                         intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2342         }
2343
2344         drm_dbg_kms(&dev_priv->drm,
2345                     "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2346                      planea_wm, planeb_wm, cwm, srwm);
2347
2348         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2349         fwater_hi = (cwm & 0x1f);
2350
2351         /* Set request length to 8 cachelines per fetch */
2352         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2353         fwater_hi = fwater_hi | (1 << 8);
2354
2355         intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2356         intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2357
2358         if (crtc)
2359                 intel_set_memory_cxsr(dev_priv, true);
2360 }
2361
2362 static void i845_update_wm(struct drm_i915_private *dev_priv)
2363 {
2364         struct intel_crtc *crtc;
2365         u32 fwater_lo;
2366         int planea_wm;
2367
2368         crtc = single_enabled_crtc(dev_priv);
2369         if (crtc == NULL)
2370                 return;
2371
2372         planea_wm = intel_calculate_wm(dev_priv, crtc->config->pixel_rate,
2373                                        &i845_wm_info,
2374                                        i845_get_fifo_size(dev_priv, PLANE_A),
2375                                        4, pessimal_latency_ns);
2376         fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2377         fwater_lo |= (3<<8) | planea_wm;
2378
2379         drm_dbg_kms(&dev_priv->drm,
2380                     "Setting FIFO watermarks - A: %d\n", planea_wm);
2381
2382         intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2383 }
2384
2385 /* latency must be in 0.1us units. */
2386 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2387                                    unsigned int cpp,
2388                                    unsigned int latency)
2389 {
2390         unsigned int ret;
2391
2392         ret = intel_wm_method1(pixel_rate, cpp, latency);
2393         ret = DIV_ROUND_UP(ret, 64) + 2;
2394
2395         return ret;
2396 }
2397
2398 /* latency must be in 0.1us units. */
2399 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2400                                    unsigned int htotal,
2401                                    unsigned int width,
2402                                    unsigned int cpp,
2403                                    unsigned int latency)
2404 {
2405         unsigned int ret;
2406
2407         ret = intel_wm_method2(pixel_rate, htotal,
2408                                width, cpp, latency);
2409         ret = DIV_ROUND_UP(ret, 64) + 2;
2410
2411         return ret;
2412 }
2413
2414 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2415 {
2416         /*
2417          * Neither of these should be possible since this function shouldn't be
2418          * called if the CRTC is off or the plane is invisible.  But let's be
2419          * extra paranoid to avoid a potential divide-by-zero if we screw up
2420          * elsewhere in the driver.
2421          */
2422         if (WARN_ON(!cpp))
2423                 return 0;
2424         if (WARN_ON(!horiz_pixels))
2425                 return 0;
2426
2427         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2428 }
2429
2430 struct ilk_wm_maximums {
2431         u16 pri;
2432         u16 spr;
2433         u16 cur;
2434         u16 fbc;
2435 };
2436
2437 /*
2438  * For both WM_PIPE and WM_LP.
2439  * mem_value must be in 0.1us units.
2440  */
2441 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2442                               const struct intel_plane_state *plane_state,
2443                               u32 mem_value, bool is_lp)
2444 {
2445         u32 method1, method2;
2446         int cpp;
2447
2448         if (mem_value == 0)
2449                 return U32_MAX;
2450
2451         if (!intel_wm_plane_visible(crtc_state, plane_state))
2452                 return 0;
2453
2454         cpp = plane_state->hw.fb->format->cpp[0];
2455
2456         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2457
2458         if (!is_lp)
2459                 return method1;
2460
2461         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2462                                  crtc_state->hw.pipe_mode.crtc_htotal,
2463                                  drm_rect_width(&plane_state->uapi.src) >> 16,
2464                                  cpp, mem_value);
2465
2466         return min(method1, method2);
2467 }
2468
2469 /*
2470  * For both WM_PIPE and WM_LP.
2471  * mem_value must be in 0.1us units.
2472  */
2473 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2474                               const struct intel_plane_state *plane_state,
2475                               u32 mem_value)
2476 {
2477         u32 method1, method2;
2478         int cpp;
2479
2480         if (mem_value == 0)
2481                 return U32_MAX;
2482
2483         if (!intel_wm_plane_visible(crtc_state, plane_state))
2484                 return 0;
2485
2486         cpp = plane_state->hw.fb->format->cpp[0];
2487
2488         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2489         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2490                                  crtc_state->hw.pipe_mode.crtc_htotal,
2491                                  drm_rect_width(&plane_state->uapi.src) >> 16,
2492                                  cpp, mem_value);
2493         return min(method1, method2);
2494 }
2495
2496 /*
2497  * For both WM_PIPE and WM_LP.
2498  * mem_value must be in 0.1us units.
2499  */
2500 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2501                               const struct intel_plane_state *plane_state,
2502                               u32 mem_value)
2503 {
2504         int cpp;
2505
2506         if (mem_value == 0)
2507                 return U32_MAX;
2508
2509         if (!intel_wm_plane_visible(crtc_state, plane_state))
2510                 return 0;
2511
2512         cpp = plane_state->hw.fb->format->cpp[0];
2513
2514         return ilk_wm_method2(crtc_state->pixel_rate,
2515                               crtc_state->hw.pipe_mode.crtc_htotal,
2516                               drm_rect_width(&plane_state->uapi.src) >> 16,
2517                               cpp, mem_value);
2518 }
2519
2520 /* Only for WM_LP. */
2521 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2522                               const struct intel_plane_state *plane_state,
2523                               u32 pri_val)
2524 {
2525         int cpp;
2526
2527         if (!intel_wm_plane_visible(crtc_state, plane_state))
2528                 return 0;
2529
2530         cpp = plane_state->hw.fb->format->cpp[0];
2531
2532         return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2533                           cpp);
2534 }
2535
2536 static unsigned int
2537 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2538 {
2539         if (DISPLAY_VER(dev_priv) >= 8)
2540                 return 3072;
2541         else if (DISPLAY_VER(dev_priv) >= 7)
2542                 return 768;
2543         else
2544                 return 512;
2545 }
2546
2547 static unsigned int
2548 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2549                      int level, bool is_sprite)
2550 {
2551         if (DISPLAY_VER(dev_priv) >= 8)
2552                 /* BDW primary/sprite plane watermarks */
2553                 return level == 0 ? 255 : 2047;
2554         else if (DISPLAY_VER(dev_priv) >= 7)
2555                 /* IVB/HSW primary/sprite plane watermarks */
2556                 return level == 0 ? 127 : 1023;
2557         else if (!is_sprite)
2558                 /* ILK/SNB primary plane watermarks */
2559                 return level == 0 ? 127 : 511;
2560         else
2561                 /* ILK/SNB sprite plane watermarks */
2562                 return level == 0 ? 63 : 255;
2563 }
2564
2565 static unsigned int
2566 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2567 {
2568         if (DISPLAY_VER(dev_priv) >= 7)
2569                 return level == 0 ? 63 : 255;
2570         else
2571                 return level == 0 ? 31 : 63;
2572 }
2573
2574 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2575 {
2576         if (DISPLAY_VER(dev_priv) >= 8)
2577                 return 31;
2578         else
2579                 return 15;
2580 }
2581
2582 /* Calculate the maximum primary/sprite plane watermark */
2583 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2584                                      int level,
2585                                      const struct intel_wm_config *config,
2586                                      enum intel_ddb_partitioning ddb_partitioning,
2587                                      bool is_sprite)
2588 {
2589         unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2590
2591         /* if sprites aren't enabled, sprites get nothing */
2592         if (is_sprite && !config->sprites_enabled)
2593                 return 0;
2594
2595         /* HSW allows LP1+ watermarks even with multiple pipes */
2596         if (level == 0 || config->num_pipes_active > 1) {
2597                 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2598
2599                 /*
2600                  * For some reason the non self refresh
2601                  * FIFO size is only half of the self
2602                  * refresh FIFO size on ILK/SNB.
2603                  */
2604                 if (DISPLAY_VER(dev_priv) < 7)
2605                         fifo_size /= 2;
2606         }
2607
2608         if (config->sprites_enabled) {
2609                 /* level 0 is always calculated with 1:1 split */
2610                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2611                         if (is_sprite)
2612                                 fifo_size *= 5;
2613                         fifo_size /= 6;
2614                 } else {
2615                         fifo_size /= 2;
2616                 }
2617         }
2618
2619         /* clamp to max that the registers can hold */
2620         return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2621 }
2622
2623 /* Calculate the maximum cursor plane watermark */
2624 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2625                                       int level,
2626                                       const struct intel_wm_config *config)
2627 {
2628         /* HSW LP1+ watermarks w/ multiple pipes */
2629         if (level > 0 && config->num_pipes_active > 1)
2630                 return 64;
2631
2632         /* otherwise just report max that registers can hold */
2633         return ilk_cursor_wm_reg_max(dev_priv, level);
2634 }
2635
2636 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2637                                     int level,
2638                                     const struct intel_wm_config *config,
2639                                     enum intel_ddb_partitioning ddb_partitioning,
2640                                     struct ilk_wm_maximums *max)
2641 {
2642         max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2643         max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2644         max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2645         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2646 }
2647
2648 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2649                                         int level,
2650                                         struct ilk_wm_maximums *max)
2651 {
2652         max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2653         max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2654         max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2655         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2656 }
2657
2658 static bool ilk_validate_wm_level(struct drm_i915_private *i915,
2659                                   int level,
2660                                   const struct ilk_wm_maximums *max,
2661                                   struct intel_wm_level *result)
2662 {
2663         bool ret;
2664
2665         /* already determined to be invalid? */
2666         if (!result->enable)
2667                 return false;
2668
2669         result->enable = result->pri_val <= max->pri &&
2670                          result->spr_val <= max->spr &&
2671                          result->cur_val <= max->cur;
2672
2673         ret = result->enable;
2674
2675         /*
2676          * HACK until we can pre-compute everything,
2677          * and thus fail gracefully if LP0 watermarks
2678          * are exceeded...
2679          */
2680         if (level == 0 && !result->enable) {
2681                 if (result->pri_val > max->pri)
2682                         drm_dbg_kms(&i915->drm,
2683                                     "Primary WM%d too large %u (max %u)\n",
2684                                     level, result->pri_val, max->pri);
2685                 if (result->spr_val > max->spr)
2686                         drm_dbg_kms(&i915->drm,
2687                                     "Sprite WM%d too large %u (max %u)\n",
2688                                     level, result->spr_val, max->spr);
2689                 if (result->cur_val > max->cur)
2690                         drm_dbg_kms(&i915->drm,
2691                                     "Cursor WM%d too large %u (max %u)\n",
2692                                     level, result->cur_val, max->cur);
2693
2694                 result->pri_val = min_t(u32, result->pri_val, max->pri);
2695                 result->spr_val = min_t(u32, result->spr_val, max->spr);
2696                 result->cur_val = min_t(u32, result->cur_val, max->cur);
2697                 result->enable = true;
2698         }
2699
2700         return ret;
2701 }
2702
2703 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2704                                  const struct intel_crtc *crtc,
2705                                  int level,
2706                                  struct intel_crtc_state *crtc_state,
2707                                  const struct intel_plane_state *pristate,
2708                                  const struct intel_plane_state *sprstate,
2709                                  const struct intel_plane_state *curstate,
2710                                  struct intel_wm_level *result)
2711 {
2712         u16 pri_latency = dev_priv->display.wm.pri_latency[level];
2713         u16 spr_latency = dev_priv->display.wm.spr_latency[level];
2714         u16 cur_latency = dev_priv->display.wm.cur_latency[level];
2715
2716         /* WM1+ latency values stored in 0.5us units */
2717         if (level > 0) {
2718                 pri_latency *= 5;
2719                 spr_latency *= 5;
2720                 cur_latency *= 5;
2721         }
2722
2723         if (pristate) {
2724                 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2725                                                      pri_latency, level);
2726                 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2727         }
2728
2729         if (sprstate)
2730                 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2731
2732         if (curstate)
2733                 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2734
2735         result->enable = true;
2736 }
2737
2738 static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2739 {
2740         u64 sskpd;
2741
2742         i915->display.wm.num_levels = 5;
2743
2744         sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);
2745
2746         wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2747         if (wm[0] == 0)
2748                 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2749         wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2750         wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2751         wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2752         wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2753 }
2754
2755 static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2756 {
2757         u32 sskpd;
2758
2759         i915->display.wm.num_levels = 4;
2760
2761         sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);
2762
2763         wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2764         wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2765         wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2766         wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2767 }
2768
2769 static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2770 {
2771         u32 mltr;
2772
2773         i915->display.wm.num_levels = 3;
2774
2775         mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);
2776
2777         /* ILK primary LP0 latency is 700 ns */
2778         wm[0] = 7;
2779         wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2780         wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2781 }
2782
2783 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2784                                        u16 wm[5])
2785 {
2786         /* ILK sprite LP0 latency is 1300 ns */
2787         if (DISPLAY_VER(dev_priv) == 5)
2788                 wm[0] = 13;
2789 }
2790
2791 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2792                                        u16 wm[5])
2793 {
2794         /* ILK cursor LP0 latency is 1300 ns */
2795         if (DISPLAY_VER(dev_priv) == 5)
2796                 wm[0] = 13;
2797 }
2798
2799 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2800                                     u16 wm[5], u16 min)
2801 {
2802         int level;
2803
2804         if (wm[0] >= min)
2805                 return false;
2806
2807         wm[0] = max(wm[0], min);
2808         for (level = 1; level < dev_priv->display.wm.num_levels; level++)
2809                 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2810
2811         return true;
2812 }
2813
2814 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2815 {
2816         bool changed;
2817
2818         /*
2819          * The BIOS provided WM memory latency values are often
2820          * inadequate for high resolution displays. Adjust them.
2821          */
2822         changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);
2823         changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);
2824         changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);
2825
2826         if (!changed)
2827                 return;
2828
2829         drm_dbg_kms(&dev_priv->drm,
2830                     "WM latency values increased to avoid potential underruns\n");
2831         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2832         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2833         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2834 }
2835
2836 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
2837 {
2838         /*
2839          * On some SNB machines (Thinkpad X220 Tablet at least)
2840          * LP3 usage can cause vblank interrupts to be lost.
2841          * The DEIIR bit will go high but it looks like the CPU
2842          * never gets interrupted.
2843          *
2844          * It's not clear whether other interrupt source could
2845          * be affected or if this is somehow limited to vblank
2846          * interrupts only. To play it safe we disable LP3
2847          * watermarks entirely.
2848          */
2849         if (dev_priv->display.wm.pri_latency[3] == 0 &&
2850             dev_priv->display.wm.spr_latency[3] == 0 &&
2851             dev_priv->display.wm.cur_latency[3] == 0)
2852                 return;
2853
2854         dev_priv->display.wm.pri_latency[3] = 0;
2855         dev_priv->display.wm.spr_latency[3] = 0;
2856         dev_priv->display.wm.cur_latency[3] = 0;
2857
2858         drm_dbg_kms(&dev_priv->drm,
2859                     "LP3 watermarks disabled due to potential for lost interrupts\n");
2860         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2861         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2862         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2863 }
2864
2865 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
2866 {
2867         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2868                 hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2869         else if (DISPLAY_VER(dev_priv) >= 6)
2870                 snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2871         else
2872                 ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2873
2874         memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,
2875                sizeof(dev_priv->display.wm.pri_latency));
2876         memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,
2877                sizeof(dev_priv->display.wm.pri_latency));
2878
2879         intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);
2880         intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);
2881
2882         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2883         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2884         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2885
2886         if (DISPLAY_VER(dev_priv) == 6) {
2887                 snb_wm_latency_quirk(dev_priv);
2888                 snb_wm_lp3_irq_quirk(dev_priv);
2889         }
2890 }
2891
2892 static bool ilk_validate_pipe_wm(struct drm_i915_private *dev_priv,
2893                                  struct intel_pipe_wm *pipe_wm)
2894 {
2895         /* LP0 watermark maximums depend on this pipe alone */
2896         const struct intel_wm_config config = {
2897                 .num_pipes_active = 1,
2898                 .sprites_enabled = pipe_wm->sprites_enabled,
2899                 .sprites_scaled = pipe_wm->sprites_scaled,
2900         };
2901         struct ilk_wm_maximums max;
2902
2903         /* LP0 watermarks always use 1/2 DDB partitioning */
2904         ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
2905
2906         /* At least LP0 must be valid */
2907         if (!ilk_validate_wm_level(dev_priv, 0, &max, &pipe_wm->wm[0])) {
2908                 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
2909                 return false;
2910         }
2911
2912         return true;
2913 }
2914
2915 /* Compute new watermarks for the pipe */
2916 static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
2917                                struct intel_crtc *crtc)
2918 {
2919         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2920         struct intel_crtc_state *crtc_state =
2921                 intel_atomic_get_new_crtc_state(state, crtc);
2922         struct intel_pipe_wm *pipe_wm;
2923         struct intel_plane *plane;
2924         const struct intel_plane_state *plane_state;
2925         const struct intel_plane_state *pristate = NULL;
2926         const struct intel_plane_state *sprstate = NULL;
2927         const struct intel_plane_state *curstate = NULL;
2928         struct ilk_wm_maximums max;
2929         int level, usable_level;
2930
2931         pipe_wm = &crtc_state->wm.ilk.optimal;
2932
2933         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
2934                 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
2935                         pristate = plane_state;
2936                 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
2937                         sprstate = plane_state;
2938                 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
2939                         curstate = plane_state;
2940         }
2941
2942         pipe_wm->pipe_enabled = crtc_state->hw.active;
2943         pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
2944         pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
2945
2946         usable_level = dev_priv->display.wm.num_levels - 1;
2947
2948         /* ILK/SNB: LP2+ watermarks only w/o sprites */
2949         if (DISPLAY_VER(dev_priv) < 7 && pipe_wm->sprites_enabled)
2950                 usable_level = 1;
2951
2952         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
2953         if (pipe_wm->sprites_scaled)
2954                 usable_level = 0;
2955
2956         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
2957         ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
2958                              pristate, sprstate, curstate, &pipe_wm->wm[0]);
2959
2960         if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
2961                 return -EINVAL;
2962
2963         ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
2964
2965         for (level = 1; level <= usable_level; level++) {
2966                 struct intel_wm_level *wm = &pipe_wm->wm[level];
2967
2968                 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
2969                                      pristate, sprstate, curstate, wm);
2970
2971                 /*
2972                  * Disable any watermark level that exceeds the
2973                  * register maximums since such watermarks are
2974                  * always invalid.
2975                  */
2976                 if (!ilk_validate_wm_level(dev_priv, level, &max, wm)) {
2977                         memset(wm, 0, sizeof(*wm));
2978                         break;
2979                 }
2980         }
2981
2982         return 0;
2983 }
2984
2985 /*
2986  * Build a set of 'intermediate' watermark values that satisfy both the old
2987  * state and the new state.  These can be programmed to the hardware
2988  * immediately.
2989  */
2990 static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
2991                                        struct intel_crtc *crtc)
2992 {
2993         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2994         struct intel_crtc_state *new_crtc_state =
2995                 intel_atomic_get_new_crtc_state(state, crtc);
2996         const struct intel_crtc_state *old_crtc_state =
2997                 intel_atomic_get_old_crtc_state(state, crtc);
2998         struct intel_pipe_wm *intermediate = &new_crtc_state->wm.ilk.intermediate;
2999         const struct intel_pipe_wm *optimal = &new_crtc_state->wm.ilk.optimal;
3000         const struct intel_pipe_wm *active = &old_crtc_state->wm.ilk.optimal;
3001         int level;
3002
3003         /*
3004          * Start with the final, target watermarks, then combine with the
3005          * currently active watermarks to get values that are safe both before
3006          * and after the vblank.
3007          */
3008         *intermediate = *optimal;
3009         if (!new_crtc_state->hw.active ||
3010             intel_crtc_needs_modeset(new_crtc_state) ||
3011             state->skip_intermediate_wm)
3012                 return 0;
3013
3014         intermediate->pipe_enabled |= active->pipe_enabled;
3015         intermediate->sprites_enabled |= active->sprites_enabled;
3016         intermediate->sprites_scaled |= active->sprites_scaled;
3017
3018         for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
3019                 struct intel_wm_level *intermediate_wm = &intermediate->wm[level];
3020                 const struct intel_wm_level *active_wm = &active->wm[level];
3021
3022                 intermediate_wm->enable &= active_wm->enable;
3023                 intermediate_wm->pri_val = max(intermediate_wm->pri_val,
3024                                                active_wm->pri_val);
3025                 intermediate_wm->spr_val = max(intermediate_wm->spr_val,
3026                                                active_wm->spr_val);
3027                 intermediate_wm->cur_val = max(intermediate_wm->cur_val,
3028                                                active_wm->cur_val);
3029                 intermediate_wm->fbc_val = max(intermediate_wm->fbc_val,
3030                                                active_wm->fbc_val);
3031         }
3032
3033         /*
3034          * We need to make sure that these merged watermark values are
3035          * actually a valid configuration themselves.  If they're not,
3036          * there's no safe way to transition from the old state to
3037          * the new state, so we need to fail the atomic transaction.
3038          */
3039         if (!ilk_validate_pipe_wm(dev_priv, intermediate))
3040                 return -EINVAL;
3041
3042         /*
3043          * If our intermediate WM are identical to the final WM, then we can
3044          * omit the post-vblank programming; only update if it's different.
3045          */
3046         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
3047                 new_crtc_state->wm.need_postvbl_update = true;
3048
3049         return 0;
3050 }
3051
3052 static int ilk_compute_watermarks(struct intel_atomic_state *state,
3053                                   struct intel_crtc *crtc)
3054 {
3055         int ret;
3056
3057         ret = ilk_compute_pipe_wm(state, crtc);
3058         if (ret)
3059                 return ret;
3060
3061         ret = ilk_compute_intermediate_wm(state, crtc);
3062         if (ret)
3063                 return ret;
3064
3065         return 0;
3066 }
3067
3068 /*
3069  * Merge the watermarks from all active pipes for a specific level.
3070  */
3071 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3072                                int level,
3073                                struct intel_wm_level *ret_wm)
3074 {
3075         const struct intel_crtc *crtc;
3076
3077         ret_wm->enable = true;
3078
3079         for_each_intel_crtc(&dev_priv->drm, crtc) {
3080                 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3081                 const struct intel_wm_level *wm = &active->wm[level];
3082
3083                 if (!active->pipe_enabled)
3084                         continue;
3085
3086                 /*
3087                  * The watermark values may have been used in the past,
3088                  * so we must maintain them in the registers for some
3089                  * time even if the level is now disabled.
3090                  */
3091                 if (!wm->enable)
3092                         ret_wm->enable = false;
3093
3094                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3095                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3096                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3097                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3098         }
3099 }
3100
3101 /*
3102  * Merge all low power watermarks for all active pipes.
3103  */
3104 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3105                          const struct intel_wm_config *config,
3106                          const struct ilk_wm_maximums *max,
3107                          struct intel_pipe_wm *merged)
3108 {
3109         int level, num_levels = dev_priv->display.wm.num_levels;
3110         int last_enabled_level = num_levels - 1;
3111
3112         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3113         if ((DISPLAY_VER(dev_priv) < 7 || IS_IVYBRIDGE(dev_priv)) &&
3114             config->num_pipes_active > 1)
3115                 last_enabled_level = 0;
3116
3117         /* ILK: FBC WM must be disabled always */
3118         merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3119
3120         /* merge each WM1+ level */
3121         for (level = 1; level < num_levels; level++) {
3122                 struct intel_wm_level *wm = &merged->wm[level];
3123
3124                 ilk_merge_wm_level(dev_priv, level, wm);
3125
3126                 if (level > last_enabled_level)
3127                         wm->enable = false;
3128                 else if (!ilk_validate_wm_level(dev_priv, level, max, wm))
3129                         /* make sure all following levels get disabled */
3130                         last_enabled_level = level - 1;
3131
3132                 /*
3133                  * The spec says it is preferred to disable
3134                  * FBC WMs instead of disabling a WM level.
3135                  */
3136                 if (wm->fbc_val > max->fbc) {
3137                         if (wm->enable)
3138                                 merged->fbc_wm_enabled = false;
3139                         wm->fbc_val = 0;
3140                 }
3141         }
3142
3143         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3144         if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
3145             dev_priv->display.params.enable_fbc && !merged->fbc_wm_enabled) {
3146                 for (level = 2; level < num_levels; level++) {
3147                         struct intel_wm_level *wm = &merged->wm[level];
3148
3149                         wm->enable = false;
3150                 }
3151         }
3152 }
3153
3154 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3155 {
3156         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3157         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3158 }
3159
3160 /* The value we need to program into the WM_LPx latency field */
3161 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3162                                       int level)
3163 {
3164         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3165                 return 2 * level;
3166         else
3167                 return dev_priv->display.wm.pri_latency[level];
3168 }
3169
3170 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3171                                    const struct intel_pipe_wm *merged,
3172                                    enum intel_ddb_partitioning partitioning,
3173                                    struct ilk_wm_values *results)
3174 {
3175         struct intel_crtc *crtc;
3176         int level, wm_lp;
3177
3178         results->enable_fbc_wm = merged->fbc_wm_enabled;
3179         results->partitioning = partitioning;
3180
3181         /* LP1+ register values */
3182         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3183                 const struct intel_wm_level *r;
3184
3185                 level = ilk_wm_lp_to_level(wm_lp, merged);
3186
3187                 r = &merged->wm[level];
3188
3189                 /*
3190                  * Maintain the watermark values even if the level is
3191                  * disabled. Doing otherwise could cause underruns.
3192                  */
3193                 results->wm_lp[wm_lp - 1] =
3194                         WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3195                         WM_LP_PRIMARY(r->pri_val) |
3196                         WM_LP_CURSOR(r->cur_val);
3197
3198                 if (r->enable)
3199                         results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3200
3201                 if (DISPLAY_VER(dev_priv) >= 8)
3202                         results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3203                 else
3204                         results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3205
3206                 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3207
3208                 /*
3209                  * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3210                  * level is disabled. Doing otherwise could cause underruns.
3211                  */
3212                 if (DISPLAY_VER(dev_priv) < 7 && r->spr_val) {
3213                         drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3214                         results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3215                 }
3216         }
3217
3218         /* LP0 register values */
3219         for_each_intel_crtc(&dev_priv->drm, crtc) {
3220                 enum pipe pipe = crtc->pipe;
3221                 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3222                 const struct intel_wm_level *r = &pipe_wm->wm[0];
3223
3224                 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3225                         continue;
3226
3227                 results->wm_pipe[pipe] =
3228                         WM0_PIPE_PRIMARY(r->pri_val) |
3229                         WM0_PIPE_SPRITE(r->spr_val) |
3230                         WM0_PIPE_CURSOR(r->cur_val);
3231         }
3232 }
3233
3234 /*
3235  * Find the result with the highest level enabled. Check for enable_fbc_wm in
3236  * case both are at the same level. Prefer r1 in case they're the same.
3237  */
3238 static struct intel_pipe_wm *
3239 ilk_find_best_result(struct drm_i915_private *dev_priv,
3240                      struct intel_pipe_wm *r1,
3241                      struct intel_pipe_wm *r2)
3242 {
3243         int level, level1 = 0, level2 = 0;
3244
3245         for (level = 1; level < dev_priv->display.wm.num_levels; level++) {
3246                 if (r1->wm[level].enable)
3247                         level1 = level;
3248                 if (r2->wm[level].enable)
3249                         level2 = level;
3250         }
3251
3252         if (level1 == level2) {
3253                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3254                         return r2;
3255                 else
3256                         return r1;
3257         } else if (level1 > level2) {
3258                 return r1;
3259         } else {
3260                 return r2;
3261         }
3262 }
3263
3264 /* dirty bits used to track which watermarks need changes */
3265 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3266 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3267 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3268 #define WM_DIRTY_FBC (1 << 24)
3269 #define WM_DIRTY_DDB (1 << 25)
3270
3271 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3272                                          const struct ilk_wm_values *old,
3273                                          const struct ilk_wm_values *new)
3274 {
3275         unsigned int dirty = 0;
3276         enum pipe pipe;
3277         int wm_lp;
3278
3279         for_each_pipe(dev_priv, pipe) {
3280                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3281                         dirty |= WM_DIRTY_PIPE(pipe);
3282                         /* Must disable LP1+ watermarks too */
3283                         dirty |= WM_DIRTY_LP_ALL;
3284                 }
3285         }
3286
3287         if (old->enable_fbc_wm != new->enable_fbc_wm) {
3288                 dirty |= WM_DIRTY_FBC;
3289                 /* Must disable LP1+ watermarks too */
3290                 dirty |= WM_DIRTY_LP_ALL;
3291         }
3292
3293         if (old->partitioning != new->partitioning) {
3294                 dirty |= WM_DIRTY_DDB;
3295                 /* Must disable LP1+ watermarks too */
3296                 dirty |= WM_DIRTY_LP_ALL;
3297         }
3298
3299         /* LP1+ watermarks already deemed dirty, no need to continue */
3300         if (dirty & WM_DIRTY_LP_ALL)
3301                 return dirty;
3302
3303         /* Find the lowest numbered LP1+ watermark in need of an update... */
3304         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3305                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3306                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3307                         break;
3308         }
3309
3310         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3311         for (; wm_lp <= 3; wm_lp++)
3312                 dirty |= WM_DIRTY_LP(wm_lp);
3313
3314         return dirty;
3315 }
3316
3317 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3318                                unsigned int dirty)
3319 {
3320         struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3321         bool changed = false;
3322
3323         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3324                 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3325                 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3326                 changed = true;
3327         }
3328         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3329                 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3330                 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3331                 changed = true;
3332         }
3333         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3334                 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3335                 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3336                 changed = true;
3337         }
3338
3339         /*
3340          * Don't touch WM_LP_SPRITE_ENABLE here.
3341          * Doing so could cause underruns.
3342          */
3343
3344         return changed;
3345 }
3346
3347 /*
3348  * The spec says we shouldn't write when we don't need, because every write
3349  * causes WMs to be re-evaluated, expending some power.
3350  */
3351 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3352                                 struct ilk_wm_values *results)
3353 {
3354         struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3355         unsigned int dirty;
3356
3357         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3358         if (!dirty)
3359                 return;
3360
3361         _ilk_disable_lp_wm(dev_priv, dirty);
3362
3363         if (dirty & WM_DIRTY_PIPE(PIPE_A))
3364                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3365         if (dirty & WM_DIRTY_PIPE(PIPE_B))
3366                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3367         if (dirty & WM_DIRTY_PIPE(PIPE_C))
3368                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3369
3370         if (dirty & WM_DIRTY_DDB) {
3371                 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3372                         intel_uncore_rmw(&dev_priv->uncore, WM_MISC, WM_MISC_DATA_PARTITION_5_6,
3373                                          results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3374                                          WM_MISC_DATA_PARTITION_5_6);
3375                 else
3376                         intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,
3377                                          results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3378                                          DISP_DATA_PARTITION_5_6);
3379         }
3380
3381         if (dirty & WM_DIRTY_FBC)
3382                 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, DISP_FBC_WM_DIS,
3383                                  results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);
3384
3385         if (dirty & WM_DIRTY_LP(1) &&
3386             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3387                 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3388
3389         if (DISPLAY_VER(dev_priv) >= 7) {
3390                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3391                         intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3392                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3393                         intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3394         }
3395
3396         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3397                 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3398         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3399                 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3400         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3401                 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3402
3403         dev_priv->display.wm.hw = *results;
3404 }
3405
3406 bool ilk_disable_cxsr(struct drm_i915_private *dev_priv)
3407 {
3408         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3409 }
3410
3411 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
3412                                   struct intel_wm_config *config)
3413 {
3414         struct intel_crtc *crtc;
3415
3416         /* Compute the currently _active_ config */
3417         for_each_intel_crtc(&dev_priv->drm, crtc) {
3418                 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3419
3420                 if (!wm->pipe_enabled)
3421                         continue;
3422
3423                 config->sprites_enabled |= wm->sprites_enabled;
3424                 config->sprites_scaled |= wm->sprites_scaled;
3425                 config->num_pipes_active++;
3426         }
3427 }
3428
3429 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
3430 {
3431         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3432         struct ilk_wm_maximums max;
3433         struct intel_wm_config config = {};
3434         struct ilk_wm_values results = {};
3435         enum intel_ddb_partitioning partitioning;
3436
3437         ilk_compute_wm_config(dev_priv, &config);
3438
3439         ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
3440         ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
3441
3442         /* 5/6 split only in single pipe config on IVB+ */
3443         if (DISPLAY_VER(dev_priv) >= 7 &&
3444             config.num_pipes_active == 1 && config.sprites_enabled) {
3445                 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
3446                 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
3447
3448                 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
3449         } else {
3450                 best_lp_wm = &lp_wm_1_2;
3451         }
3452
3453         partitioning = (best_lp_wm == &lp_wm_1_2) ?
3454                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3455
3456         ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
3457
3458         ilk_write_wm_values(dev_priv, &results);
3459 }
3460
3461 static void ilk_initial_watermarks(struct intel_atomic_state *state,
3462                                    struct intel_crtc *crtc)
3463 {
3464         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3465         const struct intel_crtc_state *crtc_state =
3466                 intel_atomic_get_new_crtc_state(state, crtc);
3467
3468         mutex_lock(&dev_priv->display.wm.wm_mutex);
3469         crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
3470         ilk_program_watermarks(dev_priv);
3471         mutex_unlock(&dev_priv->display.wm.wm_mutex);
3472 }
3473
3474 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
3475                                     struct intel_crtc *crtc)
3476 {
3477         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3478         const struct intel_crtc_state *crtc_state =
3479                 intel_atomic_get_new_crtc_state(state, crtc);
3480
3481         if (!crtc_state->wm.need_postvbl_update)
3482                 return;
3483
3484         mutex_lock(&dev_priv->display.wm.wm_mutex);
3485         crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
3486         ilk_program_watermarks(dev_priv);
3487         mutex_unlock(&dev_priv->display.wm.wm_mutex);
3488 }
3489
3490 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
3491 {
3492         struct drm_device *dev = crtc->base.dev;
3493         struct drm_i915_private *dev_priv = to_i915(dev);
3494         struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3495         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
3496         struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
3497         enum pipe pipe = crtc->pipe;
3498
3499         hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
3500
3501         memset(active, 0, sizeof(*active));
3502
3503         active->pipe_enabled = crtc->active;
3504
3505         if (active->pipe_enabled) {
3506                 u32 tmp = hw->wm_pipe[pipe];
3507
3508                 /*
3509                  * For active pipes LP0 watermark is marked as
3510                  * enabled, and LP1+ watermaks as disabled since
3511                  * we can't really reverse compute them in case
3512                  * multiple pipes are active.
3513                  */
3514                 active->wm[0].enable = true;
3515                 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
3516                 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
3517                 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
3518         } else {
3519                 int level;
3520
3521                 /*
3522                  * For inactive pipes, all watermark levels
3523                  * should be marked as enabled but zeroed,
3524                  * which is what we'd compute them to.
3525                  */
3526                 for (level = 0; level < dev_priv->display.wm.num_levels; level++)
3527                         active->wm[level].enable = true;
3528         }
3529
3530         crtc->wm.active.ilk = *active;
3531 }
3532
3533 static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_state *state)
3534 {
3535         struct drm_plane *plane;
3536         struct intel_crtc *crtc;
3537
3538         for_each_intel_crtc(state->dev, crtc) {
3539                 struct intel_crtc_state *crtc_state;
3540
3541                 crtc_state = intel_atomic_get_crtc_state(state, crtc);
3542                 if (IS_ERR(crtc_state))
3543                         return PTR_ERR(crtc_state);
3544
3545                 if (crtc_state->hw.active) {
3546                         /*
3547                          * Preserve the inherited flag to avoid
3548                          * taking the full modeset path.
3549                          */
3550                         crtc_state->inherited = true;
3551                 }
3552         }
3553
3554         drm_for_each_plane(plane, state->dev) {
3555                 struct drm_plane_state *plane_state;
3556
3557                 plane_state = drm_atomic_get_plane_state(state, plane);
3558                 if (IS_ERR(plane_state))
3559                         return PTR_ERR(plane_state);
3560         }
3561
3562         return 0;
3563 }
3564
3565 /*
3566  * Calculate what we think the watermarks should be for the state we've read
3567  * out of the hardware and then immediately program those watermarks so that
3568  * we ensure the hardware settings match our internal state.
3569  *
3570  * We can calculate what we think WM's should be by creating a duplicate of the
3571  * current state (which was constructed during hardware readout) and running it
3572  * through the atomic check code to calculate new watermark values in the
3573  * state object.
3574  */
3575 void ilk_wm_sanitize(struct drm_i915_private *dev_priv)
3576 {
3577         struct drm_atomic_state *state;
3578         struct intel_atomic_state *intel_state;
3579         struct intel_crtc *crtc;
3580         struct intel_crtc_state *crtc_state;
3581         struct drm_modeset_acquire_ctx ctx;
3582         int ret;
3583         int i;
3584
3585         /* Only supported on platforms that use atomic watermark design */
3586         if (!dev_priv->display.funcs.wm->optimize_watermarks)
3587                 return;
3588
3589         if (drm_WARN_ON(&dev_priv->drm, DISPLAY_VER(dev_priv) >= 9))
3590                 return;
3591
3592         state = drm_atomic_state_alloc(&dev_priv->drm);
3593         if (drm_WARN_ON(&dev_priv->drm, !state))
3594                 return;
3595
3596         intel_state = to_intel_atomic_state(state);
3597
3598         drm_modeset_acquire_init(&ctx, 0);
3599
3600         state->acquire_ctx = &ctx;
3601         to_intel_atomic_state(state)->internal = true;
3602
3603 retry:
3604         /*
3605          * Hardware readout is the only time we don't want to calculate
3606          * intermediate watermarks (since we don't trust the current
3607          * watermarks).
3608          */
3609         if (!HAS_GMCH(dev_priv))
3610                 intel_state->skip_intermediate_wm = true;
3611
3612         ret = ilk_sanitize_watermarks_add_affected(state);
3613         if (ret)
3614                 goto fail;
3615
3616         ret = intel_atomic_check(&dev_priv->drm, state);
3617         if (ret)
3618                 goto fail;
3619
3620         /* Write calculated watermark values back */
3621         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
3622                 crtc_state->wm.need_postvbl_update = true;
3623                 intel_optimize_watermarks(intel_state, crtc);
3624
3625                 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
3626         }
3627
3628 fail:
3629         if (ret == -EDEADLK) {
3630                 drm_atomic_state_clear(state);
3631                 drm_modeset_backoff(&ctx);
3632                 goto retry;
3633         }
3634
3635         /*
3636          * If we fail here, it means that the hardware appears to be
3637          * programmed in a way that shouldn't be possible, given our
3638          * understanding of watermark requirements.  This might mean a
3639          * mistake in the hardware readout code or a mistake in the
3640          * watermark calculations for a given platform.  Raise a WARN
3641          * so that this is noticeable.
3642          *
3643          * If this actually happens, we'll have to just leave the
3644          * BIOS-programmed watermarks untouched and hope for the best.
3645          */
3646         drm_WARN(&dev_priv->drm, ret,
3647                  "Could not determine valid watermarks for inherited state\n");
3648
3649         drm_atomic_state_put(state);
3650
3651         drm_modeset_drop_locks(&ctx);
3652         drm_modeset_acquire_fini(&ctx);
3653 }
3654
3655 #define _FW_WM(value, plane) \
3656         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3657 #define _FW_WM_VLV(value, plane) \
3658         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3659
3660 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
3661                                struct g4x_wm_values *wm)
3662 {
3663         u32 tmp;
3664
3665         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1(dev_priv));
3666         wm->sr.plane = _FW_WM(tmp, SR);
3667         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3668         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
3669         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
3670
3671         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2(dev_priv));
3672         wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
3673         wm->sr.fbc = _FW_WM(tmp, FBC_SR);
3674         wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
3675         wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
3676         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3677         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
3678
3679         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));
3680         wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
3681         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3682         wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
3683         wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
3684 }
3685
3686 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
3687                                struct vlv_wm_values *wm)
3688 {
3689         enum pipe pipe;
3690         u32 tmp;
3691
3692         for_each_pipe(dev_priv, pipe) {
3693                 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
3694
3695                 wm->ddl[pipe].plane[PLANE_PRIMARY] =
3696                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3697                 wm->ddl[pipe].plane[PLANE_CURSOR] =
3698                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3699                 wm->ddl[pipe].plane[PLANE_SPRITE0] =
3700                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3701                 wm->ddl[pipe].plane[PLANE_SPRITE1] =
3702                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3703         }
3704
3705         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1(dev_priv));
3706         wm->sr.plane = _FW_WM(tmp, SR);
3707         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3708         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
3709         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
3710
3711         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2(dev_priv));
3712         wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
3713         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3714         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
3715
3716         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));
3717         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3718
3719         if (IS_CHERRYVIEW(dev_priv)) {
3720                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
3721                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3722                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3723
3724                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
3725                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
3726                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
3727
3728                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
3729                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
3730                 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
3731
3732                 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3733                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3734                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3735                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3736                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
3737                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3738                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3739                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3740                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3741                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3742                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3743         } else {
3744                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
3745                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3746                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3747
3748                 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3749                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3750                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3751                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3752                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3753                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3754                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3755                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3756         }
3757 }
3758
3759 #undef _FW_WM
3760 #undef _FW_WM_VLV
3761
3762 static void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
3763 {
3764         struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;
3765         struct intel_crtc *crtc;
3766
3767         g4x_read_wm_values(dev_priv, wm);
3768
3769         wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
3770
3771         for_each_intel_crtc(&dev_priv->drm, crtc) {
3772                 struct intel_crtc_state *crtc_state =
3773                         to_intel_crtc_state(crtc->base.state);
3774                 struct g4x_wm_state *active = &crtc->wm.active.g4x;
3775                 struct g4x_pipe_wm *raw;
3776                 enum pipe pipe = crtc->pipe;
3777                 enum plane_id plane_id;
3778                 int level, max_level;
3779
3780                 active->cxsr = wm->cxsr;
3781                 active->hpll_en = wm->hpll_en;
3782                 active->fbc_en = wm->fbc_en;
3783
3784                 active->sr = wm->sr;
3785                 active->hpll = wm->hpll;
3786
3787                 for_each_plane_id_on_crtc(crtc, plane_id) {
3788                         active->wm.plane[plane_id] =
3789                                 wm->pipe[pipe].plane[plane_id];
3790                 }
3791
3792                 if (wm->cxsr && wm->hpll_en)
3793                         max_level = G4X_WM_LEVEL_HPLL;
3794                 else if (wm->cxsr)
3795                         max_level = G4X_WM_LEVEL_SR;
3796                 else
3797                         max_level = G4X_WM_LEVEL_NORMAL;
3798
3799                 level = G4X_WM_LEVEL_NORMAL;
3800                 raw = &crtc_state->wm.g4x.raw[level];
3801                 for_each_plane_id_on_crtc(crtc, plane_id)
3802                         raw->plane[plane_id] = active->wm.plane[plane_id];
3803
3804                 level = G4X_WM_LEVEL_SR;
3805                 if (level > max_level)
3806                         goto out;
3807
3808                 raw = &crtc_state->wm.g4x.raw[level];
3809                 raw->plane[PLANE_PRIMARY] = active->sr.plane;
3810                 raw->plane[PLANE_CURSOR] = active->sr.cursor;
3811                 raw->plane[PLANE_SPRITE0] = 0;
3812                 raw->fbc = active->sr.fbc;
3813
3814                 level = G4X_WM_LEVEL_HPLL;
3815                 if (level > max_level)
3816                         goto out;
3817
3818                 raw = &crtc_state->wm.g4x.raw[level];
3819                 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
3820                 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
3821                 raw->plane[PLANE_SPRITE0] = 0;
3822                 raw->fbc = active->hpll.fbc;
3823
3824                 level++;
3825         out:
3826                 for_each_plane_id_on_crtc(crtc, plane_id)
3827                         g4x_raw_plane_wm_set(crtc_state, level,
3828                                              plane_id, USHRT_MAX);
3829                 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
3830
3831                 g4x_invalidate_wms(crtc, active, level);
3832
3833                 crtc_state->wm.g4x.optimal = *active;
3834                 crtc_state->wm.g4x.intermediate = *active;
3835
3836                 drm_dbg_kms(&dev_priv->drm,
3837                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
3838                             pipe_name(pipe),
3839                             wm->pipe[pipe].plane[PLANE_PRIMARY],
3840                             wm->pipe[pipe].plane[PLANE_CURSOR],
3841                             wm->pipe[pipe].plane[PLANE_SPRITE0]);
3842         }
3843
3844         drm_dbg_kms(&dev_priv->drm,
3845                     "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
3846                     wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
3847         drm_dbg_kms(&dev_priv->drm,
3848                     "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
3849                     wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
3850         drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
3851                     str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
3852                     str_yes_no(wm->fbc_en));
3853 }
3854
3855 static void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
3856 {
3857         struct intel_display *display = &dev_priv->display;
3858         struct intel_plane *plane;
3859         struct intel_crtc *crtc;
3860
3861         mutex_lock(&dev_priv->display.wm.wm_mutex);
3862
3863         for_each_intel_plane(&dev_priv->drm, plane) {
3864                 struct intel_crtc *crtc =
3865                         intel_crtc_for_pipe(display, plane->pipe);
3866                 struct intel_crtc_state *crtc_state =
3867                         to_intel_crtc_state(crtc->base.state);
3868                 struct intel_plane_state *plane_state =
3869                         to_intel_plane_state(plane->base.state);
3870                 enum plane_id plane_id = plane->id;
3871                 int level;
3872
3873                 if (plane_state->uapi.visible)
3874                         continue;
3875
3876                 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
3877                         struct g4x_pipe_wm *raw =
3878                                 &crtc_state->wm.g4x.raw[level];
3879
3880                         raw->plane[plane_id] = 0;
3881
3882                         if (plane_id == PLANE_PRIMARY)
3883                                 raw->fbc = 0;
3884                 }
3885         }
3886
3887         for_each_intel_crtc(&dev_priv->drm, crtc) {
3888                 struct intel_crtc_state *crtc_state =
3889                         to_intel_crtc_state(crtc->base.state);
3890                 int ret;
3891
3892                 ret = _g4x_compute_pipe_wm(crtc_state);
3893                 drm_WARN_ON(&dev_priv->drm, ret);
3894
3895                 crtc_state->wm.g4x.intermediate =
3896                         crtc_state->wm.g4x.optimal;
3897                 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
3898         }
3899
3900         g4x_program_watermarks(dev_priv);
3901
3902         mutex_unlock(&dev_priv->display.wm.wm_mutex);
3903 }
3904
3905 static void g4x_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3906 {
3907         g4x_wm_get_hw_state(i915);
3908         g4x_wm_sanitize(i915);
3909 }
3910
3911 static void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
3912 {
3913         struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;
3914         struct intel_crtc *crtc;
3915         u32 val;
3916
3917         vlv_read_wm_values(dev_priv, wm);
3918
3919         wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3920         wm->level = VLV_WM_LEVEL_PM2;
3921
3922         if (IS_CHERRYVIEW(dev_priv)) {
3923                 vlv_punit_get(dev_priv);
3924
3925                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
3926                 if (val & DSP_MAXFIFO_PM5_ENABLE)
3927                         wm->level = VLV_WM_LEVEL_PM5;
3928
3929                 /*
3930                  * If DDR DVFS is disabled in the BIOS, Punit
3931                  * will never ack the request. So if that happens
3932                  * assume we don't have to enable/disable DDR DVFS
3933                  * dynamically. To test that just set the REQ_ACK
3934                  * bit to poke the Punit, but don't change the
3935                  * HIGH/LOW bits so that we don't actually change
3936                  * the current state.
3937                  */
3938                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3939                 val |= FORCE_DDR_FREQ_REQ_ACK;
3940                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
3941
3942                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
3943                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
3944                         drm_dbg_kms(&dev_priv->drm,
3945                                     "Punit not acking DDR DVFS request, "
3946                                     "assuming DDR DVFS is disabled\n");
3947                         dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM5 + 1;
3948                 } else {
3949                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3950                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
3951                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
3952                 }
3953
3954                 vlv_punit_put(dev_priv);
3955         }
3956
3957         for_each_intel_crtc(&dev_priv->drm, crtc) {
3958                 struct intel_crtc_state *crtc_state =
3959                         to_intel_crtc_state(crtc->base.state);
3960                 struct vlv_wm_state *active = &crtc->wm.active.vlv;
3961                 const struct vlv_fifo_state *fifo_state =
3962                         &crtc_state->wm.vlv.fifo_state;
3963                 enum pipe pipe = crtc->pipe;
3964                 enum plane_id plane_id;
3965                 int level;
3966
3967                 vlv_get_fifo_size(crtc_state);
3968
3969                 active->num_levels = wm->level + 1;
3970                 active->cxsr = wm->cxsr;
3971
3972                 for (level = 0; level < active->num_levels; level++) {
3973                         struct g4x_pipe_wm *raw =
3974                                 &crtc_state->wm.vlv.raw[level];
3975
3976                         active->sr[level].plane = wm->sr.plane;
3977                         active->sr[level].cursor = wm->sr.cursor;
3978
3979                         for_each_plane_id_on_crtc(crtc, plane_id) {
3980                                 active->wm[level].plane[plane_id] =
3981                                         wm->pipe[pipe].plane[plane_id];
3982
3983                                 raw->plane[plane_id] =
3984                                         vlv_invert_wm_value(active->wm[level].plane[plane_id],
3985                                                             fifo_state->plane[plane_id]);
3986                         }
3987                 }
3988
3989                 for_each_plane_id_on_crtc(crtc, plane_id)
3990                         vlv_raw_plane_wm_set(crtc_state, level,
3991                                              plane_id, USHRT_MAX);
3992                 vlv_invalidate_wms(crtc, active, level);
3993
3994                 crtc_state->wm.vlv.optimal = *active;
3995                 crtc_state->wm.vlv.intermediate = *active;
3996
3997                 drm_dbg_kms(&dev_priv->drm,
3998                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
3999                             pipe_name(pipe),
4000                             wm->pipe[pipe].plane[PLANE_PRIMARY],
4001                             wm->pipe[pipe].plane[PLANE_CURSOR],
4002                             wm->pipe[pipe].plane[PLANE_SPRITE0],
4003                             wm->pipe[pipe].plane[PLANE_SPRITE1]);
4004         }
4005
4006         drm_dbg_kms(&dev_priv->drm,
4007                     "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
4008                     wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
4009 }
4010
4011 static void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
4012 {
4013         struct intel_display *display = &dev_priv->display;
4014         struct intel_plane *plane;
4015         struct intel_crtc *crtc;
4016
4017         mutex_lock(&dev_priv->display.wm.wm_mutex);
4018
4019         for_each_intel_plane(&dev_priv->drm, plane) {
4020                 struct intel_crtc *crtc =
4021                         intel_crtc_for_pipe(display, plane->pipe);
4022                 struct intel_crtc_state *crtc_state =
4023                         to_intel_crtc_state(crtc->base.state);
4024                 struct intel_plane_state *plane_state =
4025                         to_intel_plane_state(plane->base.state);
4026                 enum plane_id plane_id = plane->id;
4027                 int level;
4028
4029                 if (plane_state->uapi.visible)
4030                         continue;
4031
4032                 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
4033                         struct g4x_pipe_wm *raw =
4034                                 &crtc_state->wm.vlv.raw[level];
4035
4036                         raw->plane[plane_id] = 0;
4037                 }
4038         }
4039
4040         for_each_intel_crtc(&dev_priv->drm, crtc) {
4041                 struct intel_crtc_state *crtc_state =
4042                         to_intel_crtc_state(crtc->base.state);
4043                 int ret;
4044
4045                 ret = _vlv_compute_pipe_wm(crtc_state);
4046                 drm_WARN_ON(&dev_priv->drm, ret);
4047
4048                 crtc_state->wm.vlv.intermediate =
4049                         crtc_state->wm.vlv.optimal;
4050                 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
4051         }
4052
4053         vlv_program_watermarks(dev_priv);
4054
4055         mutex_unlock(&dev_priv->display.wm.wm_mutex);
4056 }
4057
4058 static void vlv_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
4059 {
4060         vlv_wm_get_hw_state(i915);
4061         vlv_wm_sanitize(i915);
4062 }
4063
4064 /*
4065  * FIXME should probably kill this and improve
4066  * the real watermark readout/sanitation instead
4067  */
4068 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
4069 {
4070         intel_uncore_rmw(&dev_priv->uncore, WM3_LP_ILK, WM_LP_ENABLE, 0);
4071         intel_uncore_rmw(&dev_priv->uncore, WM2_LP_ILK, WM_LP_ENABLE, 0);
4072         intel_uncore_rmw(&dev_priv->uncore, WM1_LP_ILK, WM_LP_ENABLE, 0);
4073
4074         /*
4075          * Don't touch WM_LP_SPRITE_ENABLE here.
4076          * Doing so could cause underruns.
4077          */
4078 }
4079
4080 static void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
4081 {
4082         struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
4083         struct intel_crtc *crtc;
4084
4085         ilk_init_lp_watermarks(dev_priv);
4086
4087         for_each_intel_crtc(&dev_priv->drm, crtc)
4088                 ilk_pipe_wm_get_hw_state(crtc);
4089
4090         hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
4091         hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
4092         hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
4093
4094         hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
4095         if (DISPLAY_VER(dev_priv) >= 7) {
4096                 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
4097                 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
4098         }
4099
4100         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
4101                 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) &
4102                                     WM_MISC_DATA_PARTITION_5_6) ?
4103                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4104         else if (IS_IVYBRIDGE(dev_priv))
4105                 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) &
4106                                     DISP_DATA_PARTITION_5_6) ?
4107                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4108
4109         hw->enable_fbc_wm =
4110                 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
4111 }
4112
4113 static const struct intel_wm_funcs ilk_wm_funcs = {
4114         .compute_watermarks = ilk_compute_watermarks,
4115         .initial_watermarks = ilk_initial_watermarks,
4116         .optimize_watermarks = ilk_optimize_watermarks,
4117         .get_hw_state = ilk_wm_get_hw_state,
4118 };
4119
4120 static const struct intel_wm_funcs vlv_wm_funcs = {
4121         .compute_watermarks = vlv_compute_watermarks,
4122         .initial_watermarks = vlv_initial_watermarks,
4123         .optimize_watermarks = vlv_optimize_watermarks,
4124         .atomic_update_watermarks = vlv_atomic_update_fifo,
4125         .get_hw_state = vlv_wm_get_hw_state_and_sanitize,
4126 };
4127
4128 static const struct intel_wm_funcs g4x_wm_funcs = {
4129         .compute_watermarks = g4x_compute_watermarks,
4130         .initial_watermarks = g4x_initial_watermarks,
4131         .optimize_watermarks = g4x_optimize_watermarks,
4132         .get_hw_state = g4x_wm_get_hw_state_and_sanitize,
4133 };
4134
4135 static const struct intel_wm_funcs pnv_wm_funcs = {
4136         .compute_watermarks = i9xx_compute_watermarks,
4137         .update_wm = pnv_update_wm,
4138 };
4139
4140 static const struct intel_wm_funcs i965_wm_funcs = {
4141         .compute_watermarks = i9xx_compute_watermarks,
4142         .update_wm = i965_update_wm,
4143 };
4144
4145 static const struct intel_wm_funcs i9xx_wm_funcs = {
4146         .compute_watermarks = i9xx_compute_watermarks,
4147         .update_wm = i9xx_update_wm,
4148 };
4149
4150 static const struct intel_wm_funcs i845_wm_funcs = {
4151         .compute_watermarks = i9xx_compute_watermarks,
4152         .update_wm = i845_update_wm,
4153 };
4154
4155 static const struct intel_wm_funcs nop_funcs = {
4156 };
4157
4158 void i9xx_wm_init(struct drm_i915_private *dev_priv)
4159 {
4160         /* For FIFO watermark updates */
4161         if (HAS_PCH_SPLIT(dev_priv)) {
4162                 ilk_setup_wm_latency(dev_priv);
4163                 dev_priv->display.funcs.wm = &ilk_wm_funcs;
4164         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
4165                 vlv_setup_wm_latency(dev_priv);
4166                 dev_priv->display.funcs.wm = &vlv_wm_funcs;
4167         } else if (IS_G4X(dev_priv)) {
4168                 g4x_setup_wm_latency(dev_priv);
4169                 dev_priv->display.funcs.wm = &g4x_wm_funcs;
4170         } else if (IS_PINEVIEW(dev_priv)) {
4171                 if (!pnv_get_cxsr_latency(dev_priv)) {
4172                         drm_info(&dev_priv->drm, "Unknown FSB/MEM, disabling CxSR\n");
4173                         /* Disable CxSR and never update its watermark again */
4174                         intel_set_memory_cxsr(dev_priv, false);
4175                         dev_priv->display.funcs.wm = &nop_funcs;
4176                 } else {
4177                         dev_priv->display.funcs.wm = &pnv_wm_funcs;
4178                 }
4179         } else if (DISPLAY_VER(dev_priv) == 4) {
4180                 dev_priv->display.funcs.wm = &i965_wm_funcs;
4181         } else if (DISPLAY_VER(dev_priv) == 3) {
4182                 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4183         } else if (DISPLAY_VER(dev_priv) == 2) {
4184                 if (INTEL_NUM_PIPES(dev_priv) == 1)
4185                         dev_priv->display.funcs.wm = &i845_wm_funcs;
4186                 else
4187                         dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4188         } else {
4189                 drm_err(&dev_priv->drm,
4190                         "unexpected fall-through in %s\n", __func__);
4191                 dev_priv->display.funcs.wm = &nop_funcs;
4192         }
4193 }