drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <[email protected]>
  25  *
  26  */
  27
  28 #include <linux/module.h>
  29 #include <linux/string_helpers.h>
  30 #include <linux/pm_runtime.h>
  31
  32 #include <drm/drm_atomic_helper.h>
  33 #include <drm/drm_blend.h>
  34 #include <drm/drm_fourcc.h>
  35
  36 #include "display/intel_atomic.h"
  37 #include "display/intel_atomic_plane.h"
  38 #include "display/intel_bw.h"
  39 #include "display/intel_de.h"
  40 #include "display/intel_display_trace.h"
  41 #include "display/intel_display_types.h"
  42 #include "display/intel_fb.h"
  43 #include "display/intel_fbc.h"
  44 #include "display/intel_sprite.h"
  45 #include "display/skl_universal_plane.h"
  46
  47 #include "gt/intel_engine_regs.h"
  48 #include "gt/intel_gt_regs.h"
  49 #include "gt/intel_llc.h"
  50
  51 #include "i915_drv.h"
  52 #include "i915_fixed.h"
  53 #include "i915_irq.h"
  54 #include "intel_mchbar_regs.h"
  55 #include "intel_pcode.h"
  56 #include "intel_pm.h"
  57 #include "vlv_sideband.h"
  58 #include "../../../platform/x86/intel_ips.h"
  59
  60 static void skl_sagv_disable(struct drm_i915_private *dev_priv);
  61
  62 struct drm_i915_clock_gating_funcs {
  63         void (*init_clock_gating)(struct drm_i915_private *i915);
  64 };
  65
  66 /* Stores plane specific WM parameters */
  67 struct skl_wm_params {
  68         bool x_tiled, y_tiled;
  69         bool rc_surface;
  70         bool is_planar;
  71         u32 width;
  72         u8 cpp;
  73         u32 plane_pixel_rate;
  74         u32 y_min_scanlines;
  75         u32 plane_bytes_per_line;
  76         uint_fixed_16_16_t plane_blocks_per_line;
  77         uint_fixed_16_16_t y_tile_minimum;
  78         u32 linetime_us;
  79         u32 dbuf_block_size;
  80 };
  81
  82 /* used in computing the new watermarks state */
  83 struct intel_wm_config {
  84         unsigned int num_pipes_active;
  85         bool sprites_enabled;
  86         bool sprites_scaled;
  87 };
  88
  89 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
  90 {
  91         if (HAS_LLC(dev_priv)) {
  92                 /*
  93                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  94                  * Display WA #0390: skl,kbl
  95                  *
  96                  * Must match Sampler, Pixel Back End, and Media. See
  97                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  98                  */
  99                 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
 100                            intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) |
 101                            SKL_DE_COMPRESSED_HASH_MODE);
 102         }
 103
 104         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
 105         intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
 106                    intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
 107
 108         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
 109         intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
 110                    intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
 111
 112         /*
 113          * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
 114          * Display WA #0859: skl,bxt,kbl,glk,cfl
 115          */
 116         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
 117                    DISP_FBC_MEMORY_WAKE);
 118 }
 119
 120 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
 121 {
 122         gen9_init_clock_gating(dev_priv);
 123
 124         /* WaDisableSDEUnitClockGating:bxt */
 125         intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
 126                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 127
 128         /*
 129          * FIXME:
 130          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
 131          */
 132         intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
 133                    GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
 134
 135         /*
 136          * Wa: Backlight PWM may stop in the asserted state, causing backlight
 137          * to stay fully on.
 138          */
 139         intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
 140                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 141
 142         /*
 143          * Lower the display internal timeout.
 144          * This is needed to avoid any hard hangs when DSI port PLL
 145          * is off and a MMIO access is attempted by any privilege
 146          * application, using batch buffers or any other means.
 147          */
 148         intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
 149
 150         /*
 151          * WaFbcTurnOffFbcWatermark:bxt
 152          * Display WA #0562: bxt
 153          */
 154         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
 155                    DISP_FBC_WM_DIS);
 156
 157         /*
 158          * WaFbcHighMemBwCorruptionAvoidance:bxt
 159          * Display WA #0883: bxt
 160          */
 161         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 162                            intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
 163                            DPFC_DISABLE_DUMMY0);
 164 }
 165
 166 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
 167 {
 168         gen9_init_clock_gating(dev_priv);
 169
 170         /*
 171          * WaDisablePWMClockGating:glk
 172          * Backlight PWM may stop in the asserted state, causing backlight
 173          * to stay fully on.
 174          */
 175         intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
 176                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 177 }
 178
 179 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
 180 {
 181         u32 tmp;
 182
 183         tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
 184
 185         switch (tmp & CLKCFG_FSB_MASK) {
 186         case CLKCFG_FSB_533:
 187                 dev_priv->fsb_freq = 533; /* 133*4 */
 188                 break;
 189         case CLKCFG_FSB_800:
 190                 dev_priv->fsb_freq = 800; /* 200*4 */
 191                 break;
 192         case CLKCFG_FSB_667:
 193                 dev_priv->fsb_freq =  667; /* 167*4 */
 194                 break;
 195         case CLKCFG_FSB_400:
 196                 dev_priv->fsb_freq = 400; /* 100*4 */
 197                 break;
 198         }
 199
 200         switch (tmp & CLKCFG_MEM_MASK) {
 201         case CLKCFG_MEM_533:
 202                 dev_priv->mem_freq = 533;
 203                 break;
 204         case CLKCFG_MEM_667:
 205                 dev_priv->mem_freq = 667;
 206                 break;
 207         case CLKCFG_MEM_800:
 208                 dev_priv->mem_freq = 800;
 209                 break;
 210         }
 211
 212         /* detect pineview DDR3 setting */
 213         tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
 214         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 215 }
 216
 217 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
 218 {
 219         u16 ddrpll, csipll;
 220
 221         ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
 222         csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
 223
 224         switch (ddrpll & 0xff) {
 225         case 0xc:
 226                 dev_priv->mem_freq = 800;
 227                 break;
 228         case 0x10:
 229                 dev_priv->mem_freq = 1066;
 230                 break;
 231         case 0x14:
 232                 dev_priv->mem_freq = 1333;
 233                 break;
 234         case 0x18:
 235                 dev_priv->mem_freq = 1600;
 236                 break;
 237         default:
 238                 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
 239                         ddrpll & 0xff);
 240                 dev_priv->mem_freq = 0;
 241                 break;
 242         }
 243
 244         switch (csipll & 0x3ff) {
 245         case 0x00c:
 246                 dev_priv->fsb_freq = 3200;
 247                 break;
 248         case 0x00e:
 249                 dev_priv->fsb_freq = 3733;
 250                 break;
 251         case 0x010:
 252                 dev_priv->fsb_freq = 4266;
 253                 break;
 254         case 0x012:
 255                 dev_priv->fsb_freq = 4800;
 256                 break;
 257         case 0x014:
 258                 dev_priv->fsb_freq = 5333;
 259                 break;
 260         case 0x016:
 261                 dev_priv->fsb_freq = 5866;
 262                 break;
 263         case 0x018:
 264                 dev_priv->fsb_freq = 6400;
 265                 break;
 266         default:
 267                 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
 268                         csipll & 0x3ff);
 269                 dev_priv->fsb_freq = 0;
 270                 break;
 271         }
 272 }
 273
 274 static const struct cxsr_latency cxsr_latency_table[] = {
 275         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 276         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 277         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 278         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 279         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 280
 281         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 282         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 283         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 284         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 285         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 286
 287         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 288         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 289         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 290         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 291         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 292
 293         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 294         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 295         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 296         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 297         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 298
 299         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 300         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 301         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 302         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 303         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 304
 305         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 306         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 307         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 308         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 309         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 310 };
 311
 312 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
 313                                                          bool is_ddr3,
 314                                                          int fsb,
 315                                                          int mem)
 316 {
 317         const struct cxsr_latency *latency;
 318         int i;
 319
 320         if (fsb == 0 || mem == 0)
 321                 return NULL;
 322
 323         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 324                 latency = &cxsr_latency_table[i];
 325                 if (is_desktop == latency->is_desktop &&
 326                     is_ddr3 == latency->is_ddr3 &&
 327                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 328                         return latency;
 329         }
 330
 331         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 332
 333         return NULL;
 334 }
 335
 336 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 337 {
 338         u32 val;
 339
 340         vlv_punit_get(dev_priv);
 341
 342         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 343         if (enable)
 344                 val &= ~FORCE_DDR_HIGH_FREQ;
 345         else
 346                 val |= FORCE_DDR_HIGH_FREQ;
 347         val &= ~FORCE_DDR_LOW_FREQ;
 348         val |= FORCE_DDR_FREQ_REQ_ACK;
 349         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 350
 351         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 352                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 353                 drm_err(&dev_priv->drm,
 354                         "timed out waiting for Punit DDR DVFS request\n");
 355
 356         vlv_punit_put(dev_priv);
 357 }
 358
 359 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 360 {
 361         u32 val;
 362
 363         vlv_punit_get(dev_priv);
 364
 365         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 366         if (enable)
 367                 val |= DSP_MAXFIFO_PM5_ENABLE;
 368         else
 369                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 370         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 371
 372         vlv_punit_put(dev_priv);
 373 }
 374
 375 #define FW_WM(value, plane) \
 376         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 377
 378 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 379 {
 380         bool was_enabled;
 381         u32 val;
 382
 383         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 384                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 385                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 386                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
 387         } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
 388                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
 389                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 390                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
 391         } else if (IS_PINEVIEW(dev_priv)) {
 392                 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
 393                 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 394                 if (enable)
 395                         val |= PINEVIEW_SELF_REFRESH_EN;
 396                 else
 397                         val &= ~PINEVIEW_SELF_REFRESH_EN;
 398                 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
 399                 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
 400         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 401                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
 402                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 403                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 404                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
 405                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
 406         } else if (IS_I915GM(dev_priv)) {
 407                 /*
 408                  * FIXME can't find a bit like this for 915G, and
 409                  * and yet it does have the related watermark in
 410                  * FW_BLC_SELF. What's going on?
 411                  */
 412                 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
 413                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 414                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 415                 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
 416                 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
 417         } else {
 418                 return false;
 419         }
 420
 421         trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
 422
 423         drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
 424                     str_enabled_disabled(enable),
 425                     str_enabled_disabled(was_enabled));
 426
 427         return was_enabled;
 428 }
 429
 430 /**
 431  * intel_set_memory_cxsr - Configure CxSR state
 432  * @dev_priv: i915 device
 433  * @enable: Allow vs. disallow CxSR
 434  *
 435  * Allow or disallow the system to enter a special CxSR
 436  * (C-state self refresh) state. What typically happens in CxSR mode
 437  * is that several display FIFOs may get combined into a single larger
 438  * FIFO for a particular plane (so called max FIFO mode) to allow the
 439  * system to defer memory fetches longer, and the memory will enter
 440  * self refresh.
 441  *
 442  * Note that enabling CxSR does not guarantee that the system enter
 443  * this special mode, nor does it guarantee that the system stays
 444  * in that mode once entered. So this just allows/disallows the system
 445  * to autonomously utilize the CxSR mode. Other factors such as core
 446  * C-states will affect when/if the system actually enters/exits the
 447  * CxSR mode.
 448  *
 449  * Note that on VLV/CHV this actually only controls the max FIFO mode,
 450  * and the system is free to enter/exit memory self refresh at any time
 451  * even when the use of CxSR has been disallowed.
 452  *
 453  * While the system is actually in the CxSR/max FIFO mode, some plane
 454  * control registers will not get latched on vblank. Thus in order to
 455  * guarantee the system will respond to changes in the plane registers
 456  * we must always disallow CxSR prior to making changes to those registers.
 457  * Unfortunately the system will re-evaluate the CxSR conditions at
 458  * frame start which happens after vblank start (which is when the plane
 459  * registers would get latched), so we can't proceed with the plane update
 460  * during the same frame where we disallowed CxSR.
 461  *
 462  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 463  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 464  * the hardware w.r.t. HPLL SR when writing to plane registers.
 465  * Disallowing just CxSR is sufficient.
 466  */
 467 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 468 {
 469         bool ret;
 470
 471         mutex_lock(&dev_priv->display.wm.wm_mutex);
 472         ret = _intel_set_memory_cxsr(dev_priv, enable);
 473         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 474                 dev_priv->display.wm.vlv.cxsr = enable;
 475         else if (IS_G4X(dev_priv))
 476                 dev_priv->display.wm.g4x.cxsr = enable;
 477         mutex_unlock(&dev_priv->display.wm.wm_mutex);
 478
 479         return ret;
 480 }
 481
 482 /*
 483  * Latency for FIFO fetches is dependent on several factors:
 484  *   - memory configuration (speed, channels)
 485  *   - chipset
 486  *   - current MCH state
 487  * It can be fairly high in some situations, so here we assume a fairly
 488  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 489  * set this value too high, the FIFO will fetch frequently to stay full)
 490  * and power consumption (set it too low to save power and we might see
 491  * FIFO underruns and display "flicker").
 492  *
 493  * A value of 5us seems to be a good balance; safe for very low end
 494  * platforms but not overly aggressive on lower latency configs.
 495  */
 496 static const int pessimal_latency_ns = 5000;
 497
 498 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 499         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 500
 501 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 502 {
 503         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 504         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 505         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 506         enum pipe pipe = crtc->pipe;
 507         int sprite0_start, sprite1_start;
 508         u32 dsparb, dsparb2, dsparb3;
 509
 510         switch (pipe) {
 511         case PIPE_A:
 512                 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 513                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 514                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 515                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 516                 break;
 517         case PIPE_B:
 518                 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 519                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 520                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 521                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 522                 break;
 523         case PIPE_C:
 524                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 525                 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
 526                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 527                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 528                 break;
 529         default:
 530                 MISSING_CASE(pipe);
 531                 return;
 532         }
 533
 534         fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 535         fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 536         fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 537         fifo_state->plane[PLANE_CURSOR] = 63;
 538 }
 539
 540 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
 541                               enum i9xx_plane_id i9xx_plane)
 542 {
 543         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 544         int size;
 545
 546         size = dsparb & 0x7f;
 547         if (i9xx_plane == PLANE_B)
 548                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 549
 550         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 551                     dsparb, plane_name(i9xx_plane), size);
 552
 553         return size;
 554 }
 555
 556 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
 557                               enum i9xx_plane_id i9xx_plane)
 558 {
 559         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 560         int size;
 561
 562         size = dsparb & 0x1ff;
 563         if (i9xx_plane == PLANE_B)
 564                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 565         size >>= 1; /* Convert to cachelines */
 566
 567         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 568                     dsparb, plane_name(i9xx_plane), size);
 569
 570         return size;
 571 }
 572
 573 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
 574                               enum i9xx_plane_id i9xx_plane)
 575 {
 576         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 577         int size;
 578
 579         size = dsparb & 0x7f;
 580         size >>= 2; /* Convert to cachelines */
 581
 582         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 583                     dsparb, plane_name(i9xx_plane), size);
 584
 585         return size;
 586 }
 587
 588 /* Pineview has different values for various configs */
 589 static const struct intel_watermark_params pnv_display_wm = {
 590         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 591         .max_wm = PINEVIEW_MAX_WM,
 592         .default_wm = PINEVIEW_DFT_WM,
 593         .guard_size = PINEVIEW_GUARD_WM,
 594         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 595 };
 596
 597 static const struct intel_watermark_params pnv_display_hplloff_wm = {
 598         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 599         .max_wm = PINEVIEW_MAX_WM,
 600         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 601         .guard_size = PINEVIEW_GUARD_WM,
 602         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 603 };
 604
 605 static const struct intel_watermark_params pnv_cursor_wm = {
 606         .fifo_size = PINEVIEW_CURSOR_FIFO,
 607         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 608         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 609         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 610         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 611 };
 612
 613 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
 614         .fifo_size = PINEVIEW_CURSOR_FIFO,
 615         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 616         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 617         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 618         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 619 };
 620
 621 static const struct intel_watermark_params i965_cursor_wm_info = {
 622         .fifo_size = I965_CURSOR_FIFO,
 623         .max_wm = I965_CURSOR_MAX_WM,
 624         .default_wm = I965_CURSOR_DFT_WM,
 625         .guard_size = 2,
 626         .cacheline_size = I915_FIFO_LINE_SIZE,
 627 };
 628
 629 static const struct intel_watermark_params i945_wm_info = {
 630         .fifo_size = I945_FIFO_SIZE,
 631         .max_wm = I915_MAX_WM,
 632         .default_wm = 1,
 633         .guard_size = 2,
 634         .cacheline_size = I915_FIFO_LINE_SIZE,
 635 };
 636
 637 static const struct intel_watermark_params i915_wm_info = {
 638         .fifo_size = I915_FIFO_SIZE,
 639         .max_wm = I915_MAX_WM,
 640         .default_wm = 1,
 641         .guard_size = 2,
 642         .cacheline_size = I915_FIFO_LINE_SIZE,
 643 };
 644
 645 static const struct intel_watermark_params i830_a_wm_info = {
 646         .fifo_size = I855GM_FIFO_SIZE,
 647         .max_wm = I915_MAX_WM,
 648         .default_wm = 1,
 649         .guard_size = 2,
 650         .cacheline_size = I830_FIFO_LINE_SIZE,
 651 };
 652
 653 static const struct intel_watermark_params i830_bc_wm_info = {
 654         .fifo_size = I855GM_FIFO_SIZE,
 655         .max_wm = I915_MAX_WM/2,
 656         .default_wm = 1,
 657         .guard_size = 2,
 658         .cacheline_size = I830_FIFO_LINE_SIZE,
 659 };
 660
 661 static const struct intel_watermark_params i845_wm_info = {
 662         .fifo_size = I830_FIFO_SIZE,
 663         .max_wm = I915_MAX_WM,
 664         .default_wm = 1,
 665         .guard_size = 2,
 666         .cacheline_size = I830_FIFO_LINE_SIZE,
 667 };
 668
 669 /**
 670  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 671  * @pixel_rate: Pipe pixel rate in kHz
 672  * @cpp: Plane bytes per pixel
 673  * @latency: Memory wakeup latency in 0.1us units
 674  *
 675  * Compute the watermark using the method 1 or "small buffer"
 676  * formula. The caller may additonally add extra cachelines
 677  * to account for TLB misses and clock crossings.
 678  *
 679  * This method is concerned with the short term drain rate
 680  * of the FIFO, ie. it does not account for blanking periods
 681  * which would effectively reduce the average drain rate across
 682  * a longer period. The name "small" refers to the fact the
 683  * FIFO is relatively small compared to the amount of data
 684  * fetched.
 685  *
 686  * The FIFO level vs. time graph might look something like:
 687  *
 688  *   |\   |\
 689  *   | \  | \
 690  * __---__---__ (- plane active, _ blanking)
 691  * -> time
 692  *
 693  * or perhaps like this:
 694  *
 695  *   |\|\  |\|\
 696  * __----__----__ (- plane active, _ blanking)
 697  * -> time
 698  *
 699  * Returns:
 700  * The watermark in bytes
 701  */
 702 static unsigned int intel_wm_method1(unsigned int pixel_rate,
 703                                      unsigned int cpp,
 704                                      unsigned int latency)
 705 {
 706         u64 ret;
 707
 708         ret = mul_u32_u32(pixel_rate, cpp * latency);
 709         ret = DIV_ROUND_UP_ULL(ret, 10000);
 710
 711         return ret;
 712 }
 713
 714 /**
 715  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 716  * @pixel_rate: Pipe pixel rate in kHz
 717  * @htotal: Pipe horizontal total
 718  * @width: Plane width in pixels
 719  * @cpp: Plane bytes per pixel
 720  * @latency: Memory wakeup latency in 0.1us units
 721  *
 722  * Compute the watermark using the method 2 or "large buffer"
 723  * formula. The caller may additonally add extra cachelines
 724  * to account for TLB misses and clock crossings.
 725  *
 726  * This method is concerned with the long term drain rate
 727  * of the FIFO, ie. it does account for blanking periods
 728  * which effectively reduce the average drain rate across
 729  * a longer period. The name "large" refers to the fact the
 730  * FIFO is relatively large compared to the amount of data
 731  * fetched.
 732  *
 733  * The FIFO level vs. time graph might look something like:
 734  *
 735  *    |\___       |\___
 736  *    |    \___   |    \___
 737  *    |        \  |        \
 738  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 739  * -> time
 740  *
 741  * Returns:
 742  * The watermark in bytes
 743  */
 744 static unsigned int intel_wm_method2(unsigned int pixel_rate,
 745                                      unsigned int htotal,
 746                                      unsigned int width,
 747                                      unsigned int cpp,
 748                                      unsigned int latency)
 749 {
 750         unsigned int ret;
 751
 752         /*
 753          * FIXME remove once all users are computing
 754          * watermarks in the correct place.
 755          */
 756         if (WARN_ON_ONCE(htotal == 0))
 757                 htotal = 1;
 758
 759         ret = (latency * pixel_rate) / (htotal * 10000);
 760         ret = (ret + 1) * width * cpp;
 761
 762         return ret;
 763 }
 764
 765 /**
 766  * intel_calculate_wm - calculate watermark level
 767  * @pixel_rate: pixel clock
 768  * @wm: chip FIFO params
 769  * @fifo_size: size of the FIFO buffer
 770  * @cpp: bytes per pixel
 771  * @latency_ns: memory latency for the platform
 772  *
 773  * Calculate the watermark level (the level at which the display plane will
 774  * start fetching from memory again).  Each chip has a different display
 775  * FIFO size and allocation, so the caller needs to figure that out and pass
 776  * in the correct intel_watermark_params structure.
 777  *
 778  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 779  * on the pixel size.  When it reaches the watermark level, it'll start
 780  * fetching FIFO line sized based chunks from memory until the FIFO fills
 781  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 782  * will occur, and a display engine hang could result.
 783  */
 784 static unsigned int intel_calculate_wm(int pixel_rate,
 785                                        const struct intel_watermark_params *wm,
 786                                        int fifo_size, int cpp,
 787                                        unsigned int latency_ns)
 788 {
 789         int entries, wm_size;
 790
 791         /*
 792          * Note: we need to make sure we don't overflow for various clock &
 793          * latency values.
 794          * clocks go from a few thousand to several hundred thousand.
 795          * latency is usually a few thousand
 796          */
 797         entries = intel_wm_method1(pixel_rate, cpp,
 798                                    latency_ns / 100);
 799         entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 800                 wm->guard_size;
 801         DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
 802
 803         wm_size = fifo_size - entries;
 804         DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
 805
 806         /* Don't promote wm_size to unsigned... */
 807         if (wm_size > wm->max_wm)
 808                 wm_size = wm->max_wm;
 809         if (wm_size <= 0)
 810                 wm_size = wm->default_wm;
 811
 812         /*
 813          * Bspec seems to indicate that the value shouldn't be lower than
 814          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 815          * Lets go for 8 which is the burst size since certain platforms
 816          * already use a hardcoded 8 (which is what the spec says should be
 817          * done).
 818          */
 819         if (wm_size <= 8)
 820                 wm_size = 8;
 821
 822         return wm_size;
 823 }
 824
 825 static bool is_disabling(int old, int new, int threshold)
 826 {
 827         return old >= threshold && new < threshold;
 828 }
 829
 830 static bool is_enabling(int old, int new, int threshold)
 831 {
 832         return old < threshold && new >= threshold;
 833 }
 834
 835 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
 836 {
 837         return dev_priv->display.wm.max_level + 1;
 838 }
 839
 840 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
 841                                    const struct intel_plane_state *plane_state)
 842 {
 843         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 844
 845         /* FIXME check the 'enable' instead */
 846         if (!crtc_state->hw.active)
 847                 return false;
 848
 849         /*
 850          * Treat cursor with fb as always visible since cursor updates
 851          * can happen faster than the vrefresh rate, and the current
 852          * watermark code doesn't handle that correctly. Cursor updates
 853          * which set/clear the fb or change the cursor size are going
 854          * to get throttled by intel_legacy_cursor_update() to work
 855          * around this problem with the watermark code.
 856          */
 857         if (plane->id == PLANE_CURSOR)
 858                 return plane_state->hw.fb != NULL;
 859         else
 860                 return plane_state->uapi.visible;
 861 }
 862
 863 static bool intel_crtc_active(struct intel_crtc *crtc)
 864 {
 865         /* Be paranoid as we can arrive here with only partial
 866          * state retrieved from the hardware during setup.
 867          *
 868          * We can ditch the adjusted_mode.crtc_clock check as soon
 869          * as Haswell has gained clock readout/fastboot support.
 870          *
 871          * We can ditch the crtc->primary->state->fb check as soon as we can
 872          * properly reconstruct framebuffers.
 873          *
 874          * FIXME: The intel_crtc->active here should be switched to
 875          * crtc->state->active once we have proper CRTC states wired up
 876          * for atomic.
 877          */
 878         return crtc && crtc->active && crtc->base.primary->state->fb &&
 879                 crtc->config->hw.adjusted_mode.crtc_clock;
 880 }
 881
 882 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 883 {
 884         struct intel_crtc *crtc, *enabled = NULL;
 885
 886         for_each_intel_crtc(&dev_priv->drm, crtc) {
 887                 if (intel_crtc_active(crtc)) {
 888                         if (enabled)
 889                                 return NULL;
 890                         enabled = crtc;
 891                 }
 892         }
 893
 894         return enabled;
 895 }
 896
 897 static void pnv_update_wm(struct drm_i915_private *dev_priv)
 898 {
 899         struct intel_crtc *crtc;
 900         const struct cxsr_latency *latency;
 901         u32 reg;
 902         unsigned int wm;
 903
 904         latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
 905                                          dev_priv->is_ddr3,
 906                                          dev_priv->fsb_freq,
 907                                          dev_priv->mem_freq);
 908         if (!latency) {
 909                 drm_dbg_kms(&dev_priv->drm,
 910                             "Unknown FSB/MEM found, disable CxSR\n");
 911                 intel_set_memory_cxsr(dev_priv, false);
 912                 return;
 913         }
 914
 915         crtc = single_enabled_crtc(dev_priv);
 916         if (crtc) {
 917                 const struct drm_framebuffer *fb =
 918                         crtc->base.primary->state->fb;
 919                 int pixel_rate = crtc->config->pixel_rate;
 920                 int cpp = fb->format->cpp[0];
 921
 922                 /* Display SR */
 923                 wm = intel_calculate_wm(pixel_rate, &pnv_display_wm,
 924                                         pnv_display_wm.fifo_size,
 925                                         cpp, latency->display_sr);
 926                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
 927                 reg &= ~DSPFW_SR_MASK;
 928                 reg |= FW_WM(wm, SR);
 929                 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
 930                 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
 931
 932                 /* cursor SR */
 933                 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_wm,
 934                                         pnv_display_wm.fifo_size,
 935                                         4, latency->cursor_sr);
 936                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
 937                 reg &= ~DSPFW_CURSOR_SR_MASK;
 938                 reg |= FW_WM(wm, CURSOR_SR);
 939                 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
 940
 941                 /* Display HPLL off SR */
 942                 wm = intel_calculate_wm(pixel_rate, &pnv_display_hplloff_wm,
 943                                         pnv_display_hplloff_wm.fifo_size,
 944                                         cpp, latency->display_hpll_disable);
 945                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
 946                 reg &= ~DSPFW_HPLL_SR_MASK;
 947                 reg |= FW_WM(wm, HPLL_SR);
 948                 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
 949
 950                 /* cursor HPLL off SR */
 951                 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_hplloff_wm,
 952                                         pnv_display_hplloff_wm.fifo_size,
 953                                         4, latency->cursor_hpll_disable);
 954                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
 955                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 956                 reg |= FW_WM(wm, HPLL_CURSOR);
 957                 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
 958                 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
 959
 960                 intel_set_memory_cxsr(dev_priv, true);
 961         } else {
 962                 intel_set_memory_cxsr(dev_priv, false);
 963         }
 964 }
 965
 966 /*
 967  * Documentation says:
 968  * "If the line size is small, the TLB fetches can get in the way of the
 969  *  data fetches, causing some lag in the pixel data return which is not
 970  *  accounted for in the above formulas. The following adjustment only
 971  *  needs to be applied if eight whole lines fit in the buffer at once.
 972  *  The WM is adjusted upwards by the difference between the FIFO size
 973  *  and the size of 8 whole lines. This adjustment is always performed
 974  *  in the actual pixel depth regardless of whether FBC is enabled or not."
 975  */
 976 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 977 {
 978         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 979
 980         return max(0, tlb_miss);
 981 }
 982
 983 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 984                                 const struct g4x_wm_values *wm)
 985 {
 986         enum pipe pipe;
 987
 988         for_each_pipe(dev_priv, pipe)
 989                 trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
 990
 991         intel_uncore_write(&dev_priv->uncore, DSPFW1,
 992                    FW_WM(wm->sr.plane, SR) |
 993                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 994                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 995                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 996         intel_uncore_write(&dev_priv->uncore, DSPFW2,
 997                    (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 998                    FW_WM(wm->sr.fbc, FBC_SR) |
 999                    FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
1000                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
1001                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1002                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1003         intel_uncore_write(&dev_priv->uncore, DSPFW3,
1004                    (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
1005                    FW_WM(wm->sr.cursor, CURSOR_SR) |
1006                    FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
1007                    FW_WM(wm->hpll.plane, HPLL_SR));
1008
1009         intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1010 }
1011
1012 #define FW_WM_VLV(value, plane) \
1013         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
1014
1015 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
1016                                 const struct vlv_wm_values *wm)
1017 {
1018         enum pipe pipe;
1019
1020         for_each_pipe(dev_priv, pipe) {
1021                 trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
1022
1023                 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
1024                            (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1025                            (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1026                            (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1027                            (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1028         }
1029
1030         /*
1031          * Zero the (unused) WM1 watermarks, and also clear all the
1032          * high order bits so that there are no out of bounds values
1033          * present in the registers during the reprogramming.
1034          */
1035         intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
1036         intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
1037         intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
1038         intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
1039         intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
1040
1041         intel_uncore_write(&dev_priv->uncore, DSPFW1,
1042                    FW_WM(wm->sr.plane, SR) |
1043                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1044                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1045                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1046         intel_uncore_write(&dev_priv->uncore, DSPFW2,
1047                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1048                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1049                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1050         intel_uncore_write(&dev_priv->uncore, DSPFW3,
1051                    FW_WM(wm->sr.cursor, CURSOR_SR));
1052
1053         if (IS_CHERRYVIEW(dev_priv)) {
1054                 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1055                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1056                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1057                 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1058                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1059                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1060                 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1061                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1062                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1063                 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1064                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1065                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1066                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1067                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1068                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1069                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1070                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1071                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1072                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1073                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1074         } else {
1075                 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1076                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1077                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1078                 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1079                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1080                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1081                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1082                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1083                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1084                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1085                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1086         }
1087
1088         intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1089 }
1090
1091 #undef FW_WM_VLV
1092
1093 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1094 {
1095         /* all latencies in usec */
1096         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1097         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1098         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1099
1100         dev_priv->display.wm.max_level = G4X_WM_LEVEL_HPLL;
1101 }
1102
1103 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1104 {
1105         /*
1106          * DSPCNTR[13] supposedly controls whether the
1107          * primary plane can use the FIFO space otherwise
1108          * reserved for the sprite plane. It's not 100% clear
1109          * what the actual FIFO size is, but it looks like we
1110          * can happily set both primary and sprite watermarks
1111          * up to 127 cachelines. So that would seem to mean
1112          * that either DSPCNTR[13] doesn't do anything, or that
1113          * the total FIFO is >= 256 cachelines in size. Either
1114          * way, we don't seem to have to worry about this
1115          * repartitioning as the maximum watermark value the
1116          * register can hold for each plane is lower than the
1117          * minimum FIFO size.
1118          */
1119         switch (plane_id) {
1120         case PLANE_CURSOR:
1121                 return 63;
1122         case PLANE_PRIMARY:
1123                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1124         case PLANE_SPRITE0:
1125                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1126         default:
1127                 MISSING_CASE(plane_id);
1128                 return 0;
1129         }
1130 }
1131
1132 static int g4x_fbc_fifo_size(int level)
1133 {
1134         switch (level) {
1135         case G4X_WM_LEVEL_SR:
1136                 return 7;
1137         case G4X_WM_LEVEL_HPLL:
1138                 return 15;
1139         default:
1140                 MISSING_CASE(level);
1141                 return 0;
1142         }
1143 }
1144
1145 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1146                           const struct intel_plane_state *plane_state,
1147                           int level)
1148 {
1149         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1150         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1151         const struct drm_display_mode *pipe_mode =
1152                 &crtc_state->hw.pipe_mode;
1153         unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;
1154         unsigned int pixel_rate, htotal, cpp, width, wm;
1155
1156         if (latency == 0)
1157                 return USHRT_MAX;
1158
1159         if (!intel_wm_plane_visible(crtc_state, plane_state))
1160                 return 0;
1161
1162         cpp = plane_state->hw.fb->format->cpp[0];
1163
1164         /*
1165          * WaUse32BppForSRWM:ctg,elk
1166          *
1167          * The spec fails to list this restriction for the
1168          * HPLL watermark, which seems a little strange.
1169          * Let's use 32bpp for the HPLL watermark as well.
1170          */
1171         if (plane->id == PLANE_PRIMARY &&
1172             level != G4X_WM_LEVEL_NORMAL)
1173                 cpp = max(cpp, 4u);
1174
1175         pixel_rate = crtc_state->pixel_rate;
1176         htotal = pipe_mode->crtc_htotal;
1177         width = drm_rect_width(&plane_state->uapi.src) >> 16;
1178
1179         if (plane->id == PLANE_CURSOR) {
1180                 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1181         } else if (plane->id == PLANE_PRIMARY &&
1182                    level == G4X_WM_LEVEL_NORMAL) {
1183                 wm = intel_wm_method1(pixel_rate, cpp, latency);
1184         } else {
1185                 unsigned int small, large;
1186
1187                 small = intel_wm_method1(pixel_rate, cpp, latency);
1188                 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1189
1190                 wm = min(small, large);
1191         }
1192
1193         wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1194                               width, cpp);
1195
1196         wm = DIV_ROUND_UP(wm, 64) + 2;
1197
1198         return min_t(unsigned int, wm, USHRT_MAX);
1199 }
1200
1201 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1202                                  int level, enum plane_id plane_id, u16 value)
1203 {
1204         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1205         bool dirty = false;
1206
1207         for (; level < intel_wm_num_levels(dev_priv); level++) {
1208                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1209
1210                 dirty |= raw->plane[plane_id] != value;
1211                 raw->plane[plane_id] = value;
1212         }
1213
1214         return dirty;
1215 }
1216
1217 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1218                                int level, u16 value)
1219 {
1220         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1221         bool dirty = false;
1222
1223         /* NORMAL level doesn't have an FBC watermark */
1224         level = max(level, G4X_WM_LEVEL_SR);
1225
1226         for (; level < intel_wm_num_levels(dev_priv); level++) {
1227                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1228
1229                 dirty |= raw->fbc != value;
1230                 raw->fbc = value;
1231         }
1232
1233         return dirty;
1234 }
1235
1236 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1237                               const struct intel_plane_state *plane_state,
1238                               u32 pri_val);
1239
1240 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1241                                      const struct intel_plane_state *plane_state)
1242 {
1243         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1244         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1245         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1246         enum plane_id plane_id = plane->id;
1247         bool dirty = false;
1248         int level;
1249
1250         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1251                 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1252                 if (plane_id == PLANE_PRIMARY)
1253                         dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1254                 goto out;
1255         }
1256
1257         for (level = 0; level < num_levels; level++) {
1258                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1259                 int wm, max_wm;
1260
1261                 wm = g4x_compute_wm(crtc_state, plane_state, level);
1262                 max_wm = g4x_plane_fifo_size(plane_id, level);
1263
1264                 if (wm > max_wm)
1265                         break;
1266
1267                 dirty |= raw->plane[plane_id] != wm;
1268                 raw->plane[plane_id] = wm;
1269
1270                 if (plane_id != PLANE_PRIMARY ||
1271                     level == G4X_WM_LEVEL_NORMAL)
1272                         continue;
1273
1274                 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1275                                         raw->plane[plane_id]);
1276                 max_wm = g4x_fbc_fifo_size(level);
1277
1278                 /*
1279                  * FBC wm is not mandatory as we
1280                  * can always just disable its use.
1281                  */
1282                 if (wm > max_wm)
1283                         wm = USHRT_MAX;
1284
1285                 dirty |= raw->fbc != wm;
1286                 raw->fbc = wm;
1287         }
1288
1289         /* mark watermarks as invalid */
1290         dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1291
1292         if (plane_id == PLANE_PRIMARY)
1293                 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1294
1295  out:
1296         if (dirty) {
1297                 drm_dbg_kms(&dev_priv->drm,
1298                             "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1299                             plane->base.name,
1300                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1301                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1302                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1303
1304                 if (plane_id == PLANE_PRIMARY)
1305                         drm_dbg_kms(&dev_priv->drm,
1306                                     "FBC watermarks: SR=%d, HPLL=%d\n",
1307                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1308                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1309         }
1310
1311         return dirty;
1312 }
1313
1314 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1315                                       enum plane_id plane_id, int level)
1316 {
1317         const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1318
1319         return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1320 }
1321
1322 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1323                                      int level)
1324 {
1325         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1326
1327         if (level > dev_priv->display.wm.max_level)
1328                 return false;
1329
1330         return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1331                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1332                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1333 }
1334
1335 /* mark all levels starting from 'level' as invalid */
1336 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1337                                struct g4x_wm_state *wm_state, int level)
1338 {
1339         if (level <= G4X_WM_LEVEL_NORMAL) {
1340                 enum plane_id plane_id;
1341
1342                 for_each_plane_id_on_crtc(crtc, plane_id)
1343                         wm_state->wm.plane[plane_id] = USHRT_MAX;
1344         }
1345
1346         if (level <= G4X_WM_LEVEL_SR) {
1347                 wm_state->cxsr = false;
1348                 wm_state->sr.cursor = USHRT_MAX;
1349                 wm_state->sr.plane = USHRT_MAX;
1350                 wm_state->sr.fbc = USHRT_MAX;
1351         }
1352
1353         if (level <= G4X_WM_LEVEL_HPLL) {
1354                 wm_state->hpll_en = false;
1355                 wm_state->hpll.cursor = USHRT_MAX;
1356                 wm_state->hpll.plane = USHRT_MAX;
1357                 wm_state->hpll.fbc = USHRT_MAX;
1358         }
1359 }
1360
1361 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1362                                int level)
1363 {
1364         if (level < G4X_WM_LEVEL_SR)
1365                 return false;
1366
1367         if (level >= G4X_WM_LEVEL_SR &&
1368             wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1369                 return false;
1370
1371         if (level >= G4X_WM_LEVEL_HPLL &&
1372             wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1373                 return false;
1374
1375         return true;
1376 }
1377
1378 static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1379                                struct intel_crtc *crtc)
1380 {
1381         struct intel_crtc_state *crtc_state =
1382                 intel_atomic_get_new_crtc_state(state, crtc);
1383         struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1384         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1385         const struct g4x_pipe_wm *raw;
1386         const struct intel_plane_state *old_plane_state;
1387         const struct intel_plane_state *new_plane_state;
1388         struct intel_plane *plane;
1389         enum plane_id plane_id;
1390         int i, level;
1391         unsigned int dirty = 0;
1392
1393         for_each_oldnew_intel_plane_in_state(state, plane,
1394                                              old_plane_state,
1395                                              new_plane_state, i) {
1396                 if (new_plane_state->hw.crtc != &crtc->base &&
1397                     old_plane_state->hw.crtc != &crtc->base)
1398                         continue;
1399
1400                 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1401                         dirty |= BIT(plane->id);
1402         }
1403
1404         if (!dirty)
1405                 return 0;
1406
1407         level = G4X_WM_LEVEL_NORMAL;
1408         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1409                 goto out;
1410
1411         raw = &crtc_state->wm.g4x.raw[level];
1412         for_each_plane_id_on_crtc(crtc, plane_id)
1413                 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1414
1415         level = G4X_WM_LEVEL_SR;
1416         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1417                 goto out;
1418
1419         raw = &crtc_state->wm.g4x.raw[level];
1420         wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1421         wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1422         wm_state->sr.fbc = raw->fbc;
1423
1424         wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1425
1426         level = G4X_WM_LEVEL_HPLL;
1427         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1428                 goto out;
1429
1430         raw = &crtc_state->wm.g4x.raw[level];
1431         wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1432         wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1433         wm_state->hpll.fbc = raw->fbc;
1434
1435         wm_state->hpll_en = wm_state->cxsr;
1436
1437         level++;
1438
1439  out:
1440         if (level == G4X_WM_LEVEL_NORMAL)
1441                 return -EINVAL;
1442
1443         /* invalidate the higher levels */
1444         g4x_invalidate_wms(crtc, wm_state, level);
1445
1446         /*
1447          * Determine if the FBC watermark(s) can be used. IF
1448          * this isn't the case we prefer to disable the FBC
1449          * watermark(s) rather than disable the SR/HPLL
1450          * level(s) entirely. 'level-1' is the highest valid
1451          * level here.
1452          */
1453         wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1454
1455         return 0;
1456 }
1457
1458 static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1459                                        struct intel_crtc *crtc)
1460 {
1461         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1462         struct intel_crtc_state *new_crtc_state =
1463                 intel_atomic_get_new_crtc_state(state, crtc);
1464         const struct intel_crtc_state *old_crtc_state =
1465                 intel_atomic_get_old_crtc_state(state, crtc);
1466         struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1467         const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1468         const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1469         enum plane_id plane_id;
1470
1471         if (!new_crtc_state->hw.active ||
1472             drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1473                 *intermediate = *optimal;
1474
1475                 intermediate->cxsr = false;
1476                 intermediate->hpll_en = false;
1477                 goto out;
1478         }
1479
1480         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1481                 !new_crtc_state->disable_cxsr;
1482         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1483                 !new_crtc_state->disable_cxsr;
1484         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1485
1486         for_each_plane_id_on_crtc(crtc, plane_id) {
1487                 intermediate->wm.plane[plane_id] =
1488                         max(optimal->wm.plane[plane_id],
1489                             active->wm.plane[plane_id]);
1490
1491                 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1492                             g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1493         }
1494
1495         intermediate->sr.plane = max(optimal->sr.plane,
1496                                      active->sr.plane);
1497         intermediate->sr.cursor = max(optimal->sr.cursor,
1498                                       active->sr.cursor);
1499         intermediate->sr.fbc = max(optimal->sr.fbc,
1500                                    active->sr.fbc);
1501
1502         intermediate->hpll.plane = max(optimal->hpll.plane,
1503                                        active->hpll.plane);
1504         intermediate->hpll.cursor = max(optimal->hpll.cursor,
1505                                         active->hpll.cursor);
1506         intermediate->hpll.fbc = max(optimal->hpll.fbc,
1507                                      active->hpll.fbc);
1508
1509         drm_WARN_ON(&dev_priv->drm,
1510                     (intermediate->sr.plane >
1511                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1512                      intermediate->sr.cursor >
1513                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1514                     intermediate->cxsr);
1515         drm_WARN_ON(&dev_priv->drm,
1516                     (intermediate->sr.plane >
1517                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1518                      intermediate->sr.cursor >
1519                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1520                     intermediate->hpll_en);
1521
1522         drm_WARN_ON(&dev_priv->drm,
1523                     intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1524                     intermediate->fbc_en && intermediate->cxsr);
1525         drm_WARN_ON(&dev_priv->drm,
1526                     intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1527                     intermediate->fbc_en && intermediate->hpll_en);
1528
1529 out:
1530         /*
1531          * If our intermediate WM are identical to the final WM, then we can
1532          * omit the post-vblank programming; only update if it's different.
1533          */
1534         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1535                 new_crtc_state->wm.need_postvbl_update = true;
1536
1537         return 0;
1538 }
1539
1540 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1541                          struct g4x_wm_values *wm)
1542 {
1543         struct intel_crtc *crtc;
1544         int num_active_pipes = 0;
1545
1546         wm->cxsr = true;
1547         wm->hpll_en = true;
1548         wm->fbc_en = true;
1549
1550         for_each_intel_crtc(&dev_priv->drm, crtc) {
1551                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1552
1553                 if (!crtc->active)
1554                         continue;
1555
1556                 if (!wm_state->cxsr)
1557                         wm->cxsr = false;
1558                 if (!wm_state->hpll_en)
1559                         wm->hpll_en = false;
1560                 if (!wm_state->fbc_en)
1561                         wm->fbc_en = false;
1562
1563                 num_active_pipes++;
1564         }
1565
1566         if (num_active_pipes != 1) {
1567                 wm->cxsr = false;
1568                 wm->hpll_en = false;
1569                 wm->fbc_en = false;
1570         }
1571
1572         for_each_intel_crtc(&dev_priv->drm, crtc) {
1573                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1574                 enum pipe pipe = crtc->pipe;
1575
1576                 wm->pipe[pipe] = wm_state->wm;
1577                 if (crtc->active && wm->cxsr)
1578                         wm->sr = wm_state->sr;
1579                 if (crtc->active && wm->hpll_en)
1580                         wm->hpll = wm_state->hpll;
1581         }
1582 }
1583
1584 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1585 {
1586         struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;
1587         struct g4x_wm_values new_wm = {};
1588
1589         g4x_merge_wm(dev_priv, &new_wm);
1590
1591         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1592                 return;
1593
1594         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1595                 _intel_set_memory_cxsr(dev_priv, false);
1596
1597         g4x_write_wm_values(dev_priv, &new_wm);
1598
1599         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1600                 _intel_set_memory_cxsr(dev_priv, true);
1601
1602         *old_wm = new_wm;
1603 }
1604
1605 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1606                                    struct intel_crtc *crtc)
1607 {
1608         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1609         const struct intel_crtc_state *crtc_state =
1610                 intel_atomic_get_new_crtc_state(state, crtc);
1611
1612         mutex_lock(&dev_priv->display.wm.wm_mutex);
1613         crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1614         g4x_program_watermarks(dev_priv);
1615         mutex_unlock(&dev_priv->display.wm.wm_mutex);
1616 }
1617
1618 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1619                                     struct intel_crtc *crtc)
1620 {
1621         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1622         const struct intel_crtc_state *crtc_state =
1623                 intel_atomic_get_new_crtc_state(state, crtc);
1624
1625         if (!crtc_state->wm.need_postvbl_update)
1626                 return;
1627
1628         mutex_lock(&dev_priv->display.wm.wm_mutex);
1629         crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1630         g4x_program_watermarks(dev_priv);
1631         mutex_unlock(&dev_priv->display.wm.wm_mutex);
1632 }
1633
1634 /* latency must be in 0.1us units. */
1635 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1636                                    unsigned int htotal,
1637                                    unsigned int width,
1638                                    unsigned int cpp,
1639                                    unsigned int latency)
1640 {
1641         unsigned int ret;
1642
1643         ret = intel_wm_method2(pixel_rate, htotal,
1644                                width, cpp, latency);
1645         ret = DIV_ROUND_UP(ret, 64);
1646
1647         return ret;
1648 }
1649
1650 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1651 {
1652         /* all latencies in usec */
1653         dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1654
1655         dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM2;
1656
1657         if (IS_CHERRYVIEW(dev_priv)) {
1658                 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1659                 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1660
1661                 dev_priv->display.wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1662         }
1663 }
1664
1665 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1666                                 const struct intel_plane_state *plane_state,
1667                                 int level)
1668 {
1669         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1670         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1671         const struct drm_display_mode *pipe_mode =
1672                 &crtc_state->hw.pipe_mode;
1673         unsigned int pixel_rate, htotal, cpp, width, wm;
1674
1675         if (dev_priv->display.wm.pri_latency[level] == 0)
1676                 return USHRT_MAX;
1677
1678         if (!intel_wm_plane_visible(crtc_state, plane_state))
1679                 return 0;
1680
1681         cpp = plane_state->hw.fb->format->cpp[0];
1682         pixel_rate = crtc_state->pixel_rate;
1683         htotal = pipe_mode->crtc_htotal;
1684         width = drm_rect_width(&plane_state->uapi.src) >> 16;
1685
1686         if (plane->id == PLANE_CURSOR) {
1687                 /*
1688                  * FIXME the formula gives values that are
1689                  * too big for the cursor FIFO, and hence we
1690                  * would never be able to use cursors. For
1691                  * now just hardcode the watermark.
1692                  */
1693                 wm = 63;
1694         } else {
1695                 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1696                                     dev_priv->display.wm.pri_latency[level] * 10);
1697         }
1698
1699         return min_t(unsigned int, wm, USHRT_MAX);
1700 }
1701
1702 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1703 {
1704         return (active_planes & (BIT(PLANE_SPRITE0) |
1705                                  BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1706 }
1707
1708 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1709 {
1710         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1711         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1712         const struct g4x_pipe_wm *raw =
1713                 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1714         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1715         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1716         int num_active_planes = hweight8(active_planes);
1717         const int fifo_size = 511;
1718         int fifo_extra, fifo_left = fifo_size;
1719         int sprite0_fifo_extra = 0;
1720         unsigned int total_rate;
1721         enum plane_id plane_id;
1722
1723         /*
1724          * When enabling sprite0 after sprite1 has already been enabled
1725          * we tend to get an underrun unless sprite0 already has some
1726          * FIFO space allcoated. Hence we always allocate at least one
1727          * cacheline for sprite0 whenever sprite1 is enabled.
1728          *
1729          * All other plane enable sequences appear immune to this problem.
1730          */
1731         if (vlv_need_sprite0_fifo_workaround(active_planes))
1732                 sprite0_fifo_extra = 1;
1733
1734         total_rate = raw->plane[PLANE_PRIMARY] +
1735                 raw->plane[PLANE_SPRITE0] +
1736                 raw->plane[PLANE_SPRITE1] +
1737                 sprite0_fifo_extra;
1738
1739         if (total_rate > fifo_size)
1740                 return -EINVAL;
1741
1742         if (total_rate == 0)
1743                 total_rate = 1;
1744
1745         for_each_plane_id_on_crtc(crtc, plane_id) {
1746                 unsigned int rate;
1747
1748                 if ((active_planes & BIT(plane_id)) == 0) {
1749                         fifo_state->plane[plane_id] = 0;
1750                         continue;
1751                 }
1752
1753                 rate = raw->plane[plane_id];
1754                 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1755                 fifo_left -= fifo_state->plane[plane_id];
1756         }
1757
1758         fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1759         fifo_left -= sprite0_fifo_extra;
1760
1761         fifo_state->plane[PLANE_CURSOR] = 63;
1762
1763         fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1764
1765         /* spread the remainder evenly */
1766         for_each_plane_id_on_crtc(crtc, plane_id) {
1767                 int plane_extra;
1768
1769                 if (fifo_left == 0)
1770                         break;
1771
1772                 if ((active_planes & BIT(plane_id)) == 0)
1773                         continue;
1774
1775                 plane_extra = min(fifo_extra, fifo_left);
1776                 fifo_state->plane[plane_id] += plane_extra;
1777                 fifo_left -= plane_extra;
1778         }
1779
1780         drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1781
1782         /* give it all to the first plane if none are active */
1783         if (active_planes == 0) {
1784                 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1785                 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1786         }
1787
1788         return 0;
1789 }
1790
1791 /* mark all levels starting from 'level' as invalid */
1792 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1793                                struct vlv_wm_state *wm_state, int level)
1794 {
1795         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1796
1797         for (; level < intel_wm_num_levels(dev_priv); level++) {
1798                 enum plane_id plane_id;
1799
1800                 for_each_plane_id_on_crtc(crtc, plane_id)
1801                         wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1802
1803                 wm_state->sr[level].cursor = USHRT_MAX;
1804                 wm_state->sr[level].plane = USHRT_MAX;
1805         }
1806 }
1807
1808 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1809 {
1810         if (wm > fifo_size)
1811                 return USHRT_MAX;
1812         else
1813                 return fifo_size - wm;
1814 }
1815
1816 /*
1817  * Starting from 'level' set all higher
1818  * levels to 'value' in the "raw" watermarks.
1819  */
1820 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1821                                  int level, enum plane_id plane_id, u16 value)
1822 {
1823         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1824         int num_levels = intel_wm_num_levels(dev_priv);
1825         bool dirty = false;
1826
1827         for (; level < num_levels; level++) {
1828                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1829
1830                 dirty |= raw->plane[plane_id] != value;
1831                 raw->plane[plane_id] = value;
1832         }
1833
1834         return dirty;
1835 }
1836
1837 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1838                                      const struct intel_plane_state *plane_state)
1839 {
1840         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1841         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1842         enum plane_id plane_id = plane->id;
1843         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1844         int level;
1845         bool dirty = false;
1846
1847         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1848                 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1849                 goto out;
1850         }
1851
1852         for (level = 0; level < num_levels; level++) {
1853                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1854                 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1855                 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1856
1857                 if (wm > max_wm)
1858                         break;
1859
1860                 dirty |= raw->plane[plane_id] != wm;
1861                 raw->plane[plane_id] = wm;
1862         }
1863
1864         /* mark all higher levels as invalid */
1865         dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1866
1867 out:
1868         if (dirty)
1869                 drm_dbg_kms(&dev_priv->drm,
1870                             "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1871                             plane->base.name,
1872                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1873                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1874                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1875
1876         return dirty;
1877 }
1878
1879 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1880                                       enum plane_id plane_id, int level)
1881 {
1882         const struct g4x_pipe_wm *raw =
1883                 &crtc_state->wm.vlv.raw[level];
1884         const struct vlv_fifo_state *fifo_state =
1885                 &crtc_state->wm.vlv.fifo_state;
1886
1887         return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1888 }
1889
1890 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1891 {
1892         return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1893                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1894                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1895                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1896 }
1897
1898 static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1899                                struct intel_crtc *crtc)
1900 {
1901         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1902         struct intel_crtc_state *crtc_state =
1903                 intel_atomic_get_new_crtc_state(state, crtc);
1904         struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1905         const struct vlv_fifo_state *fifo_state =
1906                 &crtc_state->wm.vlv.fifo_state;
1907         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1908         int num_active_planes = hweight8(active_planes);
1909         bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1910         const struct intel_plane_state *old_plane_state;
1911         const struct intel_plane_state *new_plane_state;
1912         struct intel_plane *plane;
1913         enum plane_id plane_id;
1914         int level, ret, i;
1915         unsigned int dirty = 0;
1916
1917         for_each_oldnew_intel_plane_in_state(state, plane,
1918                                              old_plane_state,
1919                                              new_plane_state, i) {
1920                 if (new_plane_state->hw.crtc != &crtc->base &&
1921                     old_plane_state->hw.crtc != &crtc->base)
1922                         continue;
1923
1924                 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1925                         dirty |= BIT(plane->id);
1926         }
1927
1928         /*
1929          * DSPARB registers may have been reset due to the
1930          * power well being turned off. Make sure we restore
1931          * them to a consistent state even if no primary/sprite
1932          * planes are initially active.
1933          */
1934         if (needs_modeset)
1935                 crtc_state->fifo_changed = true;
1936
1937         if (!dirty)
1938                 return 0;
1939
1940         /* cursor changes don't warrant a FIFO recompute */
1941         if (dirty & ~BIT(PLANE_CURSOR)) {
1942                 const struct intel_crtc_state *old_crtc_state =
1943                         intel_atomic_get_old_crtc_state(state, crtc);
1944                 const struct vlv_fifo_state *old_fifo_state =
1945                         &old_crtc_state->wm.vlv.fifo_state;
1946
1947                 ret = vlv_compute_fifo(crtc_state);
1948                 if (ret)
1949                         return ret;
1950
1951                 if (needs_modeset ||
1952                     memcmp(old_fifo_state, fifo_state,
1953                            sizeof(*fifo_state)) != 0)
1954                         crtc_state->fifo_changed = true;
1955         }
1956
1957         /* initially allow all levels */
1958         wm_state->num_levels = intel_wm_num_levels(dev_priv);
1959         /*
1960          * Note that enabling cxsr with no primary/sprite planes
1961          * enabled can wedge the pipe. Hence we only allow cxsr
1962          * with exactly one enabled primary/sprite plane.
1963          */
1964         wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1965
1966         for (level = 0; level < wm_state->num_levels; level++) {
1967                 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1968                 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1969
1970                 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1971                         break;
1972
1973                 for_each_plane_id_on_crtc(crtc, plane_id) {
1974                         wm_state->wm[level].plane[plane_id] =
1975                                 vlv_invert_wm_value(raw->plane[plane_id],
1976                                                     fifo_state->plane[plane_id]);
1977                 }
1978
1979                 wm_state->sr[level].plane =
1980                         vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1981                                                  raw->plane[PLANE_SPRITE0],
1982                                                  raw->plane[PLANE_SPRITE1]),
1983                                             sr_fifo_size);
1984
1985                 wm_state->sr[level].cursor =
1986                         vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1987                                             63);
1988         }
1989
1990         if (level == 0)
1991                 return -EINVAL;
1992
1993         /* limit to only levels we can actually handle */
1994         wm_state->num_levels = level;
1995
1996         /* invalidate the higher levels */
1997         vlv_invalidate_wms(crtc, wm_state, level);
1998
1999         return 0;
2000 }
2001
2002 #define VLV_FIFO(plane, value) \
2003         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
2004
2005 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
2006                                    struct intel_crtc *crtc)
2007 {
2008         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2009         struct intel_uncore *uncore = &dev_priv->uncore;
2010         const struct intel_crtc_state *crtc_state =
2011                 intel_atomic_get_new_crtc_state(state, crtc);
2012         const struct vlv_fifo_state *fifo_state =
2013                 &crtc_state->wm.vlv.fifo_state;
2014         int sprite0_start, sprite1_start, fifo_size;
2015         u32 dsparb, dsparb2, dsparb3;
2016
2017         if (!crtc_state->fifo_changed)
2018                 return;
2019
2020         sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2021         sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2022         fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2023
2024         drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2025         drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2026
2027         trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2028
2029         /*
2030          * uncore.lock serves a double purpose here. It allows us to
2031          * use the less expensive I915_{READ,WRITE}_FW() functions, and
2032          * it protects the DSPARB registers from getting clobbered by
2033          * parallel updates from multiple pipes.
2034          *
2035          * intel_pipe_update_start() has already disabled interrupts
2036          * for us, so a plain spin_lock() is sufficient here.
2037          */
2038         spin_lock(&uncore->lock);
2039
2040         switch (crtc->pipe) {
2041         case PIPE_A:
2042                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2043                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2044
2045                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2046                             VLV_FIFO(SPRITEB, 0xff));
2047                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2048                            VLV_FIFO(SPRITEB, sprite1_start));
2049
2050                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2051                              VLV_FIFO(SPRITEB_HI, 0x1));
2052                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2053                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2054
2055                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2056                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2057                 break;
2058         case PIPE_B:
2059                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2060                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2061
2062                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2063                             VLV_FIFO(SPRITED, 0xff));
2064                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2065                            VLV_FIFO(SPRITED, sprite1_start));
2066
2067                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2068                              VLV_FIFO(SPRITED_HI, 0xff));
2069                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2070                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2071
2072                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2073                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2074                 break;
2075         case PIPE_C:
2076                 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2077                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2078
2079                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2080                              VLV_FIFO(SPRITEF, 0xff));
2081                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2082                             VLV_FIFO(SPRITEF, sprite1_start));
2083
2084                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2085                              VLV_FIFO(SPRITEF_HI, 0xff));
2086                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2087                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2088
2089                 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2090                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2091                 break;
2092         default:
2093                 break;
2094         }
2095
2096         intel_uncore_posting_read_fw(uncore, DSPARB);
2097
2098         spin_unlock(&uncore->lock);
2099 }
2100
2101 #undef VLV_FIFO
2102
2103 static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
2104                                        struct intel_crtc *crtc)
2105 {
2106         struct intel_crtc_state *new_crtc_state =
2107                 intel_atomic_get_new_crtc_state(state, crtc);
2108         const struct intel_crtc_state *old_crtc_state =
2109                 intel_atomic_get_old_crtc_state(state, crtc);
2110         struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2111         const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2112         const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2113         int level;
2114
2115         if (!new_crtc_state->hw.active ||
2116             drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2117                 *intermediate = *optimal;
2118
2119                 intermediate->cxsr = false;
2120                 goto out;
2121         }
2122
2123         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2124         intermediate->cxsr = optimal->cxsr && active->cxsr &&
2125                 !new_crtc_state->disable_cxsr;
2126
2127         for (level = 0; level < intermediate->num_levels; level++) {
2128                 enum plane_id plane_id;
2129
2130                 for_each_plane_id_on_crtc(crtc, plane_id) {
2131                         intermediate->wm[level].plane[plane_id] =
2132                                 min(optimal->wm[level].plane[plane_id],
2133                                     active->wm[level].plane[plane_id]);
2134                 }
2135
2136                 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2137                                                     active->sr[level].plane);
2138                 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2139                                                      active->sr[level].cursor);
2140         }
2141
2142         vlv_invalidate_wms(crtc, intermediate, level);
2143
2144 out:
2145         /*
2146          * If our intermediate WM are identical to the final WM, then we can
2147          * omit the post-vblank programming; only update if it's different.
2148          */
2149         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2150                 new_crtc_state->wm.need_postvbl_update = true;
2151
2152         return 0;
2153 }
2154
2155 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2156                          struct vlv_wm_values *wm)
2157 {
2158         struct intel_crtc *crtc;
2159         int num_active_pipes = 0;
2160
2161         wm->level = dev_priv->display.wm.max_level;
2162         wm->cxsr = true;
2163
2164         for_each_intel_crtc(&dev_priv->drm, crtc) {
2165                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2166
2167                 if (!crtc->active)
2168                         continue;
2169
2170                 if (!wm_state->cxsr)
2171                         wm->cxsr = false;
2172
2173                 num_active_pipes++;
2174                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2175         }
2176
2177         if (num_active_pipes != 1)
2178                 wm->cxsr = false;
2179
2180         if (num_active_pipes > 1)
2181                 wm->level = VLV_WM_LEVEL_PM2;
2182
2183         for_each_intel_crtc(&dev_priv->drm, crtc) {
2184                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2185                 enum pipe pipe = crtc->pipe;
2186
2187                 wm->pipe[pipe] = wm_state->wm[wm->level];
2188                 if (crtc->active && wm->cxsr)
2189                         wm->sr = wm_state->sr[wm->level];
2190
2191                 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2192                 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2193                 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2194                 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2195         }
2196 }
2197
2198 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2199 {
2200         struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;
2201         struct vlv_wm_values new_wm = {};
2202
2203         vlv_merge_wm(dev_priv, &new_wm);
2204
2205         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2206                 return;
2207
2208         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2209                 chv_set_memory_dvfs(dev_priv, false);
2210
2211         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2212                 chv_set_memory_pm5(dev_priv, false);
2213
2214         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2215                 _intel_set_memory_cxsr(dev_priv, false);
2216
2217         vlv_write_wm_values(dev_priv, &new_wm);
2218
2219         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2220                 _intel_set_memory_cxsr(dev_priv, true);
2221
2222         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2223                 chv_set_memory_pm5(dev_priv, true);
2224
2225         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2226                 chv_set_memory_dvfs(dev_priv, true);
2227
2228         *old_wm = new_wm;
2229 }
2230
2231 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2232                                    struct intel_crtc *crtc)
2233 {
2234         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2235         const struct intel_crtc_state *crtc_state =
2236                 intel_atomic_get_new_crtc_state(state, crtc);
2237
2238         mutex_lock(&dev_priv->display.wm.wm_mutex);
2239         crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2240         vlv_program_watermarks(dev_priv);
2241         mutex_unlock(&dev_priv->display.wm.wm_mutex);
2242 }
2243
2244 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2245                                     struct intel_crtc *crtc)
2246 {
2247         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2248         const struct intel_crtc_state *crtc_state =
2249                 intel_atomic_get_new_crtc_state(state, crtc);
2250
2251         if (!crtc_state->wm.need_postvbl_update)
2252                 return;
2253
2254         mutex_lock(&dev_priv->display.wm.wm_mutex);
2255         crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2256         vlv_program_watermarks(dev_priv);
2257         mutex_unlock(&dev_priv->display.wm.wm_mutex);
2258 }
2259
2260 static void i965_update_wm(struct drm_i915_private *dev_priv)
2261 {
2262         struct intel_crtc *crtc;
2263         int srwm = 1;
2264         int cursor_sr = 16;
2265         bool cxsr_enabled;
2266
2267         /* Calc sr entries for one plane configs */
2268         crtc = single_enabled_crtc(dev_priv);
2269         if (crtc) {
2270                 /* self-refresh has much higher latency */
2271                 static const int sr_latency_ns = 12000;
2272                 const struct drm_display_mode *pipe_mode =
2273                         &crtc->config->hw.pipe_mode;
2274                 const struct drm_framebuffer *fb =
2275                         crtc->base.primary->state->fb;
2276                 int pixel_rate = crtc->config->pixel_rate;
2277                 int htotal = pipe_mode->crtc_htotal;
2278                 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2279                 int cpp = fb->format->cpp[0];
2280                 int entries;
2281
2282                 entries = intel_wm_method2(pixel_rate, htotal,
2283                                            width, cpp, sr_latency_ns / 100);
2284                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2285                 srwm = I965_FIFO_SIZE - entries;
2286                 if (srwm < 0)
2287                         srwm = 1;
2288                 srwm &= 0x1ff;
2289                 drm_dbg_kms(&dev_priv->drm,
2290                             "self-refresh entries: %d, wm: %d\n",
2291                             entries, srwm);
2292
2293                 entries = intel_wm_method2(pixel_rate, htotal,
2294                                            crtc->base.cursor->state->crtc_w, 4,
2295                                            sr_latency_ns / 100);
2296                 entries = DIV_ROUND_UP(entries,
2297                                        i965_cursor_wm_info.cacheline_size) +
2298                         i965_cursor_wm_info.guard_size;
2299
2300                 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2301                 if (cursor_sr > i965_cursor_wm_info.max_wm)
2302                         cursor_sr = i965_cursor_wm_info.max_wm;
2303
2304                 drm_dbg_kms(&dev_priv->drm,
2305                             "self-refresh watermark: display plane %d "
2306                             "cursor %d\n", srwm, cursor_sr);
2307
2308                 cxsr_enabled = true;
2309         } else {
2310                 cxsr_enabled = false;
2311                 /* Turn off self refresh if both pipes are enabled */
2312                 intel_set_memory_cxsr(dev_priv, false);
2313         }
2314
2315         drm_dbg_kms(&dev_priv->drm,
2316                     "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2317                     srwm);
2318
2319         /* 965 has limitations... */
2320         intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2321                    FW_WM(8, CURSORB) |
2322                    FW_WM(8, PLANEB) |
2323                    FW_WM(8, PLANEA));
2324         intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2325                    FW_WM(8, PLANEC_OLD));
2326         /* update cursor SR watermark */
2327         intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2328
2329         if (cxsr_enabled)
2330                 intel_set_memory_cxsr(dev_priv, true);
2331 }
2332
2333 #undef FW_WM
2334
2335 static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2336                                                enum i9xx_plane_id i9xx_plane)
2337 {
2338         struct intel_plane *plane;
2339
2340         for_each_intel_plane(&i915->drm, plane) {
2341                 if (plane->id == PLANE_PRIMARY &&
2342                     plane->i9xx_plane == i9xx_plane)
2343                         return intel_crtc_for_pipe(i915, plane->pipe);
2344         }
2345
2346         return NULL;
2347 }
2348
2349 static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2350 {
2351         const struct intel_watermark_params *wm_info;
2352         u32 fwater_lo;
2353         u32 fwater_hi;
2354         int cwm, srwm = 1;
2355         int fifo_size;
2356         int planea_wm, planeb_wm;
2357         struct intel_crtc *crtc;
2358
2359         if (IS_I945GM(dev_priv))
2360                 wm_info = &i945_wm_info;
2361         else if (DISPLAY_VER(dev_priv) != 2)
2362                 wm_info = &i915_wm_info;
2363         else
2364                 wm_info = &i830_a_wm_info;
2365
2366         if (DISPLAY_VER(dev_priv) == 2)
2367                 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2368         else
2369                 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2370         crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2371         if (intel_crtc_active(crtc)) {
2372                 const struct drm_framebuffer *fb =
2373                         crtc->base.primary->state->fb;
2374                 int cpp;
2375
2376                 if (DISPLAY_VER(dev_priv) == 2)
2377                         cpp = 4;
2378                 else
2379                         cpp = fb->format->cpp[0];
2380
2381                 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2382                                                wm_info, fifo_size, cpp,
2383                                                pessimal_latency_ns);
2384         } else {
2385                 planea_wm = fifo_size - wm_info->guard_size;
2386                 if (planea_wm > (long)wm_info->max_wm)
2387                         planea_wm = wm_info->max_wm;
2388         }
2389
2390         if (DISPLAY_VER(dev_priv) == 2)
2391                 wm_info = &i830_bc_wm_info;
2392
2393         if (DISPLAY_VER(dev_priv) == 2)
2394                 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2395         else
2396                 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2397         crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2398         if (intel_crtc_active(crtc)) {
2399                 const struct drm_framebuffer *fb =
2400                         crtc->base.primary->state->fb;
2401                 int cpp;
2402
2403                 if (DISPLAY_VER(dev_priv) == 2)
2404                         cpp = 4;
2405                 else
2406                         cpp = fb->format->cpp[0];
2407
2408                 planeb_wm = intel_calculate_wm(crtc->config->pixel_rate,
2409                                                wm_info, fifo_size, cpp,
2410                                                pessimal_latency_ns);
2411         } else {
2412                 planeb_wm = fifo_size - wm_info->guard_size;
2413                 if (planeb_wm > (long)wm_info->max_wm)
2414                         planeb_wm = wm_info->max_wm;
2415         }
2416
2417         drm_dbg_kms(&dev_priv->drm,
2418                     "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2419
2420         crtc = single_enabled_crtc(dev_priv);
2421         if (IS_I915GM(dev_priv) && crtc) {
2422                 struct drm_i915_gem_object *obj;
2423
2424                 obj = intel_fb_obj(crtc->base.primary->state->fb);
2425
2426                 /* self-refresh seems busted with untiled */
2427                 if (!i915_gem_object_is_tiled(obj))
2428                         crtc = NULL;
2429         }
2430
2431         /*
2432          * Overlay gets an aggressive default since video jitter is bad.
2433          */
2434         cwm = 2;
2435
2436         /* Play safe and disable self-refresh before adjusting watermarks. */
2437         intel_set_memory_cxsr(dev_priv, false);
2438
2439         /* Calc sr entries for one plane configs */
2440         if (HAS_FW_BLC(dev_priv) && crtc) {
2441                 /* self-refresh has much higher latency */
2442                 static const int sr_latency_ns = 6000;
2443                 const struct drm_display_mode *pipe_mode =
2444                         &crtc->config->hw.pipe_mode;
2445                 const struct drm_framebuffer *fb =
2446                         crtc->base.primary->state->fb;
2447                 int pixel_rate = crtc->config->pixel_rate;
2448                 int htotal = pipe_mode->crtc_htotal;
2449                 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2450                 int cpp;
2451                 int entries;
2452
2453                 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2454                         cpp = 4;
2455                 else
2456                         cpp = fb->format->cpp[0];
2457
2458                 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2459                                            sr_latency_ns / 100);
2460                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2461                 drm_dbg_kms(&dev_priv->drm,
2462                             "self-refresh entries: %d\n", entries);
2463                 srwm = wm_info->fifo_size - entries;
2464                 if (srwm < 0)
2465                         srwm = 1;
2466
2467                 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2468                         intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2469                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2470                 else
2471                         intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2472         }
2473
2474         drm_dbg_kms(&dev_priv->drm,
2475                     "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2476                      planea_wm, planeb_wm, cwm, srwm);
2477
2478         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2479         fwater_hi = (cwm & 0x1f);
2480
2481         /* Set request length to 8 cachelines per fetch */
2482         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2483         fwater_hi = fwater_hi | (1 << 8);
2484
2485         intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2486         intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2487
2488         if (crtc)
2489                 intel_set_memory_cxsr(dev_priv, true);
2490 }
2491
2492 static void i845_update_wm(struct drm_i915_private *dev_priv)
2493 {
2494         struct intel_crtc *crtc;
2495         u32 fwater_lo;
2496         int planea_wm;
2497
2498         crtc = single_enabled_crtc(dev_priv);
2499         if (crtc == NULL)
2500                 return;
2501
2502         planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2503                                        &i845_wm_info,
2504                                        i845_get_fifo_size(dev_priv, PLANE_A),
2505                                        4, pessimal_latency_ns);
2506         fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2507         fwater_lo |= (3<<8) | planea_wm;
2508
2509         drm_dbg_kms(&dev_priv->drm,
2510                     "Setting FIFO watermarks - A: %d\n", planea_wm);
2511
2512         intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2513 }
2514
2515 /* latency must be in 0.1us units. */
2516 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2517                                    unsigned int cpp,
2518                                    unsigned int latency)
2519 {
2520         unsigned int ret;
2521
2522         ret = intel_wm_method1(pixel_rate, cpp, latency);
2523         ret = DIV_ROUND_UP(ret, 64) + 2;
2524
2525         return ret;
2526 }
2527
2528 /* latency must be in 0.1us units. */
2529 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2530                                    unsigned int htotal,
2531                                    unsigned int width,
2532                                    unsigned int cpp,
2533                                    unsigned int latency)
2534 {
2535         unsigned int ret;
2536
2537         ret = intel_wm_method2(pixel_rate, htotal,
2538                                width, cpp, latency);
2539         ret = DIV_ROUND_UP(ret, 64) + 2;
2540
2541         return ret;
2542 }
2543
2544 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2545 {
2546         /*
2547          * Neither of these should be possible since this function shouldn't be
2548          * called if the CRTC is off or the plane is invisible.  But let's be
2549          * extra paranoid to avoid a potential divide-by-zero if we screw up
2550          * elsewhere in the driver.
2551          */
2552         if (WARN_ON(!cpp))
2553                 return 0;
2554         if (WARN_ON(!horiz_pixels))
2555                 return 0;
2556
2557         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2558 }
2559
2560 struct ilk_wm_maximums {
2561         u16 pri;
2562         u16 spr;
2563         u16 cur;
2564         u16 fbc;
2565 };
2566
2567 /*
2568  * For both WM_PIPE and WM_LP.
2569  * mem_value must be in 0.1us units.
2570  */
2571 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2572                               const struct intel_plane_state *plane_state,
2573                               u32 mem_value, bool is_lp)
2574 {
2575         u32 method1, method2;
2576         int cpp;
2577
2578         if (mem_value == 0)
2579                 return U32_MAX;
2580
2581         if (!intel_wm_plane_visible(crtc_state, plane_state))
2582                 return 0;
2583
2584         cpp = plane_state->hw.fb->format->cpp[0];
2585
2586         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2587
2588         if (!is_lp)
2589                 return method1;
2590
2591         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2592                                  crtc_state->hw.pipe_mode.crtc_htotal,
2593                                  drm_rect_width(&plane_state->uapi.src) >> 16,
2594                                  cpp, mem_value);
2595
2596         return min(method1, method2);
2597 }
2598
2599 /*
2600  * For both WM_PIPE and WM_LP.
2601  * mem_value must be in 0.1us units.
2602  */
2603 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2604                               const struct intel_plane_state *plane_state,
2605                               u32 mem_value)
2606 {
2607         u32 method1, method2;
2608         int cpp;
2609
2610         if (mem_value == 0)
2611                 return U32_MAX;
2612
2613         if (!intel_wm_plane_visible(crtc_state, plane_state))
2614                 return 0;
2615
2616         cpp = plane_state->hw.fb->format->cpp[0];
2617
2618         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2619         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2620                                  crtc_state->hw.pipe_mode.crtc_htotal,
2621                                  drm_rect_width(&plane_state->uapi.src) >> 16,
2622                                  cpp, mem_value);
2623         return min(method1, method2);
2624 }
2625
2626 /*
2627  * For both WM_PIPE and WM_LP.
2628  * mem_value must be in 0.1us units.
2629  */
2630 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2631                               const struct intel_plane_state *plane_state,
2632                               u32 mem_value)
2633 {
2634         int cpp;
2635
2636         if (mem_value == 0)
2637                 return U32_MAX;
2638
2639         if (!intel_wm_plane_visible(crtc_state, plane_state))
2640                 return 0;
2641
2642         cpp = plane_state->hw.fb->format->cpp[0];
2643
2644         return ilk_wm_method2(crtc_state->pixel_rate,
2645                               crtc_state->hw.pipe_mode.crtc_htotal,
2646                               drm_rect_width(&plane_state->uapi.src) >> 16,
2647                               cpp, mem_value);
2648 }
2649
2650 /* Only for WM_LP. */
2651 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2652                               const struct intel_plane_state *plane_state,
2653                               u32 pri_val)
2654 {
2655         int cpp;
2656
2657         if (!intel_wm_plane_visible(crtc_state, plane_state))
2658                 return 0;
2659
2660         cpp = plane_state->hw.fb->format->cpp[0];
2661
2662         return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2663                           cpp);
2664 }
2665
2666 static unsigned int
2667 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2668 {
2669         if (DISPLAY_VER(dev_priv) >= 8)
2670                 return 3072;
2671         else if (DISPLAY_VER(dev_priv) >= 7)
2672                 return 768;
2673         else
2674                 return 512;
2675 }
2676
2677 static unsigned int
2678 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2679                      int level, bool is_sprite)
2680 {
2681         if (DISPLAY_VER(dev_priv) >= 8)
2682                 /* BDW primary/sprite plane watermarks */
2683                 return level == 0 ? 255 : 2047;
2684         else if (DISPLAY_VER(dev_priv) >= 7)
2685                 /* IVB/HSW primary/sprite plane watermarks */
2686                 return level == 0 ? 127 : 1023;
2687         else if (!is_sprite)
2688                 /* ILK/SNB primary plane watermarks */
2689                 return level == 0 ? 127 : 511;
2690         else
2691                 /* ILK/SNB sprite plane watermarks */
2692                 return level == 0 ? 63 : 255;
2693 }
2694
2695 static unsigned int
2696 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2697 {
2698         if (DISPLAY_VER(dev_priv) >= 7)
2699                 return level == 0 ? 63 : 255;
2700         else
2701                 return level == 0 ? 31 : 63;
2702 }
2703
2704 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2705 {
2706         if (DISPLAY_VER(dev_priv) >= 8)
2707                 return 31;
2708         else
2709                 return 15;
2710 }
2711
2712 /* Calculate the maximum primary/sprite plane watermark */
2713 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2714                                      int level,
2715                                      const struct intel_wm_config *config,
2716                                      enum intel_ddb_partitioning ddb_partitioning,
2717                                      bool is_sprite)
2718 {
2719         unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2720
2721         /* if sprites aren't enabled, sprites get nothing */
2722         if (is_sprite && !config->sprites_enabled)
2723                 return 0;
2724
2725         /* HSW allows LP1+ watermarks even with multiple pipes */
2726         if (level == 0 || config->num_pipes_active > 1) {
2727                 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2728
2729                 /*
2730                  * For some reason the non self refresh
2731                  * FIFO size is only half of the self
2732                  * refresh FIFO size on ILK/SNB.
2733                  */
2734                 if (DISPLAY_VER(dev_priv) <= 6)
2735                         fifo_size /= 2;
2736         }
2737
2738         if (config->sprites_enabled) {
2739                 /* level 0 is always calculated with 1:1 split */
2740                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2741                         if (is_sprite)
2742                                 fifo_size *= 5;
2743                         fifo_size /= 6;
2744                 } else {
2745                         fifo_size /= 2;
2746                 }
2747         }
2748
2749         /* clamp to max that the registers can hold */
2750         return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2751 }
2752
2753 /* Calculate the maximum cursor plane watermark */
2754 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2755                                       int level,
2756                                       const struct intel_wm_config *config)
2757 {
2758         /* HSW LP1+ watermarks w/ multiple pipes */
2759         if (level > 0 && config->num_pipes_active > 1)
2760                 return 64;
2761
2762         /* otherwise just report max that registers can hold */
2763         return ilk_cursor_wm_reg_max(dev_priv, level);
2764 }
2765
2766 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2767                                     int level,
2768                                     const struct intel_wm_config *config,
2769                                     enum intel_ddb_partitioning ddb_partitioning,
2770                                     struct ilk_wm_maximums *max)
2771 {
2772         max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2773         max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2774         max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2775         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2776 }
2777
2778 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2779                                         int level,
2780                                         struct ilk_wm_maximums *max)
2781 {
2782         max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2783         max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2784         max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2785         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2786 }
2787
2788 static bool ilk_validate_wm_level(int level,
2789                                   const struct ilk_wm_maximums *max,
2790                                   struct intel_wm_level *result)
2791 {
2792         bool ret;
2793
2794         /* already determined to be invalid? */
2795         if (!result->enable)
2796                 return false;
2797
2798         result->enable = result->pri_val <= max->pri &&
2799                          result->spr_val <= max->spr &&
2800                          result->cur_val <= max->cur;
2801
2802         ret = result->enable;
2803
2804         /*
2805          * HACK until we can pre-compute everything,
2806          * and thus fail gracefully if LP0 watermarks
2807          * are exceeded...
2808          */
2809         if (level == 0 && !result->enable) {
2810                 if (result->pri_val > max->pri)
2811                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2812                                       level, result->pri_val, max->pri);
2813                 if (result->spr_val > max->spr)
2814                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2815                                       level, result->spr_val, max->spr);
2816                 if (result->cur_val > max->cur)
2817                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2818                                       level, result->cur_val, max->cur);
2819
2820                 result->pri_val = min_t(u32, result->pri_val, max->pri);
2821                 result->spr_val = min_t(u32, result->spr_val, max->spr);
2822                 result->cur_val = min_t(u32, result->cur_val, max->cur);
2823                 result->enable = true;
2824         }
2825
2826         return ret;
2827 }
2828
2829 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2830                                  const struct intel_crtc *crtc,
2831                                  int level,
2832                                  struct intel_crtc_state *crtc_state,
2833                                  const struct intel_plane_state *pristate,
2834                                  const struct intel_plane_state *sprstate,
2835                                  const struct intel_plane_state *curstate,
2836                                  struct intel_wm_level *result)
2837 {
2838         u16 pri_latency = dev_priv->display.wm.pri_latency[level];
2839         u16 spr_latency = dev_priv->display.wm.spr_latency[level];
2840         u16 cur_latency = dev_priv->display.wm.cur_latency[level];
2841
2842         /* WM1+ latency values stored in 0.5us units */
2843         if (level > 0) {
2844                 pri_latency *= 5;
2845                 spr_latency *= 5;
2846                 cur_latency *= 5;
2847         }
2848
2849         if (pristate) {
2850                 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2851                                                      pri_latency, level);
2852                 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2853         }
2854
2855         if (sprstate)
2856                 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2857
2858         if (curstate)
2859                 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2860
2861         result->enable = true;
2862 }
2863
2864 static void
2865 adjust_wm_latency(struct drm_i915_private *i915,
2866                   u16 wm[], int max_level, int read_latency)
2867 {
2868         bool wm_lv_0_adjust_needed = i915->dram_info.wm_lv_0_adjust_needed;
2869         int i, level;
2870
2871         /*
2872          * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2873          * need to be disabled. We make sure to sanitize the values out
2874          * of the punit to satisfy this requirement.
2875          */
2876         for (level = 1; level <= max_level; level++) {
2877                 if (wm[level] == 0) {
2878                         for (i = level + 1; i <= max_level; i++)
2879                                 wm[i] = 0;
2880
2881                         max_level = level - 1;
2882                         break;
2883                 }
2884         }
2885
2886         /*
2887          * WaWmMemoryReadLatency
2888          *
2889          * punit doesn't take into account the read latency so we need
2890          * to add proper adjustement to each valid level we retrieve
2891          * from the punit when level 0 response data is 0us.
2892          */
2893         if (wm[0] == 0) {
2894                 for (level = 0; level <= max_level; level++)
2895                         wm[level] += read_latency;
2896         }
2897
2898         /*
2899          * WA Level-0 adjustment for 16GB DIMMs: SKL+
2900          * If we could not get dimm info enable this WA to prevent from
2901          * any underrun. If not able to get Dimm info assume 16GB dimm
2902          * to avoid any underrun.
2903          */
2904         if (wm_lv_0_adjust_needed)
2905                 wm[0] += 1;
2906 }
2907
2908 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2909                                   u16 wm[])
2910 {
2911         struct intel_uncore *uncore = &dev_priv->uncore;
2912         int max_level = ilk_wm_max_level(dev_priv);
2913
2914         if (DISPLAY_VER(dev_priv) >= 14) {
2915                 u32 val;
2916
2917                 val = intel_uncore_read(uncore, MTL_LATENCY_LP0_LP1);
2918                 wm[0] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
2919                 wm[1] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
2920                 val = intel_uncore_read(uncore, MTL_LATENCY_LP2_LP3);
2921                 wm[2] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
2922                 wm[3] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
2923                 val = intel_uncore_read(uncore, MTL_LATENCY_LP4_LP5);
2924                 wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
2925                 wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
2926
2927                 adjust_wm_latency(dev_priv, wm, max_level, 6);
2928         } else if (DISPLAY_VER(dev_priv) >= 9) {
2929                 int read_latency = DISPLAY_VER(dev_priv) >= 12 ? 3 : 2;
2930                 int mult = IS_DG2(dev_priv) ? 2 : 1;
2931                 u32 val;
2932                 int ret;
2933
2934                 /* read the first set of memory latencies[0:3] */
2935                 val = 0; /* data0 to be programmed to 0 for first set */
2936                 ret = snb_pcode_read(&dev_priv->uncore, GEN9_PCODE_READ_MEM_LATENCY,
2937                                      &val, NULL);
2938
2939                 if (ret) {
2940                         drm_err(&dev_priv->drm,
2941                                 "SKL Mailbox read error = %d\n", ret);
2942                         return;
2943                 }
2944
2945                 wm[0] = (val & GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2946                 wm[1] = ((val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2947                                 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2948                 wm[2] = ((val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2949                                 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2950                 wm[3] = ((val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2951                                 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2952
2953                 /* read the second set of memory latencies[4:7] */
2954                 val = 1; /* data0 to be programmed to 1 for second set */
2955                 ret = snb_pcode_read(&dev_priv->uncore, GEN9_PCODE_READ_MEM_LATENCY,
2956                                      &val, NULL);
2957                 if (ret) {
2958                         drm_err(&dev_priv->drm,
2959                                 "SKL Mailbox read error = %d\n", ret);
2960                         return;
2961                 }
2962
2963                 wm[4] = (val & GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2964                 wm[5] = ((val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2965                                 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2966                 wm[6] = ((val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2967                                 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2968                 wm[7] = ((val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2969                                 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2970
2971                 adjust_wm_latency(dev_priv, wm, max_level, read_latency);
2972         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2973                 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2974
2975                 wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2976                 if (wm[0] == 0)
2977                         wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2978                 wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2979                 wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2980                 wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2981                 wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2982         } else if (DISPLAY_VER(dev_priv) >= 6) {
2983                 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2984
2985                 wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2986                 wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2987                 wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2988                 wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2989         } else if (DISPLAY_VER(dev_priv) >= 5) {
2990                 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2991
2992                 /* ILK primary LP0 latency is 700 ns */
2993                 wm[0] = 7;
2994                 wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2995                 wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2996         } else {
2997                 MISSING_CASE(INTEL_DEVID(dev_priv));
2998         }
2999 }
3000
3001 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
3002                                        u16 wm[5])
3003 {
3004         /* ILK sprite LP0 latency is 1300 ns */
3005         if (DISPLAY_VER(dev_priv) == 5)
3006                 wm[0] = 13;
3007 }
3008
3009 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
3010                                        u16 wm[5])
3011 {
3012         /* ILK cursor LP0 latency is 1300 ns */
3013         if (DISPLAY_VER(dev_priv) == 5)
3014                 wm[0] = 13;
3015 }
3016
3017 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
3018 {
3019         /* how many WM levels are we expecting */
3020         if (HAS_HW_SAGV_WM(dev_priv))
3021                 return 5;
3022         else if (DISPLAY_VER(dev_priv) >= 9)
3023                 return 7;
3024         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3025                 return 4;
3026         else if (DISPLAY_VER(dev_priv) >= 6)
3027                 return 3;
3028         else
3029                 return 2;
3030 }
3031
3032 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
3033                                    const char *name,
3034                                    const u16 wm[])
3035 {
3036         int level, max_level = ilk_wm_max_level(dev_priv);
3037
3038         for (level = 0; level <= max_level; level++) {
3039                 unsigned int latency = wm[level];
3040
3041                 if (latency == 0) {
3042                         drm_dbg_kms(&dev_priv->drm,
3043                                     "%s WM%d latency not provided\n",
3044                                     name, level);
3045                         continue;
3046                 }
3047
3048                 /*
3049                  * - latencies are in us on gen9.
3050                  * - before then, WM1+ latency values are in 0.5us units
3051                  */
3052                 if (DISPLAY_VER(dev_priv) >= 9)
3053                         latency *= 10;
3054                 else if (level > 0)
3055                         latency *= 5;
3056
3057                 drm_dbg_kms(&dev_priv->drm,
3058                             "%s WM%d latency %u (%u.%u usec)\n", name, level,
3059                             wm[level], latency / 10, latency % 10);
3060         }
3061 }
3062
3063 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3064                                     u16 wm[5], u16 min)
3065 {
3066         int level, max_level = ilk_wm_max_level(dev_priv);
3067
3068         if (wm[0] >= min)
3069                 return false;
3070
3071         wm[0] = max(wm[0], min);
3072         for (level = 1; level <= max_level; level++)
3073                 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3074
3075         return true;
3076 }
3077
3078 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3079 {
3080         bool changed;
3081
3082         /*
3083          * The BIOS provided WM memory latency values are often
3084          * inadequate for high resolution displays. Adjust them.
3085          */
3086         changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);
3087         changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);
3088         changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);
3089
3090         if (!changed)
3091                 return;
3092
3093         drm_dbg_kms(&dev_priv->drm,
3094                     "WM latency values increased to avoid potential underruns\n");
3095         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
3096         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
3097         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
3098 }
3099
3100 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3101 {
3102         /*
3103          * On some SNB machines (Thinkpad X220 Tablet at least)
3104          * LP3 usage can cause vblank interrupts to be lost.
3105          * The DEIIR bit will go high but it looks like the CPU
3106          * never gets interrupted.
3107          *
3108          * It's not clear whether other interrupt source could
3109          * be affected or if this is somehow limited to vblank
3110          * interrupts only. To play it safe we disable LP3
3111          * watermarks entirely.
3112          */
3113         if (dev_priv->display.wm.pri_latency[3] == 0 &&
3114             dev_priv->display.wm.spr_latency[3] == 0 &&
3115             dev_priv->display.wm.cur_latency[3] == 0)
3116                 return;
3117
3118         dev_priv->display.wm.pri_latency[3] = 0;
3119         dev_priv->display.wm.spr_latency[3] = 0;
3120         dev_priv->display.wm.cur_latency[3] = 0;
3121
3122         drm_dbg_kms(&dev_priv->drm,
3123                     "LP3 watermarks disabled due to potential for lost interrupts\n");
3124         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
3125         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
3126         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
3127 }
3128
3129 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3130 {
3131         intel_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3132
3133         memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,
3134                sizeof(dev_priv->display.wm.pri_latency));
3135         memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,
3136                sizeof(dev_priv->display.wm.pri_latency));
3137
3138         intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);
3139         intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);
3140
3141         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
3142         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
3143         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
3144
3145         if (DISPLAY_VER(dev_priv) == 6) {
3146                 snb_wm_latency_quirk(dev_priv);
3147                 snb_wm_lp3_irq_quirk(dev_priv);
3148         }
3149 }
3150
3151 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3152 {
3153         intel_read_wm_latency(dev_priv, dev_priv->display.wm.skl_latency);
3154         intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->display.wm.skl_latency);
3155 }
3156
3157 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3158                                  struct intel_pipe_wm *pipe_wm)
3159 {
3160         /* LP0 watermark maximums depend on this pipe alone */
3161         const struct intel_wm_config config = {
3162                 .num_pipes_active = 1,
3163                 .sprites_enabled = pipe_wm->sprites_enabled,
3164                 .sprites_scaled = pipe_wm->sprites_scaled,
3165         };
3166         struct ilk_wm_maximums max;
3167
3168         /* LP0 watermarks always use 1/2 DDB partitioning */
3169         ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3170
3171         /* At least LP0 must be valid */
3172         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3173                 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3174                 return false;
3175         }
3176
3177         return true;
3178 }
3179
3180 /* Compute new watermarks for the pipe */
3181 static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
3182                                struct intel_crtc *crtc)
3183 {
3184         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3185         struct intel_crtc_state *crtc_state =
3186                 intel_atomic_get_new_crtc_state(state, crtc);
3187         struct intel_pipe_wm *pipe_wm;
3188         struct intel_plane *plane;
3189         const struct intel_plane_state *plane_state;
3190         const struct intel_plane_state *pristate = NULL;
3191         const struct intel_plane_state *sprstate = NULL;
3192         const struct intel_plane_state *curstate = NULL;
3193         int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3194         struct ilk_wm_maximums max;
3195
3196         pipe_wm = &crtc_state->wm.ilk.optimal;
3197
3198         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3199                 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3200                         pristate = plane_state;
3201                 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3202                         sprstate = plane_state;
3203                 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3204                         curstate = plane_state;
3205         }
3206
3207         pipe_wm->pipe_enabled = crtc_state->hw.active;
3208         pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
3209         pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
3210
3211         usable_level = max_level;
3212
3213         /* ILK/SNB: LP2+ watermarks only w/o sprites */
3214         if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3215                 usable_level = 1;
3216
3217         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3218         if (pipe_wm->sprites_scaled)
3219                 usable_level = 0;
3220
3221         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3222         ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3223                              pristate, sprstate, curstate, &pipe_wm->wm[0]);
3224
3225         if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3226                 return -EINVAL;
3227
3228         ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3229
3230         for (level = 1; level <= usable_level; level++) {
3231                 struct intel_wm_level *wm = &pipe_wm->wm[level];
3232
3233                 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3234                                      pristate, sprstate, curstate, wm);
3235
3236                 /*
3237                  * Disable any watermark level that exceeds the
3238                  * register maximums since such watermarks are
3239                  * always invalid.
3240                  */
3241                 if (!ilk_validate_wm_level(level, &max, wm)) {
3242                         memset(wm, 0, sizeof(*wm));
3243                         break;
3244                 }
3245         }
3246
3247         return 0;
3248 }
3249
3250 /*
3251  * Build a set of 'intermediate' watermark values that satisfy both the old
3252  * state and the new state.  These can be programmed to the hardware
3253  * immediately.
3254  */
3255 static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
3256                                        struct intel_crtc *crtc)
3257 {
3258         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3259         struct intel_crtc_state *new_crtc_state =
3260                 intel_atomic_get_new_crtc_state(state, crtc);
3261         const struct intel_crtc_state *old_crtc_state =
3262                 intel_atomic_get_old_crtc_state(state, crtc);
3263         struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
3264         const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
3265         int level, max_level = ilk_wm_max_level(dev_priv);
3266
3267         /*
3268          * Start with the final, target watermarks, then combine with the
3269          * currently active watermarks to get values that are safe both before
3270          * and after the vblank.
3271          */
3272         *a = new_crtc_state->wm.ilk.optimal;
3273         if (!new_crtc_state->hw.active ||
3274             drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) ||
3275             state->skip_intermediate_wm)
3276                 return 0;
3277
3278         a->pipe_enabled |= b->pipe_enabled;
3279         a->sprites_enabled |= b->sprites_enabled;
3280         a->sprites_scaled |= b->sprites_scaled;
3281
3282         for (level = 0; level <= max_level; level++) {
3283                 struct intel_wm_level *a_wm = &a->wm[level];
3284                 const struct intel_wm_level *b_wm = &b->wm[level];
3285
3286                 a_wm->enable &= b_wm->enable;
3287                 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3288                 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3289                 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3290                 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3291         }
3292
3293         /*
3294          * We need to make sure that these merged watermark values are
3295          * actually a valid configuration themselves.  If they're not,
3296          * there's no safe way to transition from the old state to
3297          * the new state, so we need to fail the atomic transaction.
3298          */
3299         if (!ilk_validate_pipe_wm(dev_priv, a))
3300                 return -EINVAL;
3301
3302         /*
3303          * If our intermediate WM are identical to the final WM, then we can
3304          * omit the post-vblank programming; only update if it's different.
3305          */
3306         if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
3307                 new_crtc_state->wm.need_postvbl_update = true;
3308
3309         return 0;
3310 }
3311
3312 /*
3313  * Merge the watermarks from all active pipes for a specific level.
3314  */
3315 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3316                                int level,
3317                                struct intel_wm_level *ret_wm)
3318 {
3319         const struct intel_crtc *crtc;
3320
3321         ret_wm->enable = true;
3322
3323         for_each_intel_crtc(&dev_priv->drm, crtc) {
3324                 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3325                 const struct intel_wm_level *wm = &active->wm[level];
3326
3327                 if (!active->pipe_enabled)
3328                         continue;
3329
3330                 /*
3331                  * The watermark values may have been used in the past,
3332                  * so we must maintain them in the registers for some
3333                  * time even if the level is now disabled.
3334                  */
3335                 if (!wm->enable)
3336                         ret_wm->enable = false;
3337
3338                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3339                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3340                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3341                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3342         }
3343 }
3344
3345 /*
3346  * Merge all low power watermarks for all active pipes.
3347  */
3348 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3349                          const struct intel_wm_config *config,
3350                          const struct ilk_wm_maximums *max,
3351                          struct intel_pipe_wm *merged)
3352 {
3353         int level, max_level = ilk_wm_max_level(dev_priv);
3354         int last_enabled_level = max_level;
3355
3356         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3357         if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3358             config->num_pipes_active > 1)
3359                 last_enabled_level = 0;
3360
3361         /* ILK: FBC WM must be disabled always */
3362         merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3363
3364         /* merge each WM1+ level */
3365         for (level = 1; level <= max_level; level++) {
3366                 struct intel_wm_level *wm = &merged->wm[level];
3367
3368                 ilk_merge_wm_level(dev_priv, level, wm);
3369
3370                 if (level > last_enabled_level)
3371                         wm->enable = false;
3372                 else if (!ilk_validate_wm_level(level, max, wm))
3373                         /* make sure all following levels get disabled */
3374                         last_enabled_level = level - 1;
3375
3376                 /*
3377                  * The spec says it is preferred to disable
3378                  * FBC WMs instead of disabling a WM level.
3379                  */
3380                 if (wm->fbc_val > max->fbc) {
3381                         if (wm->enable)
3382                                 merged->fbc_wm_enabled = false;
3383                         wm->fbc_val = 0;
3384                 }
3385         }
3386
3387         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3388         if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
3389             dev_priv->params.enable_fbc && !merged->fbc_wm_enabled) {
3390                 for (level = 2; level <= max_level; level++) {
3391                         struct intel_wm_level *wm = &merged->wm[level];
3392
3393                         wm->enable = false;
3394                 }
3395         }
3396 }
3397
3398 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3399 {
3400         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3401         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3402 }
3403
3404 /* The value we need to program into the WM_LPx latency field */
3405 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3406                                       int level)
3407 {
3408         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3409                 return 2 * level;
3410         else
3411                 return dev_priv->display.wm.pri_latency[level];
3412 }
3413
3414 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3415                                    const struct intel_pipe_wm *merged,
3416                                    enum intel_ddb_partitioning partitioning,
3417                                    struct ilk_wm_values *results)
3418 {
3419         struct intel_crtc *crtc;
3420         int level, wm_lp;
3421
3422         results->enable_fbc_wm = merged->fbc_wm_enabled;
3423         results->partitioning = partitioning;
3424
3425         /* LP1+ register values */
3426         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3427                 const struct intel_wm_level *r;
3428
3429                 level = ilk_wm_lp_to_level(wm_lp, merged);
3430
3431                 r = &merged->wm[level];
3432
3433                 /*
3434                  * Maintain the watermark values even if the level is
3435                  * disabled. Doing otherwise could cause underruns.
3436                  */
3437                 results->wm_lp[wm_lp - 1] =
3438                         WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3439                         WM_LP_PRIMARY(r->pri_val) |
3440                         WM_LP_CURSOR(r->cur_val);
3441
3442                 if (r->enable)
3443                         results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3444
3445                 if (DISPLAY_VER(dev_priv) >= 8)
3446                         results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3447                 else
3448                         results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3449
3450                 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3451
3452                 /*
3453                  * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3454                  * level is disabled. Doing otherwise could cause underruns.
3455                  */
3456                 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3457                         drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3458                         results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3459                 }
3460         }
3461
3462         /* LP0 register values */
3463         for_each_intel_crtc(&dev_priv->drm, crtc) {
3464                 enum pipe pipe = crtc->pipe;
3465                 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3466                 const struct intel_wm_level *r = &pipe_wm->wm[0];
3467
3468                 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3469                         continue;
3470
3471                 results->wm_pipe[pipe] =
3472                         WM0_PIPE_PRIMARY(r->pri_val) |
3473                         WM0_PIPE_SPRITE(r->spr_val) |
3474                         WM0_PIPE_CURSOR(r->cur_val);
3475         }
3476 }
3477
3478 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3479  * case both are at the same level. Prefer r1 in case they're the same. */
3480 static struct intel_pipe_wm *
3481 ilk_find_best_result(struct drm_i915_private *dev_priv,
3482                      struct intel_pipe_wm *r1,
3483                      struct intel_pipe_wm *r2)
3484 {
3485         int level, max_level = ilk_wm_max_level(dev_priv);
3486         int level1 = 0, level2 = 0;
3487
3488         for (level = 1; level <= max_level; level++) {
3489                 if (r1->wm[level].enable)
3490                         level1 = level;
3491                 if (r2->wm[level].enable)
3492                         level2 = level;
3493         }
3494
3495         if (level1 == level2) {
3496                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3497                         return r2;
3498                 else
3499                         return r1;
3500         } else if (level1 > level2) {
3501                 return r1;
3502         } else {
3503                 return r2;
3504         }
3505 }
3506
3507 /* dirty bits used to track which watermarks need changes */
3508 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3509 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3510 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3511 #define WM_DIRTY_FBC (1 << 24)
3512 #define WM_DIRTY_DDB (1 << 25)
3513
3514 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3515                                          const struct ilk_wm_values *old,
3516                                          const struct ilk_wm_values *new)
3517 {
3518         unsigned int dirty = 0;
3519         enum pipe pipe;
3520         int wm_lp;
3521
3522         for_each_pipe(dev_priv, pipe) {
3523                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3524                         dirty |= WM_DIRTY_PIPE(pipe);
3525                         /* Must disable LP1+ watermarks too */
3526                         dirty |= WM_DIRTY_LP_ALL;
3527                 }
3528         }
3529
3530         if (old->enable_fbc_wm != new->enable_fbc_wm) {
3531                 dirty |= WM_DIRTY_FBC;
3532                 /* Must disable LP1+ watermarks too */
3533                 dirty |= WM_DIRTY_LP_ALL;
3534         }
3535
3536         if (old->partitioning != new->partitioning) {
3537                 dirty |= WM_DIRTY_DDB;
3538                 /* Must disable LP1+ watermarks too */
3539                 dirty |= WM_DIRTY_LP_ALL;
3540         }
3541
3542         /* LP1+ watermarks already deemed dirty, no need to continue */
3543         if (dirty & WM_DIRTY_LP_ALL)
3544                 return dirty;
3545
3546         /* Find the lowest numbered LP1+ watermark in need of an update... */
3547         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3548                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3549                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3550                         break;
3551         }
3552
3553         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3554         for (; wm_lp <= 3; wm_lp++)
3555                 dirty |= WM_DIRTY_LP(wm_lp);
3556
3557         return dirty;
3558 }
3559
3560 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3561                                unsigned int dirty)
3562 {
3563         struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3564         bool changed = false;
3565
3566         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3567                 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3568                 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3569                 changed = true;
3570         }
3571         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3572                 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3573                 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3574                 changed = true;
3575         }
3576         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3577                 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3578                 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3579                 changed = true;
3580         }
3581
3582         /*
3583          * Don't touch WM_LP_SPRITE_ENABLE here.
3584          * Doing so could cause underruns.
3585          */
3586
3587         return changed;
3588 }
3589
3590 /*
3591  * The spec says we shouldn't write when we don't need, because every write
3592  * causes WMs to be re-evaluated, expending some power.
3593  */
3594 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3595                                 struct ilk_wm_values *results)
3596 {
3597         struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3598         unsigned int dirty;
3599         u32 val;
3600
3601         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3602         if (!dirty)
3603                 return;
3604
3605         _ilk_disable_lp_wm(dev_priv, dirty);
3606
3607         if (dirty & WM_DIRTY_PIPE(PIPE_A))
3608                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3609         if (dirty & WM_DIRTY_PIPE(PIPE_B))
3610                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3611         if (dirty & WM_DIRTY_PIPE(PIPE_C))
3612                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3613
3614         if (dirty & WM_DIRTY_DDB) {
3615                 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3616                         val = intel_uncore_read(&dev_priv->uncore, WM_MISC);
3617                         if (results->partitioning == INTEL_DDB_PART_1_2)
3618                                 val &= ~WM_MISC_DATA_PARTITION_5_6;
3619                         else
3620                                 val |= WM_MISC_DATA_PARTITION_5_6;
3621                         intel_uncore_write(&dev_priv->uncore, WM_MISC, val);
3622                 } else {
3623                         val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
3624                         if (results->partitioning == INTEL_DDB_PART_1_2)
3625                                 val &= ~DISP_DATA_PARTITION_5_6;
3626                         else
3627                                 val |= DISP_DATA_PARTITION_5_6;
3628                         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
3629                 }
3630         }
3631
3632         if (dirty & WM_DIRTY_FBC) {
3633                 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL);
3634                 if (results->enable_fbc_wm)
3635                         val &= ~DISP_FBC_WM_DIS;
3636                 else
3637                         val |= DISP_FBC_WM_DIS;
3638                 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, val);
3639         }
3640
3641         if (dirty & WM_DIRTY_LP(1) &&
3642             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3643                 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3644
3645         if (DISPLAY_VER(dev_priv) >= 7) {
3646                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3647                         intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3648                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3649                         intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3650         }
3651
3652         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3653                 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3654         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3655                 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3656         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3657                 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3658
3659         dev_priv->display.wm.hw = *results;
3660 }
3661
3662 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3663 {
3664         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3665 }
3666
3667 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3668 {
3669         u8 enabled_slices = 0;
3670         enum dbuf_slice slice;
3671
3672         for_each_dbuf_slice(dev_priv, slice) {
3673                 if (intel_uncore_read(&dev_priv->uncore,
3674                                       DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
3675                         enabled_slices |= BIT(slice);
3676         }
3677
3678         return enabled_slices;
3679 }
3680
3681 /*
3682  * FIXME: We still don't have the proper code detect if we need to apply the WA,
3683  * so assume we'll always need it in order to avoid underruns.
3684  */
3685 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3686 {
3687         return DISPLAY_VER(dev_priv) == 9;
3688 }
3689
3690 static bool
3691 intel_has_sagv(struct drm_i915_private *dev_priv)
3692 {
3693         return DISPLAY_VER(dev_priv) >= 9 && !IS_LP(dev_priv) &&
3694                 dev_priv->display.sagv.status != I915_SAGV_NOT_CONTROLLED;
3695 }
3696
3697 static u32
3698 intel_sagv_block_time(struct drm_i915_private *dev_priv)
3699 {
3700         if (DISPLAY_VER(dev_priv) >= 12) {
3701                 u32 val = 0;
3702                 int ret;
3703
3704                 ret = snb_pcode_read(&dev_priv->uncore,
3705                                      GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3706                                      &val, NULL);
3707                 if (ret) {
3708                         drm_dbg_kms(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3709                         return 0;
3710                 }
3711
3712                 return val;
3713         } else if (DISPLAY_VER(dev_priv) == 11) {
3714                 return 10;
3715         } else if (DISPLAY_VER(dev_priv) == 9 && !IS_LP(dev_priv)) {
3716                 return 30;
3717         } else {
3718                 return 0;
3719         }
3720 }
3721
3722 static void intel_sagv_init(struct drm_i915_private *i915)
3723 {
3724         if (!intel_has_sagv(i915))
3725                 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
3726
3727         /*
3728          * Probe to see if we have working SAGV control.
3729          * For icl+ this was already determined by intel_bw_init_hw().
3730          */
3731         if (DISPLAY_VER(i915) < 11)
3732                 skl_sagv_disable(i915);
3733
3734         drm_WARN_ON(&i915->drm, i915->display.sagv.status == I915_SAGV_UNKNOWN);
3735
3736         i915->display.sagv.block_time_us = intel_sagv_block_time(i915);
3737
3738         drm_dbg_kms(&i915->drm, "SAGV supported: %s, original SAGV block time: %u us\n",
3739                     str_yes_no(intel_has_sagv(i915)), i915->display.sagv.block_time_us);
3740
3741         /* avoid overflow when adding with wm0 latency/etc. */
3742         if (drm_WARN(&i915->drm, i915->display.sagv.block_time_us > U16_MAX,
3743                      "Excessive SAGV block time %u, ignoring\n",
3744                      i915->display.sagv.block_time_us))
3745                 i915->display.sagv.block_time_us = 0;
3746
3747         if (!intel_has_sagv(i915))
3748                 i915->display.sagv.block_time_us = 0;
3749 }
3750
3751 /*
3752  * SAGV dynamically adjusts the system agent voltage and clock frequencies
3753  * depending on power and performance requirements. The display engine access
3754  * to system memory is blocked during the adjustment time. Because of the
3755  * blocking time, having this enabled can cause full system hangs and/or pipe
3756  * underruns if we don't meet all of the following requirements:
3757  *
3758  *  - <= 1 pipe enabled
3759  *  - All planes can enable watermarks for latencies >= SAGV engine block time
3760  *  - We're not using an interlaced display configuration
3761  */
3762 static void skl_sagv_enable(struct drm_i915_private *dev_priv)
3763 {
3764         int ret;
3765
3766         if (!intel_has_sagv(dev_priv))
3767                 return;
3768
3769         if (dev_priv->display.sagv.status == I915_SAGV_ENABLED)
3770                 return;
3771
3772         drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3773         ret = snb_pcode_write(&dev_priv->uncore, GEN9_PCODE_SAGV_CONTROL,
3774                               GEN9_SAGV_ENABLE);
3775
3776         /* We don't need to wait for SAGV when enabling */
3777
3778         /*
3779          * Some skl systems, pre-release machines in particular,
3780          * don't actually have SAGV.
3781          */
3782         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3783                 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3784                 dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
3785                 return;
3786         } else if (ret < 0) {
3787                 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3788                 return;
3789         }
3790
3791         dev_priv->display.sagv.status = I915_SAGV_ENABLED;
3792 }
3793
3794 static void skl_sagv_disable(struct drm_i915_private *dev_priv)
3795 {
3796         int ret;
3797
3798         if (!intel_has_sagv(dev_priv))
3799                 return;
3800
3801         if (dev_priv->display.sagv.status == I915_SAGV_DISABLED)
3802                 return;
3803
3804         drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3805         /* bspec says to keep retrying for at least 1 ms */
3806         ret = skl_pcode_request(&dev_priv->uncore, GEN9_PCODE_SAGV_CONTROL,
3807                                 GEN9_SAGV_DISABLE,
3808                                 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3809                                 1);
3810         /*
3811          * Some skl systems, pre-release machines in particular,
3812          * don't actually have SAGV.
3813          */
3814         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3815                 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3816                 dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
3817                 return;
3818         } else if (ret < 0) {
3819                 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3820                 return;
3821         }
3822
3823         dev_priv->display.sagv.status = I915_SAGV_DISABLED;
3824 }
3825
3826 static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
3827 {
3828         struct drm_i915_private *i915 = to_i915(state->base.dev);
3829         const struct intel_bw_state *new_bw_state =
3830                 intel_atomic_get_new_bw_state(state);
3831
3832         if (!new_bw_state)
3833                 return;
3834
3835         if (!intel_can_enable_sagv(i915, new_bw_state))
3836                 skl_sagv_disable(i915);
3837 }
3838
3839 static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
3840 {
3841         struct drm_i915_private *i915 = to_i915(state->base.dev);
3842         const struct intel_bw_state *new_bw_state =
3843                 intel_atomic_get_new_bw_state(state);
3844
3845         if (!new_bw_state)
3846                 return;
3847
3848         if (intel_can_enable_sagv(i915, new_bw_state))
3849                 skl_sagv_enable(i915);
3850 }
3851
3852 static void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
3853 {
3854         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3855         const struct intel_bw_state *old_bw_state =
3856                 intel_atomic_get_old_bw_state(state);
3857         const struct intel_bw_state *new_bw_state =
3858                 intel_atomic_get_new_bw_state(state);
3859         u16 old_mask, new_mask;
3860
3861         if (!new_bw_state)
3862                 return;
3863
3864         old_mask = old_bw_state->qgv_points_mask;
3865         new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3866
3867         if (old_mask == new_mask)
3868                 return;
3869
3870         WARN_ON(!new_bw_state->base.changed);
3871
3872         drm_dbg_kms(&dev_priv->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
3873                     old_mask, new_mask);
3874
3875         /*
3876          * Restrict required qgv points before updating the configuration.
3877          * According to BSpec we can't mask and unmask qgv points at the same
3878          * time. Also masking should be done before updating the configuration
3879          * and unmasking afterwards.
3880          */
3881         icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3882 }
3883
3884 static void icl_sagv_post_plane_update(struct intel_atomic_state *state)
3885 {
3886         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3887         const struct intel_bw_state *old_bw_state =
3888                 intel_atomic_get_old_bw_state(state);
3889         const struct intel_bw_state *new_bw_state =
3890                 intel_atomic_get_new_bw_state(state);
3891         u16 old_mask, new_mask;
3892
3893         if (!new_bw_state)
3894                 return;
3895
3896         old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3897         new_mask = new_bw_state->qgv_points_mask;
3898
3899         if (old_mask == new_mask)
3900                 return;
3901
3902         WARN_ON(!new_bw_state->base.changed);
3903
3904         drm_dbg_kms(&dev_priv->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
3905                     old_mask, new_mask);
3906
3907         /*
3908          * Allow required qgv points after updating the configuration.
3909          * According to BSpec we can't mask and unmask qgv points at the same
3910          * time. Also masking should be done before updating the configuration
3911          * and unmasking afterwards.
3912          */
3913         icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3914 }
3915
3916 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3917 {
3918         struct drm_i915_private *i915 = to_i915(state->base.dev);
3919
3920         /*
3921          * Just return if we can't control SAGV or don't have it.
3922          * This is different from situation when we have SAGV but just can't
3923          * afford it due to DBuf limitation - in case if SAGV is completely
3924          * disabled in a BIOS, we are not even allowed to send a PCode request,
3925          * as it will throw an error. So have to check it here.
3926          */
3927         if (!intel_has_sagv(i915))
3928                 return;
3929
3930         if (DISPLAY_VER(i915) >= 11)
3931                 icl_sagv_pre_plane_update(state);
3932         else
3933                 skl_sagv_pre_plane_update(state);
3934 }
3935
3936 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3937 {
3938         struct drm_i915_private *i915 = to_i915(state->base.dev);
3939
3940         /*
3941          * Just return if we can't control SAGV or don't have it.
3942          * This is different from situation when we have SAGV but just can't
3943          * afford it due to DBuf limitation - in case if SAGV is completely
3944          * disabled in a BIOS, we are not even allowed to send a PCode request,
3945          * as it will throw an error. So have to check it here.
3946          */
3947         if (!intel_has_sagv(i915))
3948                 return;
3949
3950         if (DISPLAY_VER(i915) >= 11)
3951                 icl_sagv_post_plane_update(state);
3952         else
3953                 skl_sagv_post_plane_update(state);
3954 }
3955
3956 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3957 {
3958         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3959         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3960         enum plane_id plane_id;
3961         int max_level = INT_MAX;
3962
3963         if (!intel_has_sagv(dev_priv))
3964                 return false;
3965
3966         if (!crtc_state->hw.active)
3967                 return true;
3968
3969         if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
3970                 return false;
3971
3972         for_each_plane_id_on_crtc(crtc, plane_id) {
3973                 const struct skl_plane_wm *wm =
3974                         &crtc_state->wm.skl.optimal.planes[plane_id];
3975                 int level;
3976
3977                 /* Skip this plane if it's not enabled */
3978                 if (!wm->wm[0].enable)
3979                         continue;
3980
3981                 /* Find the highest enabled wm level for this plane */
3982                 for (level = ilk_wm_max_level(dev_priv);
3983                      !wm->wm[level].enable; --level)
3984                      { }
3985
3986                 /* Highest common enabled wm level for all planes */
3987                 max_level = min(level, max_level);
3988         }
3989
3990         /* No enabled planes? */
3991         if (max_level == INT_MAX)
3992                 return true;
3993
3994         for_each_plane_id_on_crtc(crtc, plane_id) {
3995                 const struct skl_plane_wm *wm =
3996                         &crtc_state->wm.skl.optimal.planes[plane_id];
3997
3998                 /*
3999                  * All enabled planes must have enabled a common wm level that
4000                  * can tolerate memory latencies higher than sagv_block_time_us
4001                  */
4002                 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
4003                         return false;
4004         }
4005
4006         return true;
4007 }
4008
4009 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
4010 {
4011         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4012         enum plane_id plane_id;
4013
4014         if (!crtc_state->hw.active)
4015                 return true;
4016
4017         for_each_plane_id_on_crtc(crtc, plane_id) {
4018                 const struct skl_plane_wm *wm =
4019                         &crtc_state->wm.skl.optimal.planes[plane_id];
4020
4021                 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
4022                         return false;
4023         }
4024
4025         return true;
4026 }
4027
4028 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
4029 {
4030         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4031         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4032
4033         if (DISPLAY_VER(dev_priv) >= 12)
4034                 return tgl_crtc_can_enable_sagv(crtc_state);
4035         else
4036                 return skl_crtc_can_enable_sagv(crtc_state);
4037 }
4038
4039 bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
4040                            const struct intel_bw_state *bw_state)
4041 {
4042         if (DISPLAY_VER(dev_priv) < 11 &&
4043             bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
4044                 return false;
4045
4046         return bw_state->pipe_sagv_reject == 0;
4047 }
4048
4049 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
4050 {
4051         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
4052         int ret;
4053         struct intel_crtc *crtc;
4054         struct intel_crtc_state *new_crtc_state;
4055         struct intel_bw_state *new_bw_state = NULL;
4056         const struct intel_bw_state *old_bw_state = NULL;
4057         int i;
4058
4059         for_each_new_intel_crtc_in_state(state, crtc,
4060                                          new_crtc_state, i) {
4061                 new_bw_state = intel_atomic_get_bw_state(state);
4062                 if (IS_ERR(new_bw_state))
4063                         return PTR_ERR(new_bw_state);
4064
4065                 old_bw_state = intel_atomic_get_old_bw_state(state);
4066
4067                 if (intel_crtc_can_enable_sagv(new_crtc_state))
4068                         new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
4069                 else
4070                         new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
4071         }
4072
4073         if (!new_bw_state)
4074                 return 0;
4075
4076         new_bw_state->active_pipes =
4077                 intel_calc_active_pipes(state, old_bw_state->active_pipes);
4078
4079         if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
4080                 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4081                 if (ret)
4082                         return ret;
4083         }
4084
4085         if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4086             intel_can_enable_sagv(dev_priv, old_bw_state)) {
4087                 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4088                 if (ret)
4089                         return ret;
4090         } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4091                 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4092                 if (ret)
4093                         return ret;
4094         }
4095
4096         for_each_new_intel_crtc_in_state(state, crtc,
4097                                          new_crtc_state, i) {
4098                 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
4099
4100                 /*
4101                  * We store use_sagv_wm in the crtc state rather than relying on
4102                  * that bw state since we have no convenient way to get at the
4103                  * latter from the plane commit hooks (especially in the legacy
4104                  * cursor case)
4105                  */
4106                 pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(dev_priv) &&
4107                         DISPLAY_VER(dev_priv) >= 12 &&
4108                         intel_can_enable_sagv(dev_priv, new_bw_state);
4109         }
4110
4111         return 0;
4112 }
4113
4114 static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
4115                               u16 start, u16 end)
4116 {
4117         entry->start = start;
4118         entry->end = end;
4119
4120         return end;
4121 }
4122
4123 static int intel_dbuf_slice_size(struct drm_i915_private *dev_priv)
4124 {
4125         return INTEL_INFO(dev_priv)->display.dbuf.size /
4126                 hweight8(INTEL_INFO(dev_priv)->display.dbuf.slice_mask);
4127 }
4128
4129 static void
4130 skl_ddb_entry_for_slices(struct drm_i915_private *dev_priv, u8 slice_mask,
4131                          struct skl_ddb_entry *ddb)
4132 {
4133         int slice_size = intel_dbuf_slice_size(dev_priv);
4134
4135         if (!slice_mask) {
4136                 ddb->start = 0;
4137                 ddb->end = 0;
4138                 return;
4139         }
4140
4141         ddb->start = (ffs(slice_mask) - 1) * slice_size;
4142         ddb->end = fls(slice_mask) * slice_size;
4143
4144         WARN_ON(ddb->start >= ddb->end);
4145         WARN_ON(ddb->end > INTEL_INFO(dev_priv)->display.dbuf.size);
4146 }
4147
4148 static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
4149 {
4150         struct skl_ddb_entry ddb;
4151
4152         if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
4153                 slice_mask = BIT(DBUF_S1);
4154         else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
4155                 slice_mask = BIT(DBUF_S3);
4156
4157         skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
4158
4159         return ddb.start;
4160 }
4161
4162 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4163                             const struct skl_ddb_entry *entry)
4164 {
4165         int slice_size = intel_dbuf_slice_size(dev_priv);
4166         enum dbuf_slice start_slice, end_slice;
4167         u8 slice_mask = 0;
4168
4169         if (!skl_ddb_entry_size(entry))
4170                 return 0;
4171
4172         start_slice = entry->start / slice_size;
4173         end_slice = (entry->end - 1) / slice_size;
4174
4175         /*
4176          * Per plane DDB entry can in a really worst case be on multiple slices
4177          * but single entry is anyway contigious.
4178          */
4179         while (start_slice <= end_slice) {
4180                 slice_mask |= BIT(start_slice);
4181                 start_slice++;
4182         }
4183
4184         return slice_mask;
4185 }
4186
4187 static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
4188 {
4189         const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
4190         int hdisplay, vdisplay;
4191
4192         if (!crtc_state->hw.active)
4193                 return 0;
4194
4195         /*
4196          * Watermark/ddb requirement highly depends upon width of the
4197          * framebuffer, So instead of allocating DDB equally among pipes
4198          * distribute DDB based on resolution/width of the display.
4199          */
4200         drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
4201
4202         return hdisplay;
4203 }
4204
4205 static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
4206                                     enum pipe for_pipe,
4207                                     unsigned int *weight_start,
4208                                     unsigned int *weight_end,
4209                                     unsigned int *weight_total)
4210 {
4211         struct drm_i915_private *dev_priv =
4212                 to_i915(dbuf_state->base.state->base.dev);
4213         enum pipe pipe;
4214
4215         *weight_start = 0;
4216         *weight_end = 0;
4217         *weight_total = 0;
4218
4219         for_each_pipe(dev_priv, pipe) {
4220                 int weight = dbuf_state->weight[pipe];
4221
4222                 /*
4223                  * Do not account pipes using other slice sets
4224                  * luckily as of current BSpec slice sets do not partially
4225                  * intersect(pipes share either same one slice or same slice set
4226                  * i.e no partial intersection), so it is enough to check for
4227                  * equality for now.
4228                  */
4229                 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
4230                         continue;
4231
4232                 *weight_total += weight;
4233                 if (pipe < for_pipe) {
4234                         *weight_start += weight;
4235                         *weight_end += weight;
4236                 } else if (pipe == for_pipe) {
4237                         *weight_end += weight;
4238                 }
4239         }
4240 }
4241
4242 static int
4243 skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
4244 {
4245         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4246         unsigned int weight_total, weight_start, weight_end;
4247         const struct intel_dbuf_state *old_dbuf_state =
4248                 intel_atomic_get_old_dbuf_state(state);
4249         struct intel_dbuf_state *new_dbuf_state =
4250                 intel_atomic_get_new_dbuf_state(state);
4251         struct intel_crtc_state *crtc_state;
4252         struct skl_ddb_entry ddb_slices;
4253         enum pipe pipe = crtc->pipe;
4254         unsigned int mbus_offset = 0;
4255         u32 ddb_range_size;
4256         u32 dbuf_slice_mask;
4257         u32 start, end;
4258         int ret;
4259
4260         if (new_dbuf_state->weight[pipe] == 0) {
4261                 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0);
4262                 goto out;
4263         }
4264
4265         dbuf_slice_mask = new_dbuf_state->slices[pipe];
4266
4267         skl_ddb_entry_for_slices(dev_priv, dbuf_slice_mask, &ddb_slices);
4268         mbus_offset = mbus_ddb_offset(dev_priv, dbuf_slice_mask);
4269         ddb_range_size = skl_ddb_entry_size(&ddb_slices);
4270
4271         intel_crtc_dbuf_weights(new_dbuf_state, pipe,
4272                                 &weight_start, &weight_end, &weight_total);
4273
4274         start = ddb_range_size * weight_start / weight_total;
4275         end = ddb_range_size * weight_end / weight_total;
4276
4277         skl_ddb_entry_init(&new_dbuf_state->ddb[pipe],
4278                            ddb_slices.start - mbus_offset + start,
4279                            ddb_slices.start - mbus_offset + end);
4280
4281 out:
4282         if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
4283             skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
4284                                 &new_dbuf_state->ddb[pipe]))
4285                 return 0;
4286
4287         ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
4288         if (ret)
4289                 return ret;
4290
4291         crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
4292         if (IS_ERR(crtc_state))
4293                 return PTR_ERR(crtc_state);
4294
4295         /*
4296          * Used for checking overlaps, so we need absolute
4297          * offsets instead of MBUS relative offsets.
4298          */
4299         crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
4300         crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
4301
4302         drm_dbg_kms(&dev_priv->drm,
4303                     "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
4304                     crtc->base.base.id, crtc->base.name,
4305                     old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
4306                     old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
4307                     new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
4308                     old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
4309
4310         return 0;
4311 }
4312
4313 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4314                                  int width, const struct drm_format_info *format,
4315                                  u64 modifier, unsigned int rotation,
4316                                  u32 plane_pixel_rate, struct skl_wm_params *wp,
4317                                  int color_plane);
4318
4319 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4320                                  struct intel_plane *plane,
4321                                  int level,
4322                                  unsigned int latency,
4323                                  const struct skl_wm_params *wp,
4324                                  const struct skl_wm_level *result_prev,
4325                                  struct skl_wm_level *result /* out */);
4326
4327 static unsigned int
4328 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4329                       int num_active)
4330 {
4331         struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
4332         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4333         int level, max_level = ilk_wm_max_level(dev_priv);
4334         struct skl_wm_level wm = {};
4335         int ret, min_ddb_alloc = 0;
4336         struct skl_wm_params wp;
4337
4338         ret = skl_compute_wm_params(crtc_state, 256,
4339                                     drm_format_info(DRM_FORMAT_ARGB8888),
4340                                     DRM_FORMAT_MOD_LINEAR,
4341                                     DRM_MODE_ROTATE_0,
4342                                     crtc_state->pixel_rate, &wp, 0);
4343         drm_WARN_ON(&dev_priv->drm, ret);
4344
4345         for (level = 0; level <= max_level; level++) {
4346                 unsigned int latency = dev_priv->display.wm.skl_latency[level];
4347
4348                 skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm);
4349                 if (wm.min_ddb_alloc == U16_MAX)
4350                         break;
4351
4352                 min_ddb_alloc = wm.min_ddb_alloc;
4353         }
4354
4355         return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4356 }
4357
4358 static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
4359 {
4360         skl_ddb_entry_init(entry,
4361                            REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
4362                            REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
4363         if (entry->end)
4364                 entry->end++;
4365 }
4366
4367 static void
4368 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4369                            const enum pipe pipe,
4370                            const enum plane_id plane_id,
4371                            struct skl_ddb_entry *ddb,
4372                            struct skl_ddb_entry *ddb_y)
4373 {
4374         u32 val;
4375
4376         /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4377         if (plane_id == PLANE_CURSOR) {
4378                 val = intel_uncore_read(&dev_priv->uncore, CUR_BUF_CFG(pipe));
4379                 skl_ddb_entry_init_from_hw(ddb, val);
4380                 return;
4381         }
4382
4383         val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4384         skl_ddb_entry_init_from_hw(ddb, val);
4385
4386         if (DISPLAY_VER(dev_priv) >= 11)
4387                 return;
4388
4389         val = intel_uncore_read(&dev_priv->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id));
4390         skl_ddb_entry_init_from_hw(ddb_y, val);
4391 }
4392
4393 static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4394                                       struct skl_ddb_entry *ddb,
4395                                       struct skl_ddb_entry *ddb_y)
4396 {
4397         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4398         enum intel_display_power_domain power_domain;
4399         enum pipe pipe = crtc->pipe;
4400         intel_wakeref_t wakeref;
4401         enum plane_id plane_id;
4402
4403         power_domain = POWER_DOMAIN_PIPE(pipe);
4404         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4405         if (!wakeref)
4406                 return;
4407
4408         for_each_plane_id_on_crtc(crtc, plane_id)
4409                 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4410                                            plane_id,
4411                                            &ddb[plane_id],
4412                                            &ddb_y[plane_id]);
4413
4414         intel_display_power_put(dev_priv, power_domain, wakeref);
4415 }
4416
4417 struct dbuf_slice_conf_entry {
4418         u8 active_pipes;
4419         u8 dbuf_mask[I915_MAX_PIPES];
4420         bool join_mbus;
4421 };
4422
4423 /*
4424  * Table taken from Bspec 12716
4425  * Pipes do have some preferred DBuf slice affinity,
4426  * plus there are some hardcoded requirements on how
4427  * those should be distributed for multipipe scenarios.
4428  * For more DBuf slices algorithm can get even more messy
4429  * and less readable, so decided to use a table almost
4430  * as is from BSpec itself - that way it is at least easier
4431  * to compare, change and check.
4432  */
4433 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4434 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4435 {
4436         {
4437                 .active_pipes = BIT(PIPE_A),
4438                 .dbuf_mask = {
4439                         [PIPE_A] = BIT(DBUF_S1),
4440                 },
4441         },
4442         {
4443                 .active_pipes = BIT(PIPE_B),
4444                 .dbuf_mask = {
4445                         [PIPE_B] = BIT(DBUF_S1),
4446                 },
4447         },
4448         {
4449                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4450                 .dbuf_mask = {
4451                         [PIPE_A] = BIT(DBUF_S1),
4452                         [PIPE_B] = BIT(DBUF_S2),
4453                 },
4454         },
4455         {
4456                 .active_pipes = BIT(PIPE_C),
4457                 .dbuf_mask = {
4458                         [PIPE_C] = BIT(DBUF_S2),
4459                 },
4460         },
4461         {
4462                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4463                 .dbuf_mask = {
4464                         [PIPE_A] = BIT(DBUF_S1),
4465                         [PIPE_C] = BIT(DBUF_S2),
4466                 },
4467         },
4468         {
4469                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4470                 .dbuf_mask = {
4471                         [PIPE_B] = BIT(DBUF_S1),
4472                         [PIPE_C] = BIT(DBUF_S2),
4473                 },
4474         },
4475         {
4476                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4477                 .dbuf_mask = {
4478                         [PIPE_A] = BIT(DBUF_S1),
4479                         [PIPE_B] = BIT(DBUF_S1),
4480                         [PIPE_C] = BIT(DBUF_S2),
4481                 },
4482         },
4483         {}
4484 };
4485
4486 /*
4487  * Table taken from Bspec 49255
4488  * Pipes do have some preferred DBuf slice affinity,
4489  * plus there are some hardcoded requirements on how
4490  * those should be distributed for multipipe scenarios.
4491  * For more DBuf slices algorithm can get even more messy
4492  * and less readable, so decided to use a table almost
4493  * as is from BSpec itself - that way it is at least easier
4494  * to compare, change and check.
4495  */
4496 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4497 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4498 {
4499         {
4500                 .active_pipes = BIT(PIPE_A),
4501                 .dbuf_mask = {
4502                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4503                 },
4504         },
4505         {
4506                 .active_pipes = BIT(PIPE_B),
4507                 .dbuf_mask = {
4508                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4509                 },
4510         },
4511         {
4512                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4513                 .dbuf_mask = {
4514                         [PIPE_A] = BIT(DBUF_S2),
4515                         [PIPE_B] = BIT(DBUF_S1),
4516                 },
4517         },
4518         {
4519                 .active_pipes = BIT(PIPE_C),
4520                 .dbuf_mask = {
4521                         [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4522                 },
4523         },
4524         {
4525                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4526                 .dbuf_mask = {
4527                         [PIPE_A] = BIT(DBUF_S1),
4528                         [PIPE_C] = BIT(DBUF_S2),
4529                 },
4530         },
4531         {
4532                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4533                 .dbuf_mask = {
4534                         [PIPE_B] = BIT(DBUF_S1),
4535                         [PIPE_C] = BIT(DBUF_S2),
4536                 },
4537         },
4538         {
4539                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4540                 .dbuf_mask = {
4541                         [PIPE_A] = BIT(DBUF_S1),
4542                         [PIPE_B] = BIT(DBUF_S1),
4543                         [PIPE_C] = BIT(DBUF_S2),
4544                 },
4545         },
4546         {
4547                 .active_pipes = BIT(PIPE_D),
4548                 .dbuf_mask = {
4549                         [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4550                 },
4551         },
4552         {
4553                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4554                 .dbuf_mask = {
4555                         [PIPE_A] = BIT(DBUF_S1),
4556                         [PIPE_D] = BIT(DBUF_S2),
4557                 },
4558         },
4559         {
4560                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4561                 .dbuf_mask = {
4562                         [PIPE_B] = BIT(DBUF_S1),
4563                         [PIPE_D] = BIT(DBUF_S2),
4564                 },
4565         },
4566         {
4567                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4568                 .dbuf_mask = {
4569                         [PIPE_A] = BIT(DBUF_S1),
4570                         [PIPE_B] = BIT(DBUF_S1),
4571                         [PIPE_D] = BIT(DBUF_S2),
4572                 },
4573         },
4574         {
4575                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4576                 .dbuf_mask = {
4577                         [PIPE_C] = BIT(DBUF_S1),
4578                         [PIPE_D] = BIT(DBUF_S2),
4579                 },
4580         },
4581         {
4582                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4583                 .dbuf_mask = {
4584                         [PIPE_A] = BIT(DBUF_S1),
4585                         [PIPE_C] = BIT(DBUF_S2),
4586                         [PIPE_D] = BIT(DBUF_S2),
4587                 },
4588         },
4589         {
4590                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4591                 .dbuf_mask = {
4592                         [PIPE_B] = BIT(DBUF_S1),
4593                         [PIPE_C] = BIT(DBUF_S2),
4594                         [PIPE_D] = BIT(DBUF_S2),
4595                 },
4596         },
4597         {
4598                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4599                 .dbuf_mask = {
4600                         [PIPE_A] = BIT(DBUF_S1),
4601                         [PIPE_B] = BIT(DBUF_S1),
4602                         [PIPE_C] = BIT(DBUF_S2),
4603                         [PIPE_D] = BIT(DBUF_S2),
4604                 },
4605         },
4606         {}
4607 };
4608
4609 static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
4610         {
4611                 .active_pipes = BIT(PIPE_A),
4612                 .dbuf_mask = {
4613                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4614                 },
4615         },
4616         {
4617                 .active_pipes = BIT(PIPE_B),
4618                 .dbuf_mask = {
4619                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4620                 },
4621         },
4622         {
4623                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4624                 .dbuf_mask = {
4625                         [PIPE_A] = BIT(DBUF_S1),
4626                         [PIPE_B] = BIT(DBUF_S2),
4627                 },
4628         },
4629         {
4630                 .active_pipes = BIT(PIPE_C),
4631                 .dbuf_mask = {
4632                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4633                 },
4634         },
4635         {
4636                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4637                 .dbuf_mask = {
4638                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4639                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4640                 },
4641         },
4642         {
4643                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4644                 .dbuf_mask = {
4645                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4646                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4647                 },
4648         },
4649         {
4650                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4651                 .dbuf_mask = {
4652                         [PIPE_A] = BIT(DBUF_S1),
4653                         [PIPE_B] = BIT(DBUF_S2),
4654                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4655                 },
4656         },
4657         {
4658                 .active_pipes = BIT(PIPE_D),
4659                 .dbuf_mask = {
4660                         [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4661                 },
4662         },
4663         {
4664                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4665                 .dbuf_mask = {
4666                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4667                         [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4668                 },
4669         },
4670         {
4671                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4672                 .dbuf_mask = {
4673                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4674                         [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4675                 },
4676         },
4677         {
4678                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4679                 .dbuf_mask = {
4680                         [PIPE_A] = BIT(DBUF_S1),
4681                         [PIPE_B] = BIT(DBUF_S2),
4682                         [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4683                 },
4684         },
4685         {
4686                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4687                 .dbuf_mask = {
4688                         [PIPE_C] = BIT(DBUF_S3),
4689                         [PIPE_D] = BIT(DBUF_S4),
4690                 },
4691         },
4692         {
4693                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4694                 .dbuf_mask = {
4695                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4696                         [PIPE_C] = BIT(DBUF_S3),
4697                         [PIPE_D] = BIT(DBUF_S4),
4698                 },
4699         },
4700         {
4701                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4702                 .dbuf_mask = {
4703                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4704                         [PIPE_C] = BIT(DBUF_S3),
4705                         [PIPE_D] = BIT(DBUF_S4),
4706                 },
4707         },
4708         {
4709                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4710                 .dbuf_mask = {
4711                         [PIPE_A] = BIT(DBUF_S1),
4712                         [PIPE_B] = BIT(DBUF_S2),
4713                         [PIPE_C] = BIT(DBUF_S3),
4714                         [PIPE_D] = BIT(DBUF_S4),
4715                 },
4716         },
4717         {}
4718 };
4719
4720 static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
4721         /*
4722          * Keep the join_mbus cases first so check_mbus_joined()
4723          * will prefer them over the !join_mbus cases.
4724          */
4725         {
4726                 .active_pipes = BIT(PIPE_A),
4727                 .dbuf_mask = {
4728                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4729                 },
4730                 .join_mbus = true,
4731         },
4732         {
4733                 .active_pipes = BIT(PIPE_B),
4734                 .dbuf_mask = {
4735                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4736                 },
4737                 .join_mbus = true,
4738         },
4739         {
4740                 .active_pipes = BIT(PIPE_A),
4741                 .dbuf_mask = {
4742                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4743                 },
4744                 .join_mbus = false,
4745         },
4746         {
4747                 .active_pipes = BIT(PIPE_B),
4748                 .dbuf_mask = {
4749                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4750                 },
4751                 .join_mbus = false,
4752         },
4753         {
4754                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4755                 .dbuf_mask = {
4756                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4757                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4758                 },
4759         },
4760         {
4761                 .active_pipes = BIT(PIPE_C),
4762                 .dbuf_mask = {
4763                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4764                 },
4765         },
4766         {
4767                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4768                 .dbuf_mask = {
4769                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4770                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4771                 },
4772         },
4773         {
4774                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4775                 .dbuf_mask = {
4776                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4777                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4778                 },
4779         },
4780         {
4781                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4782                 .dbuf_mask = {
4783                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4784                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4785                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4786                 },
4787         },
4788         {
4789                 .active_pipes = BIT(PIPE_D),
4790                 .dbuf_mask = {
4791                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4792                 },
4793         },
4794         {
4795                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4796                 .dbuf_mask = {
4797                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4798                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4799                 },
4800         },
4801         {
4802                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4803                 .dbuf_mask = {
4804                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4805                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4806                 },
4807         },
4808         {
4809                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4810                 .dbuf_mask = {
4811                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4812                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4813                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4814                 },
4815         },
4816         {
4817                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4818                 .dbuf_mask = {
4819                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4820                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4821                 },
4822         },
4823         {
4824                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4825                 .dbuf_mask = {
4826                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4827                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4828                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4829                 },
4830         },
4831         {
4832                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4833                 .dbuf_mask = {
4834                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4835                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4836                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4837                 },
4838         },
4839         {
4840                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4841                 .dbuf_mask = {
4842                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4843                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4844                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4845                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4846                 },
4847         },
4848         {}
4849
4850 };
4851
4852 static bool check_mbus_joined(u8 active_pipes,
4853                               const struct dbuf_slice_conf_entry *dbuf_slices)
4854 {
4855         int i;
4856
4857         for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
4858                 if (dbuf_slices[i].active_pipes == active_pipes)
4859                         return dbuf_slices[i].join_mbus;
4860         }
4861         return false;
4862 }
4863
4864 static bool adlp_check_mbus_joined(u8 active_pipes)
4865 {
4866         return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
4867 }
4868
4869 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
4870                               const struct dbuf_slice_conf_entry *dbuf_slices)
4871 {
4872         int i;
4873
4874         for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
4875                 if (dbuf_slices[i].active_pipes == active_pipes &&
4876                     dbuf_slices[i].join_mbus == join_mbus)
4877                         return dbuf_slices[i].dbuf_mask[pipe];
4878         }
4879         return 0;
4880 }
4881
4882 /*
4883  * This function finds an entry with same enabled pipe configuration and
4884  * returns correspondent DBuf slice mask as stated in BSpec for particular
4885  * platform.
4886  */
4887 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
4888 {
4889         /*
4890          * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4891          * required calculating "pipe ratio" in order to determine
4892          * if one or two slices can be used for single pipe configurations
4893          * as additional constraint to the existing table.
4894          * However based on recent info, it should be not "pipe ratio"
4895          * but rather ratio between pixel_rate and cdclk with additional
4896          * constants, so for now we are using only table until this is
4897          * clarified. Also this is the reason why crtc_state param is
4898          * still here - we will need it once those additional constraints
4899          * pop up.
4900          */
4901         return compute_dbuf_slices(pipe, active_pipes, join_mbus,
4902                                    icl_allowed_dbufs);
4903 }
4904
4905 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
4906 {
4907         return compute_dbuf_slices(pipe, active_pipes, join_mbus,
4908                                    tgl_allowed_dbufs);
4909 }
4910
4911 static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
4912 {
4913         return compute_dbuf_slices(pipe, active_pipes, join_mbus,
4914                                    adlp_allowed_dbufs);
4915 }
4916
4917 static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
4918 {
4919         return compute_dbuf_slices(pipe, active_pipes, join_mbus,
4920                                    dg2_allowed_dbufs);
4921 }
4922
4923 static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
4924 {
4925         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4926         enum pipe pipe = crtc->pipe;
4927
4928         if (IS_DG2(dev_priv))
4929                 return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
4930         else if (DISPLAY_VER(dev_priv) >= 13)
4931                 return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
4932         else if (DISPLAY_VER(dev_priv) == 12)
4933                 return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
4934         else if (DISPLAY_VER(dev_priv) == 11)
4935                 return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
4936         /*
4937          * For anything else just return one slice yet.
4938          * Should be extended for other platforms.
4939          */
4940         return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4941 }
4942
4943 static bool
4944 use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
4945                      struct intel_plane *plane)
4946 {
4947         struct drm_i915_private *i915 = to_i915(plane->base.dev);
4948
4949         return DISPLAY_VER(i915) >= 13 &&
4950                crtc_state->uapi.async_flip &&
4951                plane->async_flip;
4952 }
4953
4954 static u64
4955 skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state)
4956 {
4957         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4958         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
4959         enum plane_id plane_id;
4960         u64 data_rate = 0;
4961
4962         for_each_plane_id_on_crtc(crtc, plane_id) {
4963                 if (plane_id == PLANE_CURSOR)
4964                         continue;
4965
4966                 data_rate += crtc_state->rel_data_rate[plane_id];
4967
4968                 if (DISPLAY_VER(i915) < 11)
4969                         data_rate += crtc_state->rel_data_rate_y[plane_id];
4970         }
4971
4972         return data_rate;
4973 }
4974
4975 static const struct skl_wm_level *
4976 skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
4977                    enum plane_id plane_id,
4978                    int level)
4979 {
4980         const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4981
4982         if (level == 0 && pipe_wm->use_sagv_wm)
4983                 return &wm->sagv.wm0;
4984
4985         return &wm->wm[level];
4986 }
4987
4988 static const struct skl_wm_level *
4989 skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
4990                    enum plane_id plane_id)
4991 {
4992         const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4993
4994         if (pipe_wm->use_sagv_wm)
4995                 return &wm->sagv.trans_wm;
4996
4997         return &wm->trans_wm;
4998 }
4999
5000 /*
5001  * We only disable the watermarks for each plane if
5002  * they exceed the ddb allocation of said plane. This
5003  * is done so that we don't end up touching cursor
5004  * watermarks needlessly when some other plane reduces
5005  * our max possible watermark level.
5006  *
5007  * Bspec has this to say about the PLANE_WM enable bit:
5008  * "All the watermarks at this level for all enabled
5009  *  planes must be enabled before the level will be used."
5010  * So this is actually safe to do.
5011  */
5012 static void
5013 skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb)
5014 {
5015         if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb))
5016                 memset(wm, 0, sizeof(*wm));
5017 }
5018
5019 static void
5020 skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
5021                         const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb)
5022 {
5023         if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb_y) ||
5024             uv_wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) {
5025                 memset(wm, 0, sizeof(*wm));
5026                 memset(uv_wm, 0, sizeof(*uv_wm));
5027         }
5028 }
5029
5030 static bool icl_need_wm1_wa(struct drm_i915_private *i915,
5031                             enum plane_id plane_id)
5032 {
5033         /*
5034          * Wa_1408961008:icl, ehl
5035          * Wa_14012656716:tgl, adl
5036          * Underruns with WM1+ disabled
5037          */
5038         return DISPLAY_VER(i915) == 11 ||
5039                (IS_DISPLAY_VER(i915, 12, 13) && plane_id == PLANE_CURSOR);
5040 }
5041
5042 struct skl_plane_ddb_iter {
5043         u64 data_rate;
5044         u16 start, size;
5045 };
5046
5047 static void
5048 skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
5049                        struct skl_ddb_entry *ddb,
5050                        const struct skl_wm_level *wm,
5051                        u64 data_rate)
5052 {
5053         u16 size, extra = 0;
5054
5055         if (data_rate) {
5056                 extra = min_t(u16, iter->size,
5057                               DIV64_U64_ROUND_UP(iter->size * data_rate,
5058                                                  iter->data_rate));
5059                 iter->size -= extra;
5060                 iter->data_rate -= data_rate;
5061         }
5062
5063         /*
5064          * Keep ddb entry of all disabled planes explicitly zeroed
5065          * to avoid skl_ddb_add_affected_planes() adding them to
5066          * the state when other planes change their allocations.
5067          */
5068         size = wm->min_ddb_alloc + extra;
5069         if (size)
5070                 iter->start = skl_ddb_entry_init(ddb, iter->start,
5071                                                  iter->start + size);
5072 }
5073
5074 static int
5075 skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
5076                             struct intel_crtc *crtc)
5077 {
5078         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5079         struct intel_crtc_state *crtc_state =
5080                 intel_atomic_get_new_crtc_state(state, crtc);
5081         const struct intel_dbuf_state *dbuf_state =
5082                 intel_atomic_get_new_dbuf_state(state);
5083         const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
5084         int num_active = hweight8(dbuf_state->active_pipes);
5085         struct skl_plane_ddb_iter iter;
5086         enum plane_id plane_id;
5087         u16 cursor_size;
5088         u32 blocks;
5089         int level;
5090
5091         /* Clear the partitioning for disabled planes. */
5092         memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb));
5093         memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
5094
5095         if (!crtc_state->hw.active)
5096                 return 0;
5097
5098         iter.start = alloc->start;
5099         iter.size = skl_ddb_entry_size(alloc);
5100         if (iter.size == 0)
5101                 return 0;
5102
5103         /* Allocate fixed number of blocks for cursor. */
5104         cursor_size = skl_cursor_allocation(crtc_state, num_active);
5105         iter.size -= cursor_size;
5106         skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[PLANE_CURSOR],
5107                            alloc->end - cursor_size, alloc->end);
5108
5109         iter.data_rate = skl_total_relative_data_rate(crtc_state);
5110
5111         /*
5112          * Find the highest watermark level for which we can satisfy the block
5113          * requirement of active planes.
5114          */
5115         for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
5116                 blocks = 0;
5117                 for_each_plane_id_on_crtc(crtc, plane_id) {
5118                         const struct skl_plane_wm *wm =
5119                                 &crtc_state->wm.skl.optimal.planes[plane_id];
5120
5121                         if (plane_id == PLANE_CURSOR) {
5122                                 const struct skl_ddb_entry *ddb =
5123                                         &crtc_state->wm.skl.plane_ddb[plane_id];
5124
5125                                 if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(ddb)) {
5126                                         drm_WARN_ON(&dev_priv->drm,
5127                                                     wm->wm[level].min_ddb_alloc != U16_MAX);
5128                                         blocks = U32_MAX;
5129                                         break;
5130                                 }
5131                                 continue;
5132                         }
5133
5134                         blocks += wm->wm[level].min_ddb_alloc;
5135                         blocks += wm->uv_wm[level].min_ddb_alloc;
5136                 }
5137
5138                 if (blocks <= iter.size) {
5139                         iter.size -= blocks;
5140                         break;
5141                 }
5142         }
5143
5144         if (level < 0) {
5145                 drm_dbg_kms(&dev_priv->drm,
5146                             "Requested display configuration exceeds system DDB limitations");
5147                 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
5148                             blocks, iter.size);
5149                 return -EINVAL;
5150         }
5151
5152         /* avoid the WARN later when we don't allocate any extra DDB */
5153         if (iter.data_rate == 0)
5154                 iter.size = 0;
5155
5156         /*
5157          * Grant each plane the blocks it requires at the highest achievable
5158          * watermark level, plus an extra share of the leftover blocks
5159          * proportional to its relative data rate.
5160          */
5161         for_each_plane_id_on_crtc(crtc, plane_id) {
5162                 struct skl_ddb_entry *ddb =
5163                         &crtc_state->wm.skl.plane_ddb[plane_id];
5164                 struct skl_ddb_entry *ddb_y =
5165                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
5166                 const struct skl_plane_wm *wm =
5167                         &crtc_state->wm.skl.optimal.planes[plane_id];
5168
5169                 if (plane_id == PLANE_CURSOR)
5170                         continue;
5171
5172                 if (DISPLAY_VER(dev_priv) < 11 &&
5173                     crtc_state->nv12_planes & BIT(plane_id)) {
5174                         skl_allocate_plane_ddb(&iter, ddb_y, &wm->wm[level],
5175                                                crtc_state->rel_data_rate_y[plane_id]);
5176                         skl_allocate_plane_ddb(&iter, ddb, &wm->uv_wm[level],
5177                                                crtc_state->rel_data_rate[plane_id]);
5178                 } else {
5179                         skl_allocate_plane_ddb(&iter, ddb, &wm->wm[level],
5180                                                crtc_state->rel_data_rate[plane_id]);
5181                 }
5182         }
5183         drm_WARN_ON(&dev_priv->drm, iter.size != 0 || iter.data_rate != 0);
5184
5185         /*
5186          * When we calculated watermark values we didn't know how high
5187          * of a level we'd actually be able to hit, so we just marked
5188          * all levels as "enabled."  Go back now and disable the ones
5189          * that aren't actually possible.
5190          */
5191         for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
5192                 for_each_plane_id_on_crtc(crtc, plane_id) {
5193                         const struct skl_ddb_entry *ddb =
5194                                 &crtc_state->wm.skl.plane_ddb[plane_id];
5195                         const struct skl_ddb_entry *ddb_y =
5196                                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5197                         struct skl_plane_wm *wm =
5198                                 &crtc_state->wm.skl.optimal.planes[plane_id];
5199
5200                         if (DISPLAY_VER(dev_priv) < 11 &&
5201                             crtc_state->nv12_planes & BIT(plane_id))
5202                                 skl_check_nv12_wm_level(&wm->wm[level],
5203                                                         &wm->uv_wm[level],
5204                                                         ddb_y, ddb);
5205                         else
5206                                 skl_check_wm_level(&wm->wm[level], ddb);
5207
5208                         if (icl_need_wm1_wa(dev_priv, plane_id) &&
5209                             level == 1 && wm->wm[0].enable) {
5210                                 wm->wm[level].blocks = wm->wm[0].blocks;
5211                                 wm->wm[level].lines = wm->wm[0].lines;
5212                                 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
5213                         }
5214                 }
5215         }
5216
5217         /*
5218          * Go back and disable the transition and SAGV watermarks
5219          * if it turns out we don't have enough DDB blocks for them.
5220          */
5221         for_each_plane_id_on_crtc(crtc, plane_id) {
5222                 const struct skl_ddb_entry *ddb =
5223                         &crtc_state->wm.skl.plane_ddb[plane_id];
5224                 const struct skl_ddb_entry *ddb_y =
5225                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
5226                 struct skl_plane_wm *wm =
5227                         &crtc_state->wm.skl.optimal.planes[plane_id];
5228
5229                 if (DISPLAY_VER(dev_priv) < 11 &&
5230                     crtc_state->nv12_planes & BIT(plane_id)) {
5231                         skl_check_wm_level(&wm->trans_wm, ddb_y);
5232                 } else {
5233                         WARN_ON(skl_ddb_entry_size(ddb_y));
5234
5235                         skl_check_wm_level(&wm->trans_wm, ddb);
5236                 }
5237
5238                 skl_check_wm_level(&wm->sagv.wm0, ddb);
5239                 skl_check_wm_level(&wm->sagv.trans_wm, ddb);
5240         }
5241
5242         return 0;
5243 }
5244
5245 /*
5246  * The max latency should be 257 (max the punit can code is 255 and we add 2us
5247  * for the read latency) and cpp should always be <= 8, so that
5248  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
5249  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
5250 */
5251 static uint_fixed_16_16_t
5252 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
5253                u8 cpp, u32 latency, u32 dbuf_block_size)
5254 {
5255         u32 wm_intermediate_val;
5256         uint_fixed_16_16_t ret;
5257
5258         if (latency == 0)
5259                 return FP_16_16_MAX;
5260
5261         wm_intermediate_val = latency * pixel_rate * cpp;
5262         ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5263
5264         if (DISPLAY_VER(dev_priv) >= 10)
5265                 ret = add_fixed16_u32(ret, 1);
5266
5267         return ret;
5268 }
5269
5270 static uint_fixed_16_16_t
5271 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5272                uint_fixed_16_16_t plane_blocks_per_line)
5273 {
5274         u32 wm_intermediate_val;
5275         uint_fixed_16_16_t ret;
5276
5277         if (latency == 0)
5278                 return FP_16_16_MAX;
5279
5280         wm_intermediate_val = latency * pixel_rate;
5281         wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5282                                            pipe_htotal * 1000);
5283         ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5284         return ret;
5285 }
5286
5287 static uint_fixed_16_16_t
5288 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5289 {
5290         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5291         u32 pixel_rate;
5292         u32 crtc_htotal;
5293         uint_fixed_16_16_t linetime_us;
5294
5295         if (!crtc_state->hw.active)
5296                 return u32_to_fixed16(0);
5297
5298         pixel_rate = crtc_state->pixel_rate;
5299
5300         if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5301                 return u32_to_fixed16(0);
5302
5303         crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
5304         linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5305
5306         return linetime_us;
5307 }
5308
5309 static int
5310 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5311                       int width, const struct drm_format_info *format,
5312                       u64 modifier, unsigned int rotation,
5313                       u32 plane_pixel_rate, struct skl_wm_params *wp,
5314                       int color_plane)
5315 {
5316         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5317         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5318         u32 interm_pbpl;
5319
5320         /* only planar format has two planes */
5321         if (color_plane == 1 &&
5322             !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5323                 drm_dbg_kms(&dev_priv->drm,
5324                             "Non planar format have single plane\n");
5325                 return -EINVAL;
5326         }
5327
5328         wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5329                       modifier == I915_FORMAT_MOD_4_TILED ||
5330                       modifier == I915_FORMAT_MOD_Yf_TILED ||
5331                       modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5332                       modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5333         wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5334         wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5335                          modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5336         wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5337
5338         wp->width = width;
5339         if (color_plane == 1 && wp->is_planar)
5340                 wp->width /= 2;
5341
5342         wp->cpp = format->cpp[color_plane];
5343         wp->plane_pixel_rate = plane_pixel_rate;
5344
5345         if (DISPLAY_VER(dev_priv) >= 11 &&
5346             modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
5347                 wp->dbuf_block_size = 256;
5348         else
5349                 wp->dbuf_block_size = 512;
5350
5351         if (drm_rotation_90_or_270(rotation)) {
5352                 switch (wp->cpp) {
5353                 case 1:
5354                         wp->y_min_scanlines = 16;
5355                         break;
5356                 case 2:
5357                         wp->y_min_scanlines = 8;
5358                         break;
5359                 case 4:
5360                         wp->y_min_scanlines = 4;
5361                         break;
5362                 default:
5363                         MISSING_CASE(wp->cpp);
5364                         return -EINVAL;
5365                 }
5366         } else {
5367                 wp->y_min_scanlines = 4;
5368         }
5369
5370         if (skl_needs_memory_bw_wa(dev_priv))
5371                 wp->y_min_scanlines *= 2;
5372
5373         wp->plane_bytes_per_line = wp->width * wp->cpp;
5374         if (wp->y_tiled) {
5375                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5376                                            wp->y_min_scanlines,
5377                                            wp->dbuf_block_size);
5378
5379                 if (DISPLAY_VER(dev_priv) >= 10)
5380                         interm_pbpl++;
5381
5382                 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5383                                                         wp->y_min_scanlines);
5384         } else {
5385                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5386                                            wp->dbuf_block_size);
5387
5388                 if (!wp->x_tiled || DISPLAY_VER(dev_priv) >= 10)
5389                         interm_pbpl++;
5390
5391                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5392         }
5393
5394         wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5395                                              wp->plane_blocks_per_line);
5396
5397         wp->linetime_us = fixed16_to_u32_round_up(
5398                                         intel_get_linetime_us(crtc_state));
5399
5400         return 0;
5401 }
5402
5403 static int
5404 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5405                             const struct intel_plane_state *plane_state,
5406                             struct skl_wm_params *wp, int color_plane)
5407 {
5408         const struct drm_framebuffer *fb = plane_state->hw.fb;
5409         int width;
5410
5411         /*
5412          * Src coordinates are already rotated by 270 degrees for
5413          * the 90/270 degree plane rotation cases (to match the
5414          * GTT mapping), hence no need to account for rotation here.
5415          */
5416         width = drm_rect_width(&plane_state->uapi.src) >> 16;
5417
5418         return skl_compute_wm_params(crtc_state, width,
5419                                      fb->format, fb->modifier,
5420                                      plane_state->hw.rotation,
5421                                      intel_plane_pixel_rate(crtc_state, plane_state),
5422                                      wp, color_plane);
5423 }
5424
5425 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5426 {
5427         if (DISPLAY_VER(dev_priv) >= 10)
5428                 return true;
5429
5430         /* The number of lines are ignored for the level 0 watermark. */
5431         return level > 0;
5432 }
5433
5434 static int skl_wm_max_lines(struct drm_i915_private *dev_priv)
5435 {
5436         if (DISPLAY_VER(dev_priv) >= 13)
5437                 return 255;
5438         else
5439                 return 31;
5440 }
5441
5442 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5443                                  struct intel_plane *plane,
5444                                  int level,
5445                                  unsigned int latency,
5446                                  const struct skl_wm_params *wp,
5447                                  const struct skl_wm_level *result_prev,
5448                                  struct skl_wm_level *result /* out */)
5449 {
5450         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5451         uint_fixed_16_16_t method1, method2;
5452         uint_fixed_16_16_t selected_result;
5453         u32 blocks, lines, min_ddb_alloc = 0;
5454
5455         if (latency == 0 ||
5456             (use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
5457                 /* reject it */
5458                 result->min_ddb_alloc = U16_MAX;
5459                 return;
5460         }
5461
5462         /*
5463          * WaIncreaseLatencyIPCEnabled: kbl,cfl
5464          * Display WA #1141: kbl,cfl
5465          */
5466         if ((IS_KABYLAKE(dev_priv) ||
5467              IS_COFFEELAKE(dev_priv) ||
5468              IS_COMETLAKE(dev_priv)) &&
5469             dev_priv->ipc_enabled)
5470                 latency += 4;
5471
5472         if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5473                 latency += 15;
5474
5475         method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5476                                  wp->cpp, latency, wp->dbuf_block_size);
5477         method2 = skl_wm_method2(wp->plane_pixel_rate,
5478                                  crtc_state->hw.pipe_mode.crtc_htotal,
5479                                  latency,
5480                                  wp->plane_blocks_per_line);
5481
5482         if (wp->y_tiled) {
5483                 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5484         } else {
5485                 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
5486                      wp->dbuf_block_size < 1) &&
5487                      (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5488                         selected_result = method2;
5489                 } else if (latency >= wp->linetime_us) {
5490                         if (DISPLAY_VER(dev_priv) == 9)
5491                                 selected_result = min_fixed16(method1, method2);
5492                         else
5493                                 selected_result = method2;
5494                 } else {
5495                         selected_result = method1;
5496                 }
5497         }
5498
5499         blocks = fixed16_to_u32_round_up(selected_result) + 1;
5500         /*
5501          * Lets have blocks at minimum equivalent to plane_blocks_per_line
5502          * as there will be at minimum one line for lines configuration. This
5503          * is a work around for FIFO underruns observed with resolutions like
5504          * 4k 60 Hz in single channel DRAM configurations.
5505          *
5506          * As per the Bspec 49325, if the ddb allocation can hold at least
5507          * one plane_blocks_per_line, we should have selected method2 in
5508          * the above logic. Assuming that modern versions have enough dbuf
5509          * and method2 guarantees blocks equivalent to at least 1 line,
5510          * select the blocks as plane_blocks_per_line.
5511          *
5512          * TODO: Revisit the logic when we have better understanding on DRAM
5513          * channels' impact on the level 0 memory latency and the relevant
5514          * wm calculations.
5515          */
5516         if (skl_wm_has_lines(dev_priv, level))
5517                 blocks = max(blocks,
5518                              fixed16_to_u32_round_up(wp->plane_blocks_per_line));
5519         lines = div_round_up_fixed16(selected_result,
5520                                      wp->plane_blocks_per_line);
5521
5522         if (DISPLAY_VER(dev_priv) == 9) {
5523                 /* Display WA #1125: skl,bxt,kbl */
5524                 if (level == 0 && wp->rc_surface)
5525                         blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5526
5527                 /* Display WA #1126: skl,bxt,kbl */
5528                 if (level >= 1 && level <= 7) {
5529                         if (wp->y_tiled) {
5530                                 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5531                                 lines += wp->y_min_scanlines;
5532                         } else {
5533                                 blocks++;
5534                         }
5535
5536                         /*
5537                          * Make sure result blocks for higher latency levels are
5538                          * atleast as high as level below the current level.
5539                          * Assumption in DDB algorithm optimization for special
5540                          * cases. Also covers Display WA #1125 for RC.
5541                          */
5542                         if (result_prev->blocks > blocks)
5543                                 blocks = result_prev->blocks;
5544                 }
5545         }
5546
5547         if (DISPLAY_VER(dev_priv) >= 11) {
5548                 if (wp->y_tiled) {
5549                         int extra_lines;
5550
5551                         if (lines % wp->y_min_scanlines == 0)
5552                                 extra_lines = wp->y_min_scanlines;
5553                         else
5554                                 extra_lines = wp->y_min_scanlines * 2 -
5555                                         lines % wp->y_min_scanlines;
5556
5557                         min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
5558                                                                  wp->plane_blocks_per_line);
5559                 } else {
5560                         min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
5561                 }
5562         }
5563
5564         if (!skl_wm_has_lines(dev_priv, level))
5565                 lines = 0;
5566
5567         if (lines > skl_wm_max_lines(dev_priv)) {
5568                 /* reject it */
5569                 result->min_ddb_alloc = U16_MAX;
5570                 return;
5571         }
5572
5573         /*
5574          * If lines is valid, assume we can use this watermark level
5575          * for now.  We'll come back and disable it after we calculate the
5576          * DDB allocation if it turns out we don't actually have enough
5577          * blocks to satisfy it.
5578          */
5579         result->blocks = blocks;
5580         result->lines = lines;
5581         /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5582         result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
5583         result->enable = true;
5584
5585         if (DISPLAY_VER(dev_priv) < 12 && dev_priv->display.sagv.block_time_us)
5586                 result->can_sagv = latency >= dev_priv->display.sagv.block_time_us;
5587 }
5588
5589 static void
5590 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5591                       struct intel_plane *plane,
5592                       const struct skl_wm_params *wm_params,
5593                       struct skl_wm_level *levels)
5594 {
5595         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5596         int level, max_level = ilk_wm_max_level(dev_priv);
5597         struct skl_wm_level *result_prev = &levels[0];
5598
5599         for (level = 0; level <= max_level; level++) {
5600                 struct skl_wm_level *result = &levels[level];
5601                 unsigned int latency = dev_priv->display.wm.skl_latency[level];
5602
5603                 skl_compute_plane_wm(crtc_state, plane, level, latency,
5604                                      wm_params, result_prev, result);
5605
5606                 result_prev = result;
5607         }
5608 }
5609
5610 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5611                                 struct intel_plane *plane,
5612                                 const struct skl_wm_params *wm_params,
5613                                 struct skl_plane_wm *plane_wm)
5614 {
5615         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5616         struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
5617         struct skl_wm_level *levels = plane_wm->wm;
5618         unsigned int latency = 0;
5619
5620         if (dev_priv->display.sagv.block_time_us)
5621                 latency = dev_priv->display.sagv.block_time_us + dev_priv->display.wm.skl_latency[0];
5622
5623         skl_compute_plane_wm(crtc_state, plane, 0, latency,
5624                              wm_params, &levels[0],
5625                              sagv_wm);
5626 }
5627
5628 static void skl_compute_transition_wm(struct drm_i915_private *dev_priv,
5629                                       struct skl_wm_level *trans_wm,
5630                                       const struct skl_wm_level *wm0,
5631                                       const struct skl_wm_params *wp)
5632 {
5633         u16 trans_min, trans_amount, trans_y_tile_min;
5634         u16 wm0_blocks, trans_offset, blocks;
5635
5636         /* Transition WM don't make any sense if ipc is disabled */
5637         if (!dev_priv->ipc_enabled)
5638                 return;
5639
5640         /*
5641          * WaDisableTWM:skl,kbl,cfl,bxt
5642          * Transition WM are not recommended by HW team for GEN9
5643          */
5644         if (DISPLAY_VER(dev_priv) == 9)
5645                 return;
5646
5647         if (DISPLAY_VER(dev_priv) >= 11)
5648                 trans_min = 4;
5649         else
5650                 trans_min = 14;
5651
5652         /* Display WA #1140: glk,cnl */
5653         if (DISPLAY_VER(dev_priv) == 10)
5654                 trans_amount = 0;
5655         else
5656                 trans_amount = 10; /* This is configurable amount */
5657
5658         trans_offset = trans_min + trans_amount;
5659
5660         /*
5661          * The spec asks for Selected Result Blocks for wm0 (the real value),
5662          * not Result Blocks (the integer value). Pay attention to the capital
5663          * letters. The value wm_l0->blocks is actually Result Blocks, but
5664          * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5665          * and since we later will have to get the ceiling of the sum in the
5666          * transition watermarks calculation, we can just pretend Selected
5667          * Result Blocks is Result Blocks minus 1 and it should work for the
5668          * current platforms.
5669          */
5670         wm0_blocks = wm0->blocks - 1;
5671
5672         if (wp->y_tiled) {
5673                 trans_y_tile_min =
5674                         (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5675                 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
5676         } else {
5677                 blocks = wm0_blocks + trans_offset;
5678         }
5679         blocks++;
5680
5681         /*
5682          * Just assume we can enable the transition watermark.  After
5683          * computing the DDB we'll come back and disable it if that
5684          * assumption turns out to be false.
5685          */
5686         trans_wm->blocks = blocks;
5687         trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
5688         trans_wm->enable = true;
5689 }
5690
5691 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5692                                      const struct intel_plane_state *plane_state,
5693                                      struct intel_plane *plane, int color_plane)
5694 {
5695         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5696         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5697         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
5698         struct skl_wm_params wm_params;
5699         int ret;
5700
5701         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5702                                           &wm_params, color_plane);
5703         if (ret)
5704                 return ret;
5705
5706         skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm);
5707
5708         skl_compute_transition_wm(dev_priv, &wm->trans_wm,
5709                                   &wm->wm[0], &wm_params);
5710
5711         if (DISPLAY_VER(dev_priv) >= 12) {
5712                 tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm);
5713
5714                 skl_compute_transition_wm(dev_priv, &wm->sagv.trans_wm,
5715                                           &wm->sagv.wm0, &wm_params);
5716         }
5717
5718         return 0;
5719 }
5720
5721 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5722                                  const struct intel_plane_state *plane_state,
5723                                  struct intel_plane *plane)
5724 {
5725         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
5726         struct skl_wm_params wm_params;
5727         int ret;
5728
5729         wm->is_planar = true;
5730
5731         /* uv plane watermarks must also be validated for NV12/Planar */
5732         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5733                                           &wm_params, 1);
5734         if (ret)
5735                 return ret;
5736
5737         skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm);
5738
5739         return 0;
5740 }
5741
5742 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5743                               const struct intel_plane_state *plane_state)
5744 {
5745         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5746         enum plane_id plane_id = plane->id;
5747         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5748         const struct drm_framebuffer *fb = plane_state->hw.fb;
5749         int ret;
5750
5751         memset(wm, 0, sizeof(*wm));
5752
5753         if (!intel_wm_plane_visible(crtc_state, plane_state))
5754                 return 0;
5755
5756         ret = skl_build_plane_wm_single(crtc_state, plane_state,
5757                                         plane, 0);
5758         if (ret)
5759                 return ret;
5760
5761         if (fb->format->is_yuv && fb->format->num_planes > 1) {
5762                 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5763                                             plane);
5764                 if (ret)
5765                         return ret;
5766         }
5767
5768         return 0;
5769 }
5770
5771 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5772                               const struct intel_plane_state *plane_state)
5773 {
5774         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5775         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5776         enum plane_id plane_id = plane->id;
5777         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5778         int ret;
5779
5780         /* Watermarks calculated in master */
5781         if (plane_state->planar_slave)
5782                 return 0;
5783
5784         memset(wm, 0, sizeof(*wm));
5785
5786         if (plane_state->planar_linked_plane) {
5787                 const struct drm_framebuffer *fb = plane_state->hw.fb;
5788
5789                 drm_WARN_ON(&dev_priv->drm,
5790                             !intel_wm_plane_visible(crtc_state, plane_state));
5791                 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5792                             fb->format->num_planes == 1);
5793
5794                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5795                                                 plane_state->planar_linked_plane, 0);
5796                 if (ret)
5797                         return ret;
5798
5799                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5800                                                 plane, 1);
5801                 if (ret)
5802                         return ret;
5803         } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5804                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5805                                                 plane, 0);
5806                 if (ret)
5807                         return ret;
5808         }
5809
5810         return 0;
5811 }
5812
5813 static int skl_build_pipe_wm(struct intel_atomic_state *state,
5814                              struct intel_crtc *crtc)
5815 {
5816         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5817         struct intel_crtc_state *crtc_state =
5818                 intel_atomic_get_new_crtc_state(state, crtc);
5819         const struct intel_plane_state *plane_state;
5820         struct intel_plane *plane;
5821         int ret, i;
5822
5823         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
5824                 /*
5825                  * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
5826                  * instead but we don't populate that correctly for NV12 Y
5827                  * planes so for now hack this.
5828                  */
5829                 if (plane->pipe != crtc->pipe)
5830                         continue;
5831
5832                 if (DISPLAY_VER(dev_priv) >= 11)
5833                         ret = icl_build_plane_wm(crtc_state, plane_state);
5834                 else
5835                         ret = skl_build_plane_wm(crtc_state, plane_state);
5836                 if (ret)
5837                         return ret;
5838         }
5839
5840         crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
5841
5842         return 0;
5843 }
5844
5845 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5846                                 i915_reg_t reg,
5847                                 const struct skl_ddb_entry *entry)
5848 {
5849         if (entry->end)
5850                 intel_de_write_fw(dev_priv, reg,
5851                                   PLANE_BUF_END(entry->end - 1) |
5852                                   PLANE_BUF_START(entry->start));
5853         else
5854                 intel_de_write_fw(dev_priv, reg, 0);
5855 }
5856
5857 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5858                                i915_reg_t reg,
5859                                const struct skl_wm_level *level)
5860 {
5861         u32 val = 0;
5862
5863         if (level->enable)
5864                 val |= PLANE_WM_EN;
5865         if (level->ignore_lines)
5866                 val |= PLANE_WM_IGNORE_LINES;
5867         val |= REG_FIELD_PREP(PLANE_WM_BLOCKS_MASK, level->blocks);
5868         val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
5869
5870         intel_de_write_fw(dev_priv, reg, val);
5871 }
5872
5873 void skl_write_plane_wm(struct intel_plane *plane,
5874                         const struct intel_crtc_state *crtc_state)
5875 {
5876         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5877         int level, max_level = ilk_wm_max_level(dev_priv);
5878         enum plane_id plane_id = plane->id;
5879         enum pipe pipe = plane->pipe;
5880         const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5881         const struct skl_ddb_entry *ddb =
5882                 &crtc_state->wm.skl.plane_ddb[plane_id];
5883         const struct skl_ddb_entry *ddb_y =
5884                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5885
5886         for (level = 0; level <= max_level; level++)
5887                 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5888                                    skl_plane_wm_level(pipe_wm, plane_id, level));
5889
5890         skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5891                            skl_plane_trans_wm(pipe_wm, plane_id));
5892
5893         if (HAS_HW_SAGV_WM(dev_priv)) {
5894                 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5895
5896                 skl_write_wm_level(dev_priv, PLANE_WM_SAGV(pipe, plane_id),
5897                                    &wm->sagv.wm0);
5898                 skl_write_wm_level(dev_priv, PLANE_WM_SAGV_TRANS(pipe, plane_id),
5899                                    &wm->sagv.trans_wm);
5900         }
5901
5902         skl_ddb_entry_write(dev_priv,
5903                             PLANE_BUF_CFG(pipe, plane_id), ddb);
5904
5905         if (DISPLAY_VER(dev_priv) < 11)
5906                 skl_ddb_entry_write(dev_priv,
5907                                     PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_y);
5908 }
5909
5910 void skl_write_cursor_wm(struct intel_plane *plane,
5911                          const struct intel_crtc_state *crtc_state)
5912 {
5913         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5914         int level, max_level = ilk_wm_max_level(dev_priv);
5915         enum plane_id plane_id = plane->id;
5916         enum pipe pipe = plane->pipe;
5917         const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5918         const struct skl_ddb_entry *ddb =
5919                 &crtc_state->wm.skl.plane_ddb[plane_id];
5920
5921         for (level = 0; level <= max_level; level++)
5922                 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5923                                    skl_plane_wm_level(pipe_wm, plane_id, level));
5924
5925         skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe),
5926                            skl_plane_trans_wm(pipe_wm, plane_id));
5927
5928         if (HAS_HW_SAGV_WM(dev_priv)) {
5929                 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5930
5931                 skl_write_wm_level(dev_priv, CUR_WM_SAGV(pipe),
5932                                    &wm->sagv.wm0);
5933                 skl_write_wm_level(dev_priv, CUR_WM_SAGV_TRANS(pipe),
5934                                    &wm->sagv.trans_wm);
5935         }
5936
5937         skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5938 }
5939
5940 static bool skl_wm_level_equals(const struct skl_wm_level *l1,
5941                                 const struct skl_wm_level *l2)
5942 {
5943         return l1->enable == l2->enable &&
5944                 l1->ignore_lines == l2->ignore_lines &&
5945                 l1->lines == l2->lines &&
5946                 l1->blocks == l2->blocks;
5947 }
5948
5949 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5950                                 const struct skl_plane_wm *wm1,
5951                                 const struct skl_plane_wm *wm2)
5952 {
5953         int level, max_level = ilk_wm_max_level(dev_priv);
5954
5955         for (level = 0; level <= max_level; level++) {
5956                 /*
5957                  * We don't check uv_wm as the hardware doesn't actually
5958                  * use it. It only gets used for calculating the required
5959                  * ddb allocation.
5960                  */
5961                 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5962                         return false;
5963         }
5964
5965         return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
5966                 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
5967                 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
5968 }
5969
5970 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5971                                     const struct skl_ddb_entry *b)
5972 {
5973         return a->start < b->end && b->start < a->end;
5974 }
5975
5976 static void skl_ddb_entry_union(struct skl_ddb_entry *a,
5977                                 const struct skl_ddb_entry *b)
5978 {
5979         if (a->end && b->end) {
5980                 a->start = min(a->start, b->start);
5981                 a->end = max(a->end, b->end);
5982         } else if (b->end) {
5983                 a->start = b->start;
5984                 a->end = b->end;
5985         }
5986 }
5987
5988 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5989                                  const struct skl_ddb_entry *entries,
5990                                  int num_entries, int ignore_idx)
5991 {
5992         int i;
5993
5994         for (i = 0; i < num_entries; i++) {
5995                 if (i != ignore_idx &&
5996                     skl_ddb_entries_overlap(ddb, &entries[i]))
5997                         return true;
5998         }
5999
6000         return false;
6001 }
6002
6003 static int
6004 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
6005                             struct intel_crtc_state *new_crtc_state)
6006 {
6007         struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
6008         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6009         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6010         struct intel_plane *plane;
6011
6012         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6013                 struct intel_plane_state *plane_state;
6014                 enum plane_id plane_id = plane->id;
6015
6016                 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb[plane_id],
6017                                         &new_crtc_state->wm.skl.plane_ddb[plane_id]) &&
6018                     skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
6019                                         &new_crtc_state->wm.skl.plane_ddb_y[plane_id]))
6020                         continue;
6021
6022                 plane_state = intel_atomic_get_plane_state(state, plane);
6023                 if (IS_ERR(plane_state))
6024                         return PTR_ERR(plane_state);
6025
6026                 new_crtc_state->update_planes |= BIT(plane_id);
6027         }
6028
6029         return 0;
6030 }
6031
6032 static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
6033 {
6034         struct drm_i915_private *dev_priv = to_i915(dbuf_state->base.state->base.dev);
6035         u8 enabled_slices;
6036         enum pipe pipe;
6037
6038         /*
6039          * FIXME: For now we always enable slice S1 as per
6040          * the Bspec display initialization sequence.
6041          */
6042         enabled_slices = BIT(DBUF_S1);
6043
6044         for_each_pipe(dev_priv, pipe)
6045                 enabled_slices |= dbuf_state->slices[pipe];
6046
6047         return enabled_slices;
6048 }
6049
6050 static int
6051 skl_compute_ddb(struct intel_atomic_state *state)
6052 {
6053         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6054         const struct intel_dbuf_state *old_dbuf_state;
6055         struct intel_dbuf_state *new_dbuf_state = NULL;
6056         const struct intel_crtc_state *old_crtc_state;
6057         struct intel_crtc_state *new_crtc_state;
6058         struct intel_crtc *crtc;
6059         int ret, i;
6060
6061         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6062                 new_dbuf_state = intel_atomic_get_dbuf_state(state);
6063                 if (IS_ERR(new_dbuf_state))
6064                         return PTR_ERR(new_dbuf_state);
6065
6066                 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
6067                 break;
6068         }
6069
6070         if (!new_dbuf_state)
6071                 return 0;
6072
6073         new_dbuf_state->active_pipes =
6074                 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
6075
6076         if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
6077                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6078                 if (ret)
6079                         return ret;
6080         }
6081
6082         if (HAS_MBUS_JOINING(dev_priv))
6083                 new_dbuf_state->joined_mbus =
6084                         adlp_check_mbus_joined(new_dbuf_state->active_pipes);
6085
6086         for_each_intel_crtc(&dev_priv->drm, crtc) {
6087                 enum pipe pipe = crtc->pipe;
6088
6089                 new_dbuf_state->slices[pipe] =
6090                         skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes,
6091                                                 new_dbuf_state->joined_mbus);
6092
6093                 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
6094                         continue;
6095
6096                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6097                 if (ret)
6098                         return ret;
6099         }
6100
6101         new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
6102
6103         if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
6104             old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6105                 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
6106                 if (ret)
6107                         return ret;
6108
6109                 if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6110                         /* TODO: Implement vblank synchronized MBUS joining changes */
6111                         ret = intel_modeset_all_pipes(state);
6112                         if (ret)
6113                                 return ret;
6114                 }
6115
6116                 drm_dbg_kms(&dev_priv->drm,
6117                             "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
6118                             old_dbuf_state->enabled_slices,
6119                             new_dbuf_state->enabled_slices,
6120                             INTEL_INFO(dev_priv)->display.dbuf.slice_mask,
6121                             str_yes_no(old_dbuf_state->joined_mbus),
6122                             str_yes_no(new_dbuf_state->joined_mbus));
6123         }
6124
6125         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6126                 enum pipe pipe = crtc->pipe;
6127
6128                 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
6129
6130                 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
6131                         continue;
6132
6133                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6134                 if (ret)
6135                         return ret;
6136         }
6137
6138         for_each_intel_crtc(&dev_priv->drm, crtc) {
6139                 ret = skl_crtc_allocate_ddb(state, crtc);
6140                 if (ret)
6141                         return ret;
6142         }
6143
6144         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6145                                             new_crtc_state, i) {
6146                 ret = skl_crtc_allocate_plane_ddb(state, crtc);
6147                 if (ret)
6148                         return ret;
6149
6150                 ret = skl_ddb_add_affected_planes(old_crtc_state,
6151                                                   new_crtc_state);
6152                 if (ret)
6153                         return ret;
6154         }
6155
6156         return 0;
6157 }
6158
6159 static char enast(bool enable)
6160 {
6161         return enable ? '*' : ' ';
6162 }
6163
6164 static void
6165 skl_print_wm_changes(struct intel_atomic_state *state)
6166 {
6167         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6168         const struct intel_crtc_state *old_crtc_state;
6169         const struct intel_crtc_state *new_crtc_state;
6170         struct intel_plane *plane;
6171         struct intel_crtc *crtc;
6172         int i;
6173
6174         if (!drm_debug_enabled(DRM_UT_KMS))
6175                 return;
6176
6177         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6178                                             new_crtc_state, i) {
6179                 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
6180
6181                 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
6182                 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
6183
6184                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6185                         enum plane_id plane_id = plane->id;
6186                         const struct skl_ddb_entry *old, *new;
6187
6188                         old = &old_crtc_state->wm.skl.plane_ddb[plane_id];
6189                         new = &new_crtc_state->wm.skl.plane_ddb[plane_id];
6190
6191                         if (skl_ddb_entry_equal(old, new))
6192                                 continue;
6193
6194                         drm_dbg_kms(&dev_priv->drm,
6195                                     "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
6196                                     plane->base.base.id, plane->base.name,
6197                                     old->start, old->end, new->start, new->end,
6198                                     skl_ddb_entry_size(old), skl_ddb_entry_size(new));
6199                 }
6200
6201                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6202                         enum plane_id plane_id = plane->id;
6203                         const struct skl_plane_wm *old_wm, *new_wm;
6204
6205                         old_wm = &old_pipe_wm->planes[plane_id];
6206                         new_wm = &new_pipe_wm->planes[plane_id];
6207
6208                         if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
6209                                 continue;
6210
6211                         drm_dbg_kms(&dev_priv->drm,
6212                                     "[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
6213                                     " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
6214                                     plane->base.base.id, plane->base.name,
6215                                     enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
6216                                     enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
6217                                     enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
6218                                     enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
6219                                     enast(old_wm->trans_wm.enable),
6220                                     enast(old_wm->sagv.wm0.enable),
6221                                     enast(old_wm->sagv.trans_wm.enable),
6222                                     enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
6223                                     enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
6224                                     enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
6225                                     enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
6226                                     enast(new_wm->trans_wm.enable),
6227                                     enast(new_wm->sagv.wm0.enable),
6228                                     enast(new_wm->sagv.trans_wm.enable));
6229
6230                         drm_dbg_kms(&dev_priv->drm,
6231                                     "[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
6232                                       " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
6233                                     plane->base.base.id, plane->base.name,
6234                                     enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
6235                                     enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
6236                                     enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
6237                                     enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
6238                                     enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
6239                                     enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
6240                                     enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
6241                                     enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
6242                                     enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
6243                                     enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
6244                                     enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
6245                                     enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
6246                                     enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
6247                                     enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
6248                                     enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
6249                                     enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
6250                                     enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
6251                                     enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
6252                                     enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
6253                                     enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
6254                                     enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
6255                                     enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
6256
6257                         drm_dbg_kms(&dev_priv->drm,
6258                                     "[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
6259                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6260                                     plane->base.base.id, plane->base.name,
6261                                     old_wm->wm[0].blocks, old_wm->wm[1].blocks,
6262                                     old_wm->wm[2].blocks, old_wm->wm[3].blocks,
6263                                     old_wm->wm[4].blocks, old_wm->wm[5].blocks,
6264                                     old_wm->wm[6].blocks, old_wm->wm[7].blocks,
6265                                     old_wm->trans_wm.blocks,
6266                                     old_wm->sagv.wm0.blocks,
6267                                     old_wm->sagv.trans_wm.blocks,
6268                                     new_wm->wm[0].blocks, new_wm->wm[1].blocks,
6269                                     new_wm->wm[2].blocks, new_wm->wm[3].blocks,
6270                                     new_wm->wm[4].blocks, new_wm->wm[5].blocks,
6271                                     new_wm->wm[6].blocks, new_wm->wm[7].blocks,
6272                                     new_wm->trans_wm.blocks,
6273                                     new_wm->sagv.wm0.blocks,
6274                                     new_wm->sagv.trans_wm.blocks);
6275
6276                         drm_dbg_kms(&dev_priv->drm,
6277                                     "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
6278                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6279                                     plane->base.base.id, plane->base.name,
6280                                     old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
6281                                     old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
6282                                     old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
6283                                     old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
6284                                     old_wm->trans_wm.min_ddb_alloc,
6285                                     old_wm->sagv.wm0.min_ddb_alloc,
6286                                     old_wm->sagv.trans_wm.min_ddb_alloc,
6287                                     new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
6288                                     new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
6289                                     new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
6290                                     new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
6291                                     new_wm->trans_wm.min_ddb_alloc,
6292                                     new_wm->sagv.wm0.min_ddb_alloc,
6293                                     new_wm->sagv.trans_wm.min_ddb_alloc);
6294                 }
6295         }
6296 }
6297
6298 static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
6299                                          const struct skl_pipe_wm *old_pipe_wm,
6300                                          const struct skl_pipe_wm *new_pipe_wm)
6301 {
6302         struct drm_i915_private *i915 = to_i915(plane->base.dev);
6303         int level, max_level = ilk_wm_max_level(i915);
6304
6305         for (level = 0; level <= max_level; level++) {
6306                 /*
6307                  * We don't check uv_wm as the hardware doesn't actually
6308                  * use it. It only gets used for calculating the required
6309                  * ddb allocation.
6310                  */
6311                 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
6312                                          skl_plane_wm_level(new_pipe_wm, plane->id, level)))
6313                         return false;
6314         }
6315
6316         if (HAS_HW_SAGV_WM(i915)) {
6317                 const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
6318                 const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
6319
6320                 if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
6321                     !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
6322                         return false;
6323         }
6324
6325         return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
6326                                    skl_plane_trans_wm(new_pipe_wm, plane->id));
6327 }
6328
6329 /*
6330  * To make sure the cursor watermark registers are always consistent
6331  * with our computed state the following scenario needs special
6332  * treatment:
6333  *
6334  * 1. enable cursor
6335  * 2. move cursor entirely offscreen
6336  * 3. disable cursor
6337  *
6338  * Step 2. does call .disable_plane() but does not zero the watermarks
6339  * (since we consider an offscreen cursor still active for the purposes
6340  * of watermarks). Step 3. would not normally call .disable_plane()
6341  * because the actual plane visibility isn't changing, and we don't
6342  * deallocate the cursor ddb until the pipe gets disabled. So we must
6343  * force step 3. to call .disable_plane() to update the watermark
6344  * registers properly.
6345  *
6346  * Other planes do not suffer from this issues as their watermarks are
6347  * calculated based on the actual plane visibility. The only time this
6348  * can trigger for the other planes is during the initial readout as the
6349  * default value of the watermarks registers is not zero.
6350  */
6351 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6352                                       struct intel_crtc *crtc)
6353 {
6354         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6355         const struct intel_crtc_state *old_crtc_state =
6356                 intel_atomic_get_old_crtc_state(state, crtc);
6357         struct intel_crtc_state *new_crtc_state =
6358                 intel_atomic_get_new_crtc_state(state, crtc);
6359         struct intel_plane *plane;
6360
6361         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6362                 struct intel_plane_state *plane_state;
6363                 enum plane_id plane_id = plane->id;
6364
6365                 /*
6366                  * Force a full wm update for every plane on modeset.
6367                  * Required because the reset value of the wm registers
6368                  * is non-zero, whereas we want all disabled planes to
6369                  * have zero watermarks. So if we turn off the relevant
6370                  * power well the hardware state will go out of sync
6371                  * with the software state.
6372                  */
6373                 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6374                     skl_plane_selected_wm_equals(plane,
6375                                                  &old_crtc_state->wm.skl.optimal,
6376                                                  &new_crtc_state->wm.skl.optimal))
6377                         continue;
6378
6379                 plane_state = intel_atomic_get_plane_state(state, plane);
6380                 if (IS_ERR(plane_state))
6381                         return PTR_ERR(plane_state);
6382
6383                 new_crtc_state->update_planes |= BIT(plane_id);
6384         }
6385
6386         return 0;
6387 }
6388
6389 static int
6390 skl_compute_wm(struct intel_atomic_state *state)
6391 {
6392         struct intel_crtc *crtc;
6393         struct intel_crtc_state *new_crtc_state;
6394         int ret, i;
6395
6396         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6397                 ret = skl_build_pipe_wm(state, crtc);
6398                 if (ret)
6399                         return ret;
6400         }
6401
6402         ret = skl_compute_ddb(state);
6403         if (ret)
6404                 return ret;
6405
6406         ret = intel_compute_sagv_mask(state);
6407         if (ret)
6408                 return ret;
6409
6410         /*
6411          * skl_compute_ddb() will have adjusted the final watermarks
6412          * based on how much ddb is available. Now we can actually
6413          * check if the final watermarks changed.
6414          */
6415         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6416                 ret = skl_wm_add_affected_planes(state, crtc);
6417                 if (ret)
6418                         return ret;
6419         }
6420
6421         skl_print_wm_changes(state);
6422
6423         return 0;
6424 }
6425
6426 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6427                                   struct intel_wm_config *config)
6428 {
6429         struct intel_crtc *crtc;
6430
6431         /* Compute the currently _active_ config */
6432         for_each_intel_crtc(&dev_priv->drm, crtc) {
6433                 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6434
6435                 if (!wm->pipe_enabled)
6436                         continue;
6437
6438                 config->sprites_enabled |= wm->sprites_enabled;
6439                 config->sprites_scaled |= wm->sprites_scaled;
6440                 config->num_pipes_active++;
6441         }
6442 }
6443
6444 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6445 {
6446         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6447         struct ilk_wm_maximums max;
6448         struct intel_wm_config config = {};
6449         struct ilk_wm_values results = {};
6450         enum intel_ddb_partitioning partitioning;
6451
6452         ilk_compute_wm_config(dev_priv, &config);
6453
6454         ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6455         ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6456
6457         /* 5/6 split only in single pipe config on IVB+ */
6458         if (DISPLAY_VER(dev_priv) >= 7 &&
6459             config.num_pipes_active == 1 && config.sprites_enabled) {
6460                 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6461                 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6462
6463                 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6464         } else {
6465                 best_lp_wm = &lp_wm_1_2;
6466         }
6467
6468         partitioning = (best_lp_wm == &lp_wm_1_2) ?
6469                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6470
6471         ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6472
6473         ilk_write_wm_values(dev_priv, &results);
6474 }
6475
6476 static void ilk_initial_watermarks(struct intel_atomic_state *state,
6477                                    struct intel_crtc *crtc)
6478 {
6479         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6480         const struct intel_crtc_state *crtc_state =
6481                 intel_atomic_get_new_crtc_state(state, crtc);
6482
6483         mutex_lock(&dev_priv->display.wm.wm_mutex);
6484         crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6485         ilk_program_watermarks(dev_priv);
6486         mutex_unlock(&dev_priv->display.wm.wm_mutex);
6487 }
6488
6489 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6490                                     struct intel_crtc *crtc)
6491 {
6492         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6493         const struct intel_crtc_state *crtc_state =
6494                 intel_atomic_get_new_crtc_state(state, crtc);
6495
6496         if (!crtc_state->wm.need_postvbl_update)
6497                 return;
6498
6499         mutex_lock(&dev_priv->display.wm.wm_mutex);
6500         crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6501         ilk_program_watermarks(dev_priv);
6502         mutex_unlock(&dev_priv->display.wm.wm_mutex);
6503 }
6504
6505 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6506 {
6507         level->enable = val & PLANE_WM_EN;
6508         level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6509         level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val);
6510         level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
6511 }
6512
6513 static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6514                                      struct skl_pipe_wm *out)
6515 {
6516         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6517         enum pipe pipe = crtc->pipe;
6518         int level, max_level;
6519         enum plane_id plane_id;
6520         u32 val;
6521
6522         max_level = ilk_wm_max_level(dev_priv);
6523
6524         for_each_plane_id_on_crtc(crtc, plane_id) {
6525                 struct skl_plane_wm *wm = &out->planes[plane_id];
6526
6527                 for (level = 0; level <= max_level; level++) {
6528                         if (plane_id != PLANE_CURSOR)
6529                                 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM(pipe, plane_id, level));
6530                         else
6531                                 val = intel_uncore_read(&dev_priv->uncore, CUR_WM(pipe, level));
6532
6533                         skl_wm_level_from_reg_val(val, &wm->wm[level]);
6534                 }
6535
6536                 if (plane_id != PLANE_CURSOR)
6537                         val = intel_uncore_read(&dev_priv->uncore, PLANE_WM_TRANS(pipe, plane_id));
6538                 else
6539                         val = intel_uncore_read(&dev_priv->uncore, CUR_WM_TRANS(pipe));
6540
6541                 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6542
6543                 if (HAS_HW_SAGV_WM(dev_priv)) {
6544                         if (plane_id != PLANE_CURSOR)
6545                                 val = intel_uncore_read(&dev_priv->uncore,
6546                                                         PLANE_WM_SAGV(pipe, plane_id));
6547                         else
6548                                 val = intel_uncore_read(&dev_priv->uncore,
6549                                                         CUR_WM_SAGV(pipe));
6550
6551                         skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
6552
6553                         if (plane_id != PLANE_CURSOR)
6554                                 val = intel_uncore_read(&dev_priv->uncore,
6555                                                         PLANE_WM_SAGV_TRANS(pipe, plane_id));
6556                         else
6557                                 val = intel_uncore_read(&dev_priv->uncore,
6558                                                         CUR_WM_SAGV_TRANS(pipe));
6559
6560                         skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
6561                 } else if (DISPLAY_VER(dev_priv) >= 12) {
6562                         wm->sagv.wm0 = wm->wm[0];
6563                         wm->sagv.trans_wm = wm->trans_wm;
6564                 }
6565         }
6566 }
6567
6568 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6569 {
6570         struct intel_dbuf_state *dbuf_state =
6571                 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
6572         struct intel_crtc *crtc;
6573
6574         if (HAS_MBUS_JOINING(dev_priv))
6575                 dbuf_state->joined_mbus = intel_de_read(dev_priv, MBUS_CTL) & MBUS_JOIN;
6576
6577         for_each_intel_crtc(&dev_priv->drm, crtc) {
6578                 struct intel_crtc_state *crtc_state =
6579                         to_intel_crtc_state(crtc->base.state);
6580                 enum pipe pipe = crtc->pipe;
6581                 unsigned int mbus_offset;
6582                 enum plane_id plane_id;
6583                 u8 slices;
6584
6585                 memset(&crtc_state->wm.skl.optimal, 0,
6586                        sizeof(crtc_state->wm.skl.optimal));
6587                 if (crtc_state->hw.active)
6588                         skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6589                 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
6590
6591                 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
6592
6593                 for_each_plane_id_on_crtc(crtc, plane_id) {
6594                         struct skl_ddb_entry *ddb =
6595                                 &crtc_state->wm.skl.plane_ddb[plane_id];
6596                         struct skl_ddb_entry *ddb_y =
6597                                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
6598
6599                         if (!crtc_state->hw.active)
6600                                 continue;
6601
6602                         skl_ddb_get_hw_plane_state(dev_priv, crtc->pipe,
6603                                                    plane_id, ddb, ddb_y);
6604
6605                         skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb);
6606                         skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
6607                 }
6608
6609                 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
6610
6611                 /*
6612                  * Used for checking overlaps, so we need absolute
6613                  * offsets instead of MBUS relative offsets.
6614                  */
6615                 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
6616                                                  dbuf_state->joined_mbus);
6617                 mbus_offset = mbus_ddb_offset(dev_priv, slices);
6618                 crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
6619                 crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
6620
6621                 /* The slices actually used by the planes on the pipe */
6622                 dbuf_state->slices[pipe] =
6623                         skl_ddb_dbuf_slice_mask(dev_priv, &crtc_state->wm.skl.ddb);
6624
6625                 drm_dbg_kms(&dev_priv->drm,
6626                             "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
6627                             crtc->base.base.id, crtc->base.name,
6628                             dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
6629                             dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
6630                             str_yes_no(dbuf_state->joined_mbus));
6631         }
6632
6633         dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices;
6634 }
6635
6636 static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
6637 {
6638         const struct intel_dbuf_state *dbuf_state =
6639                 to_intel_dbuf_state(i915->dbuf.obj.state);
6640         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
6641         struct intel_crtc *crtc;
6642
6643         for_each_intel_crtc(&i915->drm, crtc) {
6644                 const struct intel_crtc_state *crtc_state =
6645                         to_intel_crtc_state(crtc->base.state);
6646
6647                 entries[crtc->pipe] = crtc_state->wm.skl.ddb;
6648         }
6649
6650         for_each_intel_crtc(&i915->drm, crtc) {
6651                 const struct intel_crtc_state *crtc_state =
6652                         to_intel_crtc_state(crtc->base.state);
6653                 u8 slices;
6654
6655                 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
6656                                                  dbuf_state->joined_mbus);
6657                 if (dbuf_state->slices[crtc->pipe] & ~slices)
6658                         return true;
6659
6660                 if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries,
6661                                                 I915_MAX_PIPES, crtc->pipe))
6662                         return true;
6663         }
6664
6665         return false;
6666 }
6667
6668 void skl_wm_sanitize(struct drm_i915_private *i915)
6669 {
6670         struct intel_crtc *crtc;
6671
6672         /*
6673          * On TGL/RKL (at least) the BIOS likes to assign the planes
6674          * to the wrong DBUF slices. This will cause an infinite loop
6675          * in skl_commit_modeset_enables() as it can't find a way to
6676          * transition between the old bogus DBUF layout to the new
6677          * proper DBUF layout without DBUF allocation overlaps between
6678          * the planes (which cannot be allowed or else the hardware
6679          * may hang). If we detect a bogus DBUF layout just turn off
6680          * all the planes so that skl_commit_modeset_enables() can
6681          * simply ignore them.
6682          */
6683         if (!skl_dbuf_is_misconfigured(i915))
6684                 return;
6685
6686         drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
6687
6688         for_each_intel_crtc(&i915->drm, crtc) {
6689                 struct intel_plane *plane = to_intel_plane(crtc->base.primary);
6690                 const struct intel_plane_state *plane_state =
6691                         to_intel_plane_state(plane->base.state);
6692                 struct intel_crtc_state *crtc_state =
6693                         to_intel_crtc_state(crtc->base.state);
6694
6695                 if (plane_state->uapi.visible)
6696                         intel_plane_disable_noatomic(crtc, plane);
6697
6698                 drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
6699
6700                 memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
6701         }
6702 }
6703
6704 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6705 {
6706         struct drm_device *dev = crtc->base.dev;
6707         struct drm_i915_private *dev_priv = to_i915(dev);
6708         struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
6709         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6710         struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6711         enum pipe pipe = crtc->pipe;
6712
6713         hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
6714
6715         memset(active, 0, sizeof(*active));
6716
6717         active->pipe_enabled = crtc->active;
6718
6719         if (active->pipe_enabled) {
6720                 u32 tmp = hw->wm_pipe[pipe];
6721
6722                 /*
6723                  * For active pipes LP0 watermark is marked as
6724                  * enabled, and LP1+ watermaks as disabled since
6725                  * we can't really reverse compute them in case
6726                  * multiple pipes are active.
6727                  */
6728                 active->wm[0].enable = true;
6729                 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
6730                 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
6731                 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
6732         } else {
6733                 int level, max_level = ilk_wm_max_level(dev_priv);
6734
6735                 /*
6736                  * For inactive pipes, all watermark levels
6737                  * should be marked as enabled but zeroed,
6738                  * which is what we'd compute them to.
6739                  */
6740                 for (level = 0; level <= max_level; level++)
6741                         active->wm[level].enable = true;
6742         }
6743
6744         crtc->wm.active.ilk = *active;
6745 }
6746
6747 #define _FW_WM(value, plane) \
6748         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6749 #define _FW_WM_VLV(value, plane) \
6750         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6751
6752 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6753                                struct g4x_wm_values *wm)
6754 {
6755         u32 tmp;
6756
6757         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6758         wm->sr.plane = _FW_WM(tmp, SR);
6759         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6760         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6761         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6762
6763         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6764         wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6765         wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6766         wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6767         wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6768         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6769         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6770
6771         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6772         wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6773         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6774         wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6775         wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6776 }
6777
6778 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6779                                struct vlv_wm_values *wm)
6780 {
6781         enum pipe pipe;
6782         u32 tmp;
6783
6784         for_each_pipe(dev_priv, pipe) {
6785                 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
6786
6787                 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6788                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6789                 wm->ddl[pipe].plane[PLANE_CURSOR] =
6790                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6791                 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6792                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6793                 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6794                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6795         }
6796
6797         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6798         wm->sr.plane = _FW_WM(tmp, SR);
6799         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6800         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6801         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6802
6803         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6804         wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6805         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6806         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6807
6808         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6809         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6810
6811         if (IS_CHERRYVIEW(dev_priv)) {
6812                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
6813                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6814                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6815
6816                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
6817                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6818                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6819
6820                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
6821                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6822                 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6823
6824                 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6825                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6826                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6827                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6828                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6829                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6830                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6831                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6832                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6833                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6834                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6835         } else {
6836                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
6837                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6838                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6839
6840                 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6841                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6842                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6843                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6844                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6845                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6846                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6847                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6848         }
6849 }
6850
6851 #undef _FW_WM
6852 #undef _FW_WM_VLV
6853
6854 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6855 {
6856         struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;
6857         struct intel_crtc *crtc;
6858
6859         g4x_read_wm_values(dev_priv, wm);
6860
6861         wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
6862
6863         for_each_intel_crtc(&dev_priv->drm, crtc) {
6864                 struct intel_crtc_state *crtc_state =
6865                         to_intel_crtc_state(crtc->base.state);
6866                 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6867                 struct g4x_pipe_wm *raw;
6868                 enum pipe pipe = crtc->pipe;
6869                 enum plane_id plane_id;
6870                 int level, max_level;
6871
6872                 active->cxsr = wm->cxsr;
6873                 active->hpll_en = wm->hpll_en;
6874                 active->fbc_en = wm->fbc_en;
6875
6876                 active->sr = wm->sr;
6877                 active->hpll = wm->hpll;
6878
6879                 for_each_plane_id_on_crtc(crtc, plane_id) {
6880                         active->wm.plane[plane_id] =
6881                                 wm->pipe[pipe].plane[plane_id];
6882                 }
6883
6884                 if (wm->cxsr && wm->hpll_en)
6885                         max_level = G4X_WM_LEVEL_HPLL;
6886                 else if (wm->cxsr)
6887                         max_level = G4X_WM_LEVEL_SR;
6888                 else
6889                         max_level = G4X_WM_LEVEL_NORMAL;
6890
6891                 level = G4X_WM_LEVEL_NORMAL;
6892                 raw = &crtc_state->wm.g4x.raw[level];
6893                 for_each_plane_id_on_crtc(crtc, plane_id)
6894                         raw->plane[plane_id] = active->wm.plane[plane_id];
6895
6896                 level = G4X_WM_LEVEL_SR;
6897                 if (level > max_level)
6898                         goto out;
6899
6900                 raw = &crtc_state->wm.g4x.raw[level];
6901                 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6902                 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6903                 raw->plane[PLANE_SPRITE0] = 0;
6904                 raw->fbc = active->sr.fbc;
6905
6906                 level = G4X_WM_LEVEL_HPLL;
6907                 if (level > max_level)
6908                         goto out;
6909
6910                 raw = &crtc_state->wm.g4x.raw[level];
6911                 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6912                 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6913                 raw->plane[PLANE_SPRITE0] = 0;
6914                 raw->fbc = active->hpll.fbc;
6915
6916                 level++;
6917         out:
6918                 for_each_plane_id_on_crtc(crtc, plane_id)
6919                         g4x_raw_plane_wm_set(crtc_state, level,
6920                                              plane_id, USHRT_MAX);
6921                 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6922
6923                 crtc_state->wm.g4x.optimal = *active;
6924                 crtc_state->wm.g4x.intermediate = *active;
6925
6926                 drm_dbg_kms(&dev_priv->drm,
6927                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6928                             pipe_name(pipe),
6929                             wm->pipe[pipe].plane[PLANE_PRIMARY],
6930                             wm->pipe[pipe].plane[PLANE_CURSOR],
6931                             wm->pipe[pipe].plane[PLANE_SPRITE0]);
6932         }
6933
6934         drm_dbg_kms(&dev_priv->drm,
6935                     "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6936                     wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6937         drm_dbg_kms(&dev_priv->drm,
6938                     "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6939                     wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6940         drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6941                     str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
6942                     str_yes_no(wm->fbc_en));
6943 }
6944
6945 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6946 {
6947         struct intel_plane *plane;
6948         struct intel_crtc *crtc;
6949
6950         mutex_lock(&dev_priv->display.wm.wm_mutex);
6951
6952         for_each_intel_plane(&dev_priv->drm, plane) {
6953                 struct intel_crtc *crtc =
6954                         intel_crtc_for_pipe(dev_priv, plane->pipe);
6955                 struct intel_crtc_state *crtc_state =
6956                         to_intel_crtc_state(crtc->base.state);
6957                 struct intel_plane_state *plane_state =
6958                         to_intel_plane_state(plane->base.state);
6959                 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6960                 enum plane_id plane_id = plane->id;
6961                 int level;
6962
6963                 if (plane_state->uapi.visible)
6964                         continue;
6965
6966                 for (level = 0; level < 3; level++) {
6967                         struct g4x_pipe_wm *raw =
6968                                 &crtc_state->wm.g4x.raw[level];
6969
6970                         raw->plane[plane_id] = 0;
6971                         wm_state->wm.plane[plane_id] = 0;
6972                 }
6973
6974                 if (plane_id == PLANE_PRIMARY) {
6975                         for (level = 0; level < 3; level++) {
6976                                 struct g4x_pipe_wm *raw =
6977                                         &crtc_state->wm.g4x.raw[level];
6978                                 raw->fbc = 0;
6979                         }
6980
6981                         wm_state->sr.fbc = 0;
6982                         wm_state->hpll.fbc = 0;
6983                         wm_state->fbc_en = false;
6984                 }
6985         }
6986
6987         for_each_intel_crtc(&dev_priv->drm, crtc) {
6988                 struct intel_crtc_state *crtc_state =
6989                         to_intel_crtc_state(crtc->base.state);
6990
6991                 crtc_state->wm.g4x.intermediate =
6992                         crtc_state->wm.g4x.optimal;
6993                 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6994         }
6995
6996         g4x_program_watermarks(dev_priv);
6997
6998         mutex_unlock(&dev_priv->display.wm.wm_mutex);
6999 }
7000
7001 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
7002 {
7003         struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;
7004         struct intel_crtc *crtc;
7005         u32 val;
7006
7007         vlv_read_wm_values(dev_priv, wm);
7008
7009         wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
7010         wm->level = VLV_WM_LEVEL_PM2;
7011
7012         if (IS_CHERRYVIEW(dev_priv)) {
7013                 vlv_punit_get(dev_priv);
7014
7015                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
7016                 if (val & DSP_MAXFIFO_PM5_ENABLE)
7017                         wm->level = VLV_WM_LEVEL_PM5;
7018
7019                 /*
7020                  * If DDR DVFS is disabled in the BIOS, Punit
7021                  * will never ack the request. So if that happens
7022                  * assume we don't have to enable/disable DDR DVFS
7023                  * dynamically. To test that just set the REQ_ACK
7024                  * bit to poke the Punit, but don't change the
7025                  * HIGH/LOW bits so that we don't actually change
7026                  * the current state.
7027                  */
7028                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
7029                 val |= FORCE_DDR_FREQ_REQ_ACK;
7030                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
7031
7032                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
7033                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
7034                         drm_dbg_kms(&dev_priv->drm,
7035                                     "Punit not acking DDR DVFS request, "
7036                                     "assuming DDR DVFS is disabled\n");
7037                         dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM5;
7038                 } else {
7039                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
7040                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
7041                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
7042                 }
7043
7044                 vlv_punit_put(dev_priv);
7045         }
7046
7047         for_each_intel_crtc(&dev_priv->drm, crtc) {
7048                 struct intel_crtc_state *crtc_state =
7049                         to_intel_crtc_state(crtc->base.state);
7050                 struct vlv_wm_state *active = &crtc->wm.active.vlv;
7051                 const struct vlv_fifo_state *fifo_state =
7052                         &crtc_state->wm.vlv.fifo_state;
7053                 enum pipe pipe = crtc->pipe;
7054                 enum plane_id plane_id;
7055                 int level;
7056
7057                 vlv_get_fifo_size(crtc_state);
7058
7059                 active->num_levels = wm->level + 1;
7060                 active->cxsr = wm->cxsr;
7061
7062                 for (level = 0; level < active->num_levels; level++) {
7063                         struct g4x_pipe_wm *raw =
7064                                 &crtc_state->wm.vlv.raw[level];
7065
7066                         active->sr[level].plane = wm->sr.plane;
7067                         active->sr[level].cursor = wm->sr.cursor;
7068
7069                         for_each_plane_id_on_crtc(crtc, plane_id) {
7070                                 active->wm[level].plane[plane_id] =
7071                                         wm->pipe[pipe].plane[plane_id];
7072
7073                                 raw->plane[plane_id] =
7074                                         vlv_invert_wm_value(active->wm[level].plane[plane_id],
7075                                                             fifo_state->plane[plane_id]);
7076                         }
7077                 }
7078
7079                 for_each_plane_id_on_crtc(crtc, plane_id)
7080                         vlv_raw_plane_wm_set(crtc_state, level,
7081                                              plane_id, USHRT_MAX);
7082                 vlv_invalidate_wms(crtc, active, level);
7083
7084                 crtc_state->wm.vlv.optimal = *active;
7085                 crtc_state->wm.vlv.intermediate = *active;
7086
7087                 drm_dbg_kms(&dev_priv->drm,
7088                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
7089                             pipe_name(pipe),
7090                             wm->pipe[pipe].plane[PLANE_PRIMARY],
7091                             wm->pipe[pipe].plane[PLANE_CURSOR],
7092                             wm->pipe[pipe].plane[PLANE_SPRITE0],
7093                             wm->pipe[pipe].plane[PLANE_SPRITE1]);
7094         }
7095
7096         drm_dbg_kms(&dev_priv->drm,
7097                     "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
7098                     wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
7099 }
7100
7101 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
7102 {
7103         struct intel_plane *plane;
7104         struct intel_crtc *crtc;
7105
7106         mutex_lock(&dev_priv->display.wm.wm_mutex);
7107
7108         for_each_intel_plane(&dev_priv->drm, plane) {
7109                 struct intel_crtc *crtc =
7110                         intel_crtc_for_pipe(dev_priv, plane->pipe);
7111                 struct intel_crtc_state *crtc_state =
7112                         to_intel_crtc_state(crtc->base.state);
7113                 struct intel_plane_state *plane_state =
7114                         to_intel_plane_state(plane->base.state);
7115                 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
7116                 const struct vlv_fifo_state *fifo_state =
7117                         &crtc_state->wm.vlv.fifo_state;
7118                 enum plane_id plane_id = plane->id;
7119                 int level;
7120
7121                 if (plane_state->uapi.visible)
7122                         continue;
7123
7124                 for (level = 0; level < wm_state->num_levels; level++) {
7125                         struct g4x_pipe_wm *raw =
7126                                 &crtc_state->wm.vlv.raw[level];
7127
7128                         raw->plane[plane_id] = 0;
7129
7130                         wm_state->wm[level].plane[plane_id] =
7131                                 vlv_invert_wm_value(raw->plane[plane_id],
7132                                                     fifo_state->plane[plane_id]);
7133                 }
7134         }
7135
7136         for_each_intel_crtc(&dev_priv->drm, crtc) {
7137                 struct intel_crtc_state *crtc_state =
7138                         to_intel_crtc_state(crtc->base.state);
7139
7140                 crtc_state->wm.vlv.intermediate =
7141                         crtc_state->wm.vlv.optimal;
7142                 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
7143         }
7144
7145         vlv_program_watermarks(dev_priv);
7146
7147         mutex_unlock(&dev_priv->display.wm.wm_mutex);
7148 }
7149
7150 /*
7151  * FIXME should probably kill this and improve
7152  * the real watermark readout/sanitation instead
7153  */
7154 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
7155 {
7156         intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM_LP_ENABLE);
7157         intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM_LP_ENABLE);
7158         intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM_LP_ENABLE);
7159
7160         /*
7161          * Don't touch WM_LP_SPRITE_ENABLE here.
7162          * Doing so could cause underruns.
7163          */
7164 }
7165
7166 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
7167 {
7168         struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
7169         struct intel_crtc *crtc;
7170
7171         ilk_init_lp_watermarks(dev_priv);
7172
7173         for_each_intel_crtc(&dev_priv->drm, crtc)
7174                 ilk_pipe_wm_get_hw_state(crtc);
7175
7176         hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
7177         hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
7178         hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
7179
7180         hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
7181         if (DISPLAY_VER(dev_priv) >= 7) {
7182                 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
7183                 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
7184         }
7185
7186         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
7187                 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
7188                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7189         else if (IS_IVYBRIDGE(dev_priv))
7190                 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
7191                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7192
7193         hw->enable_fbc_wm =
7194                 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
7195 }
7196
7197 void intel_wm_state_verify(struct intel_crtc *crtc,
7198                            struct intel_crtc_state *new_crtc_state)
7199 {
7200         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7201         struct skl_hw_state {
7202                 struct skl_ddb_entry ddb[I915_MAX_PLANES];
7203                 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
7204                 struct skl_pipe_wm wm;
7205         } *hw;
7206         const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
7207         int level, max_level = ilk_wm_max_level(dev_priv);
7208         struct intel_plane *plane;
7209         u8 hw_enabled_slices;
7210
7211         if (DISPLAY_VER(dev_priv) < 9 || !new_crtc_state->hw.active)
7212                 return;
7213
7214         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
7215         if (!hw)
7216                 return;
7217
7218         skl_pipe_wm_get_hw_state(crtc, &hw->wm);
7219
7220         skl_pipe_ddb_get_hw_state(crtc, hw->ddb, hw->ddb_y);
7221
7222         hw_enabled_slices = intel_enabled_dbuf_slices_mask(dev_priv);
7223
7224         if (DISPLAY_VER(dev_priv) >= 11 &&
7225             hw_enabled_slices != dev_priv->dbuf.enabled_slices)
7226                 drm_err(&dev_priv->drm,
7227                         "mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
7228                         dev_priv->dbuf.enabled_slices,
7229                         hw_enabled_slices);
7230
7231         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
7232                 const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
7233                 const struct skl_wm_level *hw_wm_level, *sw_wm_level;
7234
7235                 /* Watermarks */
7236                 for (level = 0; level <= max_level; level++) {
7237                         hw_wm_level = &hw->wm.planes[plane->id].wm[level];
7238                         sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level);
7239
7240                         if (skl_wm_level_equals(hw_wm_level, sw_wm_level))
7241                                 continue;
7242
7243                         drm_err(&dev_priv->drm,
7244                                 "[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
7245                                 plane->base.base.id, plane->base.name, level,
7246                                 sw_wm_level->enable,
7247                                 sw_wm_level->blocks,
7248                                 sw_wm_level->lines,
7249                                 hw_wm_level->enable,
7250                                 hw_wm_level->blocks,
7251                                 hw_wm_level->lines);
7252                 }
7253
7254                 hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
7255                 sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id);
7256
7257                 if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
7258                         drm_err(&dev_priv->drm,
7259                                 "[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
7260                                 plane->base.base.id, plane->base.name,
7261                                 sw_wm_level->enable,
7262                                 sw_wm_level->blocks,
7263                                 sw_wm_level->lines,
7264                                 hw_wm_level->enable,
7265                                 hw_wm_level->blocks,
7266                                 hw_wm_level->lines);
7267                 }
7268
7269                 hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
7270                 sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
7271
7272                 if (HAS_HW_SAGV_WM(dev_priv) &&
7273                     !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
7274                         drm_err(&dev_priv->drm,
7275                                 "[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
7276                                 plane->base.base.id, plane->base.name,
7277                                 sw_wm_level->enable,
7278                                 sw_wm_level->blocks,
7279                                 sw_wm_level->lines,
7280                                 hw_wm_level->enable,
7281                                 hw_wm_level->blocks,
7282                                 hw_wm_level->lines);
7283                 }
7284
7285                 hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
7286                 sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
7287
7288                 if (HAS_HW_SAGV_WM(dev_priv) &&
7289                     !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
7290                         drm_err(&dev_priv->drm,
7291                                 "[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
7292                                 plane->base.base.id, plane->base.name,
7293                                 sw_wm_level->enable,
7294                                 sw_wm_level->blocks,
7295                                 sw_wm_level->lines,
7296                                 hw_wm_level->enable,
7297                                 hw_wm_level->blocks,
7298                                 hw_wm_level->lines);
7299                 }
7300
7301                 /* DDB */
7302                 hw_ddb_entry = &hw->ddb[PLANE_CURSOR];
7303                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR];
7304
7305                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
7306                         drm_err(&dev_priv->drm,
7307                                 "[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
7308                                 plane->base.base.id, plane->base.name,
7309                                 sw_ddb_entry->start, sw_ddb_entry->end,
7310                                 hw_ddb_entry->start, hw_ddb_entry->end);
7311                 }
7312         }
7313
7314         kfree(hw);
7315 }
7316
7317 void intel_enable_ipc(struct drm_i915_private *dev_priv)
7318 {
7319         u32 val;
7320
7321         if (!HAS_IPC(dev_priv))
7322                 return;
7323
7324         val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
7325
7326         if (dev_priv->ipc_enabled)
7327                 val |= DISP_IPC_ENABLE;
7328         else
7329                 val &= ~DISP_IPC_ENABLE;
7330
7331         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
7332 }
7333
7334 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
7335 {
7336         /* Display WA #0477 WaDisableIPC: skl */
7337         if (IS_SKYLAKE(dev_priv))
7338                 return false;
7339
7340         /* Display WA #1141: SKL:all KBL:all CFL */
7341         if (IS_KABYLAKE(dev_priv) ||
7342             IS_COFFEELAKE(dev_priv) ||
7343             IS_COMETLAKE(dev_priv))
7344                 return dev_priv->dram_info.symmetric_memory;
7345
7346         return true;
7347 }
7348
7349 void intel_init_ipc(struct drm_i915_private *dev_priv)
7350 {
7351         if (!HAS_IPC(dev_priv))
7352                 return;
7353
7354         dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
7355
7356         intel_enable_ipc(dev_priv);
7357 }
7358
7359 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
7360 {
7361         /*
7362          * On Ibex Peak and Cougar Point, we need to disable clock
7363          * gating for the panel power sequencer or it will fail to
7364          * start up when no ports are active.
7365          */
7366         intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
7367 }
7368
7369 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
7370 {
7371         enum pipe pipe;
7372
7373         for_each_pipe(dev_priv, pipe) {
7374                 intel_uncore_write(&dev_priv->uncore, DSPCNTR(pipe),
7375                            intel_uncore_read(&dev_priv->uncore, DSPCNTR(pipe)) |
7376                            DISP_TRICKLE_FEED_DISABLE);
7377
7378                 intel_uncore_write(&dev_priv->uncore, DSPSURF(pipe), intel_uncore_read(&dev_priv->uncore, DSPSURF(pipe)));
7379                 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
7380         }
7381 }
7382
7383 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
7384 {
7385         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7386
7387         /*
7388          * Required for FBC
7389          * WaFbcDisableDpfcClockGating:ilk
7390          */
7391         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
7392                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
7393                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
7394
7395         intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
7396                    MARIUNIT_CLOCK_GATE_DISABLE |
7397                    SVSMUNIT_CLOCK_GATE_DISABLE);
7398         intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
7399                    VFMUNIT_CLOCK_GATE_DISABLE);
7400
7401         /*
7402          * According to the spec the following bits should be set in
7403          * order to enable memory self-refresh
7404          * The bit 22/21 of 0x42004
7405          * The bit 5 of 0x42020
7406          * The bit 15 of 0x45000
7407          */
7408         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7409                    (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7410                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
7411         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
7412         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
7413                    (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7414                     DISP_FBC_WM_DIS));
7415
7416         /*
7417          * Based on the document from hardware guys the following bits
7418          * should be set unconditionally in order to enable FBC.
7419          * The bit 22 of 0x42000
7420          * The bit 22 of 0x42004
7421          * The bit 7,8,9 of 0x42020.
7422          */
7423         if (IS_IRONLAKE_M(dev_priv)) {
7424                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
7425                 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7426                            intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7427                            ILK_FBCQ_DIS);
7428                 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7429                            intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7430                            ILK_DPARB_GATE);
7431         }
7432
7433         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7434
7435         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7436                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7437                    ILK_ELPIN_409_SELECT);
7438
7439         g4x_disable_trickle_feed(dev_priv);
7440
7441         ibx_init_clock_gating(dev_priv);
7442 }
7443
7444 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
7445 {
7446         enum pipe pipe;
7447         u32 val;
7448
7449         /*
7450          * On Ibex Peak and Cougar Point, we need to disable clock
7451          * gating for the panel power sequencer or it will fail to
7452          * start up when no ports are active.
7453          */
7454         intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
7455                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
7456                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
7457         intel_uncore_write(&dev_priv->uncore, SOUTH_CHICKEN2, intel_uncore_read(&dev_priv->uncore, SOUTH_CHICKEN2) |
7458                    DPLS_EDP_PPS_FIX_DIS);
7459         /* The below fixes the weird display corruption, a few pixels shifted
7460          * downward, on (only) LVDS of some HP laptops with IVY.
7461          */
7462         for_each_pipe(dev_priv, pipe) {
7463                 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
7464                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
7465                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7466                 if (dev_priv->vbt.fdi_rx_polarity_inverted)
7467                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7468                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
7469                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
7470                 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
7471         }
7472         /* WADP0ClockGatingDisable */
7473         for_each_pipe(dev_priv, pipe) {
7474                 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
7475                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7476         }
7477 }
7478
7479 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
7480 {
7481         u32 tmp;
7482
7483         tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
7484         if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
7485                 drm_dbg_kms(&dev_priv->drm,
7486                             "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
7487                             tmp);
7488 }
7489
7490 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
7491 {
7492         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7493
7494         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7495
7496         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7497                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7498                    ILK_ELPIN_409_SELECT);
7499
7500         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7501                    intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7502                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
7503                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7504
7505         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
7506          * gating disable must be set.  Failure to set it results in
7507          * flickering pixels due to Z write ordering failures after
7508          * some amount of runtime in the Mesa "fire" demo, and Unigine
7509          * Sanctuary and Tropics, and apparently anything else with
7510          * alpha test or pixel discard.
7511          *
7512          * According to the spec, bit 11 (RCCUNIT) must also be set,
7513          * but we didn't debug actual testcases to find it out.
7514          *
7515          * WaDisableRCCUnitClockGating:snb
7516          * WaDisableRCPBUnitClockGating:snb
7517          */
7518         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7519                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7520                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7521
7522         /*
7523          * According to the spec the following bits should be
7524          * set in order to enable memory self-refresh and fbc:
7525          * The bit21 and bit22 of 0x42000
7526          * The bit21 and bit22 of 0x42004
7527          * The bit5 and bit7 of 0x42020
7528          * The bit14 of 0x70180
7529          * The bit14 of 0x71180
7530          *
7531          * WaFbcAsynchFlipDisableFbcQueue:snb
7532          */
7533         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7534                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7535                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7536         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7537                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7538                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7539         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
7540                    intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
7541                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
7542                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7543
7544         g4x_disable_trickle_feed(dev_priv);
7545
7546         cpt_init_clock_gating(dev_priv);
7547
7548         gen6_check_mch_setup(dev_priv);
7549 }
7550
7551 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7552 {
7553         /*
7554          * TODO: this bit should only be enabled when really needed, then
7555          * disabled when not needed anymore in order to save power.
7556          */
7557         if (HAS_PCH_LPT_LP(dev_priv))
7558                 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
7559                            intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7560                            PCH_LP_PARTITION_LEVEL_DISABLE);
7561
7562         /* WADPOClockGatingDisable:hsw */
7563         intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
7564                    intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A)) |
7565                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7566 }
7567
7568 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7569 {
7570         if (HAS_PCH_LPT_LP(dev_priv)) {
7571                 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
7572
7573                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7574                 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
7575         }
7576 }
7577
7578 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7579                                    int general_prio_credits,
7580                                    int high_prio_credits)
7581 {
7582         u32 misccpctl;
7583         u32 val;
7584
7585         /* WaTempDisableDOPClkGating:bdw */
7586         misccpctl = intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL);
7587         intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7588
7589         val = intel_uncore_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7590         val &= ~L3_PRIO_CREDITS_MASK;
7591         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7592         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7593         intel_uncore_write(&dev_priv->uncore, GEN8_L3SQCREG1, val);
7594
7595         /*
7596          * Wait at least 100 clocks before re-enabling clock gating.
7597          * See the definition of L3SQCREG1 in BSpec.
7598          */
7599         intel_uncore_posting_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7600         udelay(1);
7601         intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
7602 }
7603
7604 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7605 {
7606         /* Wa_1409120013:icl,ehl */
7607         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7608                            DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7609
7610         /*Wa_14010594013:icl, ehl */
7611         intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7612                          0, ICL_DELAY_PMRSP);
7613 }
7614
7615 static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
7616 {
7617         /* Wa_1409120013:tgl,rkl,adl-s,dg1,dg2 */
7618         if (IS_TIGERLAKE(dev_priv) || IS_ROCKETLAKE(dev_priv) ||
7619             IS_ALDERLAKE_S(dev_priv) || IS_DG1(dev_priv) || IS_DG2(dev_priv))
7620                 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7621                                    DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7622
7623         /* Wa_1409825376:tgl (pre-prod)*/
7624         if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
7625                 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7626                            TGL_VRH_GATING_DIS);
7627
7628         /* Wa_14013723622:tgl,rkl,dg1,adl-s */
7629         if (DISPLAY_VER(dev_priv) == 12)
7630                 intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
7631                                  CLKREQ_POLICY_MEM_UP_OVRD, 0);
7632 }
7633
7634 static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
7635 {
7636         gen12lp_init_clock_gating(dev_priv);
7637
7638         /* Wa_22011091694:adlp */
7639         intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
7640
7641         /* Bspec/49189 Initialize Sequence */
7642         intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1, DDI_CLOCK_REG_ACCESS, 0);
7643 }
7644
7645 static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
7646 {
7647         gen12lp_init_clock_gating(dev_priv);
7648
7649         /* Wa_1409836686:dg1[a0] */
7650         if (IS_DG1_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
7651                 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7652                            DPT_GATING_DIS);
7653 }
7654
7655 static void xehpsdv_init_clock_gating(struct drm_i915_private *dev_priv)
7656 {
7657         /* Wa_22010146351:xehpsdv */
7658         if (IS_XEHPSDV_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
7659                 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
7660 }
7661
7662 static void dg2_init_clock_gating(struct drm_i915_private *i915)
7663 {
7664         /* Wa_22010954014:dg2 */
7665         intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
7666                          SGSI_SIDECLK_DIS);
7667
7668         /*
7669          * Wa_14010733611:dg2_g10
7670          * Wa_22010146351:dg2_g10
7671          */
7672         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
7673                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
7674                                  SGR_DIS | SGGI_DIS);
7675 }
7676
7677 static void pvc_init_clock_gating(struct drm_i915_private *dev_priv)
7678 {
7679         /* Wa_14012385139:pvc */
7680         if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
7681                 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
7682
7683         /* Wa_22010954014:pvc */
7684         if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
7685                 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);
7686 }
7687
7688 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7689 {
7690         if (!HAS_PCH_CNP(dev_priv))
7691                 return;
7692
7693         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7694         intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7695                    CNP_PWM_CGE_GATING_DISABLE);
7696 }
7697
7698 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7699 {
7700         cnp_init_clock_gating(dev_priv);
7701         gen9_init_clock_gating(dev_priv);
7702
7703         /* WAC6entrylatency:cfl */
7704         intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7705                    FBC_LLC_FULLY_OPEN);
7706
7707         /*
7708          * WaFbcTurnOffFbcWatermark:cfl
7709          * Display WA #0562: cfl
7710          */
7711         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7712                    DISP_FBC_WM_DIS);
7713
7714         /*
7715          * WaFbcNukeOnHostModify:cfl
7716          * Display WA #0873: cfl
7717          */
7718         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7719                            intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
7720                            DPFC_NUKE_ON_ANY_MODIFICATION);
7721 }
7722
7723 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7724 {
7725         gen9_init_clock_gating(dev_priv);
7726
7727         /* WAC6entrylatency:kbl */
7728         intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7729                    FBC_LLC_FULLY_OPEN);
7730
7731         /* WaDisableSDEUnitClockGating:kbl */
7732         if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
7733                 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7734                            GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7735
7736         /* WaDisableGamClockGating:kbl */
7737         if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
7738                 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7739                            GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7740
7741         /*
7742          * WaFbcTurnOffFbcWatermark:kbl
7743          * Display WA #0562: kbl
7744          */
7745         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7746                    DISP_FBC_WM_DIS);
7747
7748         /*
7749          * WaFbcNukeOnHostModify:kbl
7750          * Display WA #0873: kbl
7751          */
7752         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7753                            intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
7754                            DPFC_NUKE_ON_ANY_MODIFICATION);
7755 }
7756
7757 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7758 {
7759         gen9_init_clock_gating(dev_priv);
7760
7761         /* WaDisableDopClockGating:skl */
7762         intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL) &
7763                    ~GEN7_DOP_CLOCK_GATE_ENABLE);
7764
7765         /* WAC6entrylatency:skl */
7766         intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7767                    FBC_LLC_FULLY_OPEN);
7768
7769         /*
7770          * WaFbcTurnOffFbcWatermark:skl
7771          * Display WA #0562: skl
7772          */
7773         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7774                    DISP_FBC_WM_DIS);
7775
7776         /*
7777          * WaFbcNukeOnHostModify:skl
7778          * Display WA #0873: skl
7779          */
7780         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7781                            intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
7782                            DPFC_NUKE_ON_ANY_MODIFICATION);
7783
7784         /*
7785          * WaFbcHighMemBwCorruptionAvoidance:skl
7786          * Display WA #0883: skl
7787          */
7788         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7789                            intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
7790                            DPFC_DISABLE_DUMMY0);
7791 }
7792
7793 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7794 {
7795         enum pipe pipe;
7796
7797         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7798         intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7799                    intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7800                    HSW_FBCQ_DIS);
7801
7802         /* WaSwitchSolVfFArbitrationPriority:bdw */
7803         intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7804
7805         /* WaPsrDPAMaskVBlankInSRD:bdw */
7806         intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
7807                    intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7808
7809         for_each_pipe(dev_priv, pipe) {
7810                 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7811                 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7812                            intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe)) |
7813                            BDW_DPRS_MASK_VBLANK_SRD);
7814         }
7815
7816         /* WaVSRefCountFullforceMissDisable:bdw */
7817         /* WaDSRefCountFullforceMissDisable:bdw */
7818         intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7819                    intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7820                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7821
7822         intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
7823                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7824
7825         /* WaDisableSDEUnitClockGating:bdw */
7826         intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7827                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7828
7829         /* WaProgramL3SqcReg1Default:bdw */
7830         gen8_set_l3sqc_credits(dev_priv, 30, 2);
7831
7832         /* WaKVMNotificationOnConfigChange:bdw */
7833         intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR2_1, intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR2_1)
7834                    | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7835
7836         lpt_init_clock_gating(dev_priv);
7837
7838         /* WaDisableDopClockGating:bdw
7839          *
7840          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7841          * clock gating.
7842          */
7843         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7844                    intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7845 }
7846
7847 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7848 {
7849         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7850         intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7851                    intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7852                    HSW_FBCQ_DIS);
7853
7854         /* This is required by WaCatErrorRejectionIssue:hsw */
7855         intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7856                    intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7857                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7858
7859         /* WaSwitchSolVfFArbitrationPriority:hsw */
7860         intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7861
7862         lpt_init_clock_gating(dev_priv);
7863 }
7864
7865 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7866 {
7867         u32 snpcr;
7868
7869         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7870
7871         /* WaFbcAsynchFlipDisableFbcQueue:ivb */
7872         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7873                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7874                    ILK_FBCQ_DIS);
7875
7876         /* WaDisableBackToBackFlipFix:ivb */
7877         intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7878                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7879                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
7880
7881         if (IS_IVB_GT1(dev_priv))
7882                 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7883                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7884         else {
7885                 /* must write both registers */
7886                 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7887                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7888                 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
7889                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7890         }
7891
7892         /*
7893          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7894          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7895          */
7896         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7897                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7898
7899         /* This is required by WaCatErrorRejectionIssue:ivb */
7900         intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7901                         intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7902                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7903
7904         g4x_disable_trickle_feed(dev_priv);
7905
7906         snpcr = intel_uncore_read(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR);
7907         snpcr &= ~GEN6_MBC_SNPCR_MASK;
7908         snpcr |= GEN6_MBC_SNPCR_MED;
7909         intel_uncore_write(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, snpcr);
7910
7911         if (!HAS_PCH_NOP(dev_priv))
7912                 cpt_init_clock_gating(dev_priv);
7913
7914         gen6_check_mch_setup(dev_priv);
7915 }
7916
7917 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7918 {
7919         /* WaDisableBackToBackFlipFix:vlv */
7920         intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7921                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7922                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
7923
7924         /* WaDisableDopClockGating:vlv */
7925         intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7926                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7927
7928         /* This is required by WaCatErrorRejectionIssue:vlv */
7929         intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7930                    intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7931                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7932
7933         /*
7934          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7935          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7936          */
7937         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7938                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7939
7940         /* WaDisableL3Bank2xClockGate:vlv
7941          * Disabling L3 clock gating- MMIO 940c[25] = 1
7942          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7943         intel_uncore_write(&dev_priv->uncore, GEN7_UCGCTL4,
7944                    intel_uncore_read(&dev_priv->uncore, GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7945
7946         /*
7947          * WaDisableVLVClockGating_VBIIssue:vlv
7948          * Disable clock gating on th GCFG unit to prevent a delay
7949          * in the reporting of vblank events.
7950          */
7951         intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7952 }
7953
7954 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7955 {
7956         /* WaVSRefCountFullforceMissDisable:chv */
7957         /* WaDSRefCountFullforceMissDisable:chv */
7958         intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7959                    intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7960                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7961
7962         /* WaDisableSemaphoreAndSyncFlipWait:chv */
7963         intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
7964                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7965
7966         /* WaDisableCSUnitClockGating:chv */
7967         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7968                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7969
7970         /* WaDisableSDEUnitClockGating:chv */
7971         intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7972                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7973
7974         /*
7975          * WaProgramL3SqcReg1Default:chv
7976          * See gfxspecs/Related Documents/Performance Guide/
7977          * LSQC Setting Recommendations.
7978          */
7979         gen8_set_l3sqc_credits(dev_priv, 38, 2);
7980 }
7981
7982 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7983 {
7984         u32 dspclk_gate;
7985
7986         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
7987         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7988                    GS_UNIT_CLOCK_GATE_DISABLE |
7989                    CL_UNIT_CLOCK_GATE_DISABLE);
7990         intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
7991         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7992                 OVRUNIT_CLOCK_GATE_DISABLE |
7993                 OVCUNIT_CLOCK_GATE_DISABLE;
7994         if (IS_GM45(dev_priv))
7995                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7996         intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D, dspclk_gate);
7997
7998         g4x_disable_trickle_feed(dev_priv);
7999 }
8000
8001 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
8002 {
8003         struct intel_uncore *uncore = &dev_priv->uncore;
8004
8005         intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
8006         intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
8007         intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
8008         intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
8009         intel_uncore_write16(uncore, DEUC, 0);
8010         intel_uncore_write(uncore,
8011                            MI_ARB_STATE,
8012                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
8013 }
8014
8015 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
8016 {
8017         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
8018                    I965_RCC_CLOCK_GATE_DISABLE |
8019                    I965_RCPB_CLOCK_GATE_DISABLE |
8020                    I965_ISC_CLOCK_GATE_DISABLE |
8021                    I965_FBC_CLOCK_GATE_DISABLE);
8022         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
8023         intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
8024                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
8025 }
8026
8027 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
8028 {
8029         u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
8030
8031         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
8032                 DSTATE_DOT_CLOCK_GATING;
8033         intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
8034
8035         if (IS_PINEVIEW(dev_priv))
8036                 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
8037                                    _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
8038
8039         /* IIR "flip pending" means done if this bit is set */
8040         intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
8041                            _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
8042
8043         /* interrupts should cause a wake up from C3 */
8044         intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
8045
8046         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
8047         intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
8048
8049         intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
8050                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
8051 }
8052
8053 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
8054 {
8055         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
8056
8057         /* interrupts should cause a wake up from C3 */
8058         intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
8059                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
8060
8061         intel_uncore_write(&dev_priv->uncore, MEM_MODE,
8062                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
8063
8064         /*
8065          * Have FBC ignore 3D activity since we use software
8066          * render tracking, and otherwise a pure 3D workload
8067          * (even if it just renders a single frame and then does
8068          * abosultely nothing) would not allow FBC to recompress
8069          * until a 2D blit occurs.
8070          */
8071         intel_uncore_write(&dev_priv->uncore, SCPD0,
8072                    _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
8073 }
8074
8075 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
8076 {
8077         intel_uncore_write(&dev_priv->uncore, MEM_MODE,
8078                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
8079                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
8080 }
8081
8082 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
8083 {
8084         dev_priv->clock_gating_funcs->init_clock_gating(dev_priv);
8085 }
8086
8087 void intel_suspend_hw(struct drm_i915_private *dev_priv)
8088 {
8089         if (HAS_PCH_LPT(dev_priv))
8090                 lpt_suspend_hw(dev_priv);
8091 }
8092
8093 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
8094 {
8095         drm_dbg_kms(&dev_priv->drm,
8096                     "No clock gating settings or workarounds applied.\n");
8097 }
8098
8099 #define CG_FUNCS(platform)                                              \
8100 static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
8101         .init_clock_gating = platform##_init_clock_gating,              \
8102 }
8103
8104 CG_FUNCS(pvc);
8105 CG_FUNCS(dg2);
8106 CG_FUNCS(xehpsdv);
8107 CG_FUNCS(adlp);
8108 CG_FUNCS(dg1);
8109 CG_FUNCS(gen12lp);
8110 CG_FUNCS(icl);
8111 CG_FUNCS(cfl);
8112 CG_FUNCS(skl);
8113 CG_FUNCS(kbl);
8114 CG_FUNCS(bxt);
8115 CG_FUNCS(glk);
8116 CG_FUNCS(bdw);
8117 CG_FUNCS(chv);
8118 CG_FUNCS(hsw);
8119 CG_FUNCS(ivb);
8120 CG_FUNCS(vlv);
8121 CG_FUNCS(gen6);
8122 CG_FUNCS(ilk);
8123 CG_FUNCS(g4x);
8124 CG_FUNCS(i965gm);
8125 CG_FUNCS(i965g);
8126 CG_FUNCS(gen3);
8127 CG_FUNCS(i85x);
8128 CG_FUNCS(i830);
8129 CG_FUNCS(nop);
8130 #undef CG_FUNCS
8131
8132 /**
8133  * intel_init_clock_gating_hooks - setup the clock gating hooks
8134  * @dev_priv: device private
8135  *
8136  * Setup the hooks that configure which clocks of a given platform can be
8137  * gated and also apply various GT and display specific workarounds for these
8138  * platforms. Note that some GT specific workarounds are applied separately
8139  * when GPU contexts or batchbuffers start their execution.
8140  */
8141 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
8142 {
8143         if (IS_PONTEVECCHIO(dev_priv))
8144                 dev_priv->clock_gating_funcs = &pvc_clock_gating_funcs;
8145         else if (IS_DG2(dev_priv))
8146                 dev_priv->clock_gating_funcs = &dg2_clock_gating_funcs;
8147         else if (IS_XEHPSDV(dev_priv))
8148                 dev_priv->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
8149         else if (IS_ALDERLAKE_P(dev_priv))
8150                 dev_priv->clock_gating_funcs = &adlp_clock_gating_funcs;
8151         else if (IS_DG1(dev_priv))
8152                 dev_priv->clock_gating_funcs = &dg1_clock_gating_funcs;
8153         else if (GRAPHICS_VER(dev_priv) == 12)
8154                 dev_priv->clock_gating_funcs = &gen12lp_clock_gating_funcs;
8155         else if (GRAPHICS_VER(dev_priv) == 11)
8156                 dev_priv->clock_gating_funcs = &icl_clock_gating_funcs;
8157         else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
8158                 dev_priv->clock_gating_funcs = &cfl_clock_gating_funcs;
8159         else if (IS_SKYLAKE(dev_priv))
8160                 dev_priv->clock_gating_funcs = &skl_clock_gating_funcs;
8161         else if (IS_KABYLAKE(dev_priv))
8162                 dev_priv->clock_gating_funcs = &kbl_clock_gating_funcs;
8163         else if (IS_BROXTON(dev_priv))
8164                 dev_priv->clock_gating_funcs = &bxt_clock_gating_funcs;
8165         else if (IS_GEMINILAKE(dev_priv))
8166                 dev_priv->clock_gating_funcs = &glk_clock_gating_funcs;
8167         else if (IS_BROADWELL(dev_priv))
8168                 dev_priv->clock_gating_funcs = &bdw_clock_gating_funcs;
8169         else if (IS_CHERRYVIEW(dev_priv))
8170                 dev_priv->clock_gating_funcs = &chv_clock_gating_funcs;
8171         else if (IS_HASWELL(dev_priv))
8172                 dev_priv->clock_gating_funcs = &hsw_clock_gating_funcs;
8173         else if (IS_IVYBRIDGE(dev_priv))
8174                 dev_priv->clock_gating_funcs = &ivb_clock_gating_funcs;
8175         else if (IS_VALLEYVIEW(dev_priv))
8176                 dev_priv->clock_gating_funcs = &vlv_clock_gating_funcs;
8177         else if (GRAPHICS_VER(dev_priv) == 6)
8178                 dev_priv->clock_gating_funcs = &gen6_clock_gating_funcs;
8179         else if (GRAPHICS_VER(dev_priv) == 5)
8180                 dev_priv->clock_gating_funcs = &ilk_clock_gating_funcs;
8181         else if (IS_G4X(dev_priv))
8182                 dev_priv->clock_gating_funcs = &g4x_clock_gating_funcs;
8183         else if (IS_I965GM(dev_priv))
8184                 dev_priv->clock_gating_funcs = &i965gm_clock_gating_funcs;
8185         else if (IS_I965G(dev_priv))
8186                 dev_priv->clock_gating_funcs = &i965g_clock_gating_funcs;
8187         else if (GRAPHICS_VER(dev_priv) == 3)
8188                 dev_priv->clock_gating_funcs = &gen3_clock_gating_funcs;
8189         else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
8190                 dev_priv->clock_gating_funcs = &i85x_clock_gating_funcs;
8191         else if (GRAPHICS_VER(dev_priv) == 2)
8192                 dev_priv->clock_gating_funcs = &i830_clock_gating_funcs;
8193         else {
8194                 MISSING_CASE(INTEL_DEVID(dev_priv));
8195                 dev_priv->clock_gating_funcs = &nop_clock_gating_funcs;
8196         }
8197 }
8198
8199 static const struct intel_wm_funcs skl_wm_funcs = {
8200         .compute_global_watermarks = skl_compute_wm,
8201 };
8202
8203 static const struct intel_wm_funcs ilk_wm_funcs = {
8204         .compute_pipe_wm = ilk_compute_pipe_wm,
8205         .compute_intermediate_wm = ilk_compute_intermediate_wm,
8206         .initial_watermarks = ilk_initial_watermarks,
8207         .optimize_watermarks = ilk_optimize_watermarks,
8208 };
8209
8210 static const struct intel_wm_funcs vlv_wm_funcs = {
8211         .compute_pipe_wm = vlv_compute_pipe_wm,
8212         .compute_intermediate_wm = vlv_compute_intermediate_wm,
8213         .initial_watermarks = vlv_initial_watermarks,
8214         .optimize_watermarks = vlv_optimize_watermarks,
8215         .atomic_update_watermarks = vlv_atomic_update_fifo,
8216 };
8217
8218 static const struct intel_wm_funcs g4x_wm_funcs = {
8219         .compute_pipe_wm = g4x_compute_pipe_wm,
8220         .compute_intermediate_wm = g4x_compute_intermediate_wm,
8221         .initial_watermarks = g4x_initial_watermarks,
8222         .optimize_watermarks = g4x_optimize_watermarks,
8223 };
8224
8225 static const struct intel_wm_funcs pnv_wm_funcs = {
8226         .update_wm = pnv_update_wm,
8227 };
8228
8229 static const struct intel_wm_funcs i965_wm_funcs = {
8230         .update_wm = i965_update_wm,
8231 };
8232
8233 static const struct intel_wm_funcs i9xx_wm_funcs = {
8234         .update_wm = i9xx_update_wm,
8235 };
8236
8237 static const struct intel_wm_funcs i845_wm_funcs = {
8238         .update_wm = i845_update_wm,
8239 };
8240
8241 static const struct intel_wm_funcs nop_funcs = {
8242 };
8243
8244 /* Set up chip specific power management-related functions */
8245 void intel_init_pm(struct drm_i915_private *dev_priv)
8246 {
8247         /* For cxsr */
8248         if (IS_PINEVIEW(dev_priv))
8249                 pnv_get_mem_freq(dev_priv);
8250         else if (GRAPHICS_VER(dev_priv) == 5)
8251                 ilk_get_mem_freq(dev_priv);
8252
8253         intel_sagv_init(dev_priv);
8254
8255         /* For FIFO watermark updates */
8256         if (DISPLAY_VER(dev_priv) >= 9) {
8257                 skl_setup_wm_latency(dev_priv);
8258                 dev_priv->display.funcs.wm = &skl_wm_funcs;
8259         } else if (HAS_PCH_SPLIT(dev_priv)) {
8260                 ilk_setup_wm_latency(dev_priv);
8261
8262                 if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->display.wm.pri_latency[1] &&
8263                      dev_priv->display.wm.spr_latency[1] && dev_priv->display.wm.cur_latency[1]) ||
8264                     (DISPLAY_VER(dev_priv) != 5 && dev_priv->display.wm.pri_latency[0] &&
8265                      dev_priv->display.wm.spr_latency[0] && dev_priv->display.wm.cur_latency[0])) {
8266                         dev_priv->display.funcs.wm = &ilk_wm_funcs;
8267                 } else {
8268                         drm_dbg_kms(&dev_priv->drm,
8269                                     "Failed to read display plane latency. "
8270                                     "Disable CxSR\n");
8271                         dev_priv->display.funcs.wm = &nop_funcs;
8272                 }
8273         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
8274                 vlv_setup_wm_latency(dev_priv);
8275                 dev_priv->display.funcs.wm = &vlv_wm_funcs;
8276         } else if (IS_G4X(dev_priv)) {
8277                 g4x_setup_wm_latency(dev_priv);
8278                 dev_priv->display.funcs.wm = &g4x_wm_funcs;
8279         } else if (IS_PINEVIEW(dev_priv)) {
8280                 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
8281                                             dev_priv->is_ddr3,
8282                                             dev_priv->fsb_freq,
8283                                             dev_priv->mem_freq)) {
8284                         drm_info(&dev_priv->drm,
8285                                  "failed to find known CxSR latency "
8286                                  "(found ddr%s fsb freq %d, mem freq %d), "
8287                                  "disabling CxSR\n",
8288                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
8289                                  dev_priv->fsb_freq, dev_priv->mem_freq);
8290                         /* Disable CxSR and never update its watermark again */
8291                         intel_set_memory_cxsr(dev_priv, false);
8292                         dev_priv->display.funcs.wm = &nop_funcs;
8293                 } else
8294                         dev_priv->display.funcs.wm = &pnv_wm_funcs;
8295         } else if (DISPLAY_VER(dev_priv) == 4) {
8296                 dev_priv->display.funcs.wm = &i965_wm_funcs;
8297         } else if (DISPLAY_VER(dev_priv) == 3) {
8298                 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
8299         } else if (DISPLAY_VER(dev_priv) == 2) {
8300                 if (INTEL_NUM_PIPES(dev_priv) == 1)
8301                         dev_priv->display.funcs.wm = &i845_wm_funcs;
8302                 else
8303                         dev_priv->display.funcs.wm = &i9xx_wm_funcs;
8304         } else {
8305                 drm_err(&dev_priv->drm,
8306                         "unexpected fall-through in %s\n", __func__);
8307                 dev_priv->display.funcs.wm = &nop_funcs;
8308         }
8309 }
8310
8311 void intel_pm_setup(struct drm_i915_private *dev_priv)
8312 {
8313         dev_priv->runtime_pm.suspended = false;
8314         atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
8315 }
8316
8317 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
8318 {
8319         struct intel_dbuf_state *dbuf_state;
8320
8321         dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
8322         if (!dbuf_state)
8323                 return NULL;
8324
8325         return &dbuf_state->base;
8326 }
8327
8328 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
8329                                      struct intel_global_state *state)
8330 {
8331         kfree(state);
8332 }
8333
8334 static const struct intel_global_state_funcs intel_dbuf_funcs = {
8335         .atomic_duplicate_state = intel_dbuf_duplicate_state,
8336         .atomic_destroy_state = intel_dbuf_destroy_state,
8337 };
8338
8339 struct intel_dbuf_state *
8340 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
8341 {
8342         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8343         struct intel_global_state *dbuf_state;
8344
8345         dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
8346         if (IS_ERR(dbuf_state))
8347                 return ERR_CAST(dbuf_state);
8348
8349         return to_intel_dbuf_state(dbuf_state);
8350 }
8351
8352 int intel_dbuf_init(struct drm_i915_private *dev_priv)
8353 {
8354         struct intel_dbuf_state *dbuf_state;
8355
8356         dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
8357         if (!dbuf_state)
8358                 return -ENOMEM;
8359
8360         intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
8361                                      &dbuf_state->base, &intel_dbuf_funcs);
8362
8363         return 0;
8364 }
8365
8366 /*
8367  * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
8368  * update the request state of all DBUS slices.
8369  */
8370 static void update_mbus_pre_enable(struct intel_atomic_state *state)
8371 {
8372         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8373         u32 mbus_ctl, dbuf_min_tracker_val;
8374         enum dbuf_slice slice;
8375         const struct intel_dbuf_state *dbuf_state =
8376                 intel_atomic_get_new_dbuf_state(state);
8377
8378         if (!HAS_MBUS_JOINING(dev_priv))
8379                 return;
8380
8381         /*
8382          * TODO: Implement vblank synchronized MBUS joining changes.
8383          * Must be properly coordinated with dbuf reprogramming.
8384          */
8385         if (dbuf_state->joined_mbus) {
8386                 mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
8387                         MBUS_JOIN_PIPE_SELECT_NONE;
8388                 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
8389         } else {
8390                 mbus_ctl = MBUS_HASHING_MODE_2x2 |
8391                         MBUS_JOIN_PIPE_SELECT_NONE;
8392                 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
8393         }
8394
8395         intel_de_rmw(dev_priv, MBUS_CTL,
8396                      MBUS_HASHING_MODE_MASK | MBUS_JOIN |
8397                      MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
8398
8399         for_each_dbuf_slice(dev_priv, slice)
8400                 intel_de_rmw(dev_priv, DBUF_CTL_S(slice),
8401                              DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
8402                              dbuf_min_tracker_val);
8403 }
8404
8405 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
8406 {
8407         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8408         const struct intel_dbuf_state *new_dbuf_state =
8409                 intel_atomic_get_new_dbuf_state(state);
8410         const struct intel_dbuf_state *old_dbuf_state =
8411                 intel_atomic_get_old_dbuf_state(state);
8412
8413         if (!new_dbuf_state ||
8414             ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
8415             && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8416                 return;
8417
8418         WARN_ON(!new_dbuf_state->base.changed);
8419
8420         update_mbus_pre_enable(state);
8421         gen9_dbuf_slices_update(dev_priv,
8422                                 old_dbuf_state->enabled_slices |
8423                                 new_dbuf_state->enabled_slices);
8424 }
8425
8426 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
8427 {
8428         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8429         const struct intel_dbuf_state *new_dbuf_state =
8430                 intel_atomic_get_new_dbuf_state(state);
8431         const struct intel_dbuf_state *old_dbuf_state =
8432                 intel_atomic_get_old_dbuf_state(state);
8433
8434         if (!new_dbuf_state ||
8435             ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
8436             && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8437                 return;
8438
8439         WARN_ON(!new_dbuf_state->base.changed);
8440
8441         gen9_dbuf_slices_update(dev_priv,
8442                                 new_dbuf_state->enabled_slices);
8443 }
8444
8445 void intel_mbus_dbox_update(struct intel_atomic_state *state)
8446 {
8447         struct drm_i915_private *i915 = to_i915(state->base.dev);
8448         const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
8449         const struct intel_crtc_state *new_crtc_state;
8450         const struct intel_crtc *crtc;
8451         u32 val = 0;
8452         int i;
8453
8454         if (DISPLAY_VER(i915) < 11)
8455                 return;
8456
8457         new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
8458         old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
8459         if (!new_dbuf_state ||
8460             (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus &&
8461              new_dbuf_state->active_pipes == old_dbuf_state->active_pipes))
8462                 return;
8463
8464         if (DISPLAY_VER(i915) >= 12) {
8465                 val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16);
8466                 val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1);
8467                 val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN;
8468         }
8469
8470         /* Wa_22010947358:adl-p */
8471         if (IS_ALDERLAKE_P(i915))
8472                 val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(6) :
8473                                                      MBUS_DBOX_A_CREDIT(4);
8474         else
8475                 val |= MBUS_DBOX_A_CREDIT(2);
8476
8477         if (IS_ALDERLAKE_P(i915)) {
8478                 val |= MBUS_DBOX_BW_CREDIT(2);
8479                 val |= MBUS_DBOX_B_CREDIT(8);
8480         } else if (DISPLAY_VER(i915) >= 12) {
8481                 val |= MBUS_DBOX_BW_CREDIT(2);
8482                 val |= MBUS_DBOX_B_CREDIT(12);
8483         } else {
8484                 val |= MBUS_DBOX_BW_CREDIT(1);
8485                 val |= MBUS_DBOX_B_CREDIT(8);
8486         }
8487
8488         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
8489                 if (!new_crtc_state->hw.active ||
8490                     !intel_crtc_needs_modeset(new_crtc_state))
8491                         continue;
8492
8493                 intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe), val);
8494         }
8495 }