drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <[email protected]>
  25  *
  26  */
  27
  28 #include <linux/cpufreq.h>
  29 #include <linux/pm_runtime.h>
  30 #include <drm/drm_plane_helper.h>
  31 #include "i915_drv.h"
  32 #include "intel_drv.h"
  33 #include "../../../platform/x86/intel_ips.h"
  34 #include <linux/module.h>
  35 #include <drm/drm_atomic_helper.h>
  36
  37 /**
  38  * DOC: RC6
  39  *
  40  * RC6 is a special power stage which allows the GPU to enter an very
  41  * low-voltage mode when idle, using down to 0V while at this stage.  This
  42  * stage is entered automatically when the GPU is idle when RC6 support is
  43  * enabled, and as soon as new workload arises GPU wakes up automatically as well.
  44  *
  45  * There are different RC6 modes available in Intel GPU, which differentiate
  46  * among each other with the latency required to enter and leave RC6 and
  47  * voltage consumed by the GPU in different states.
  48  *
  49  * The combination of the following flags define which states GPU is allowed
  50  * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
  51  * RC6pp is deepest RC6. Their support by hardware varies according to the
  52  * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
  53  * which brings the most power savings; deeper states save more power, but
  54  * require higher latency to switch to and wake up.
  55  */
  56
  57 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
  58 {
  59         if (HAS_LLC(dev_priv)) {
  60                 /*
  61                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  62                  * Display WA #0390: skl,kbl
  63                  *
  64                  * Must match Sampler, Pixel Back End, and Media. See
  65                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  66                  */
  67                 I915_WRITE(CHICKEN_PAR1_1,
  68                            I915_READ(CHICKEN_PAR1_1) |
  69                            SKL_DE_COMPRESSED_HASH_MODE);
  70         }
  71
  72         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  73         I915_WRITE(CHICKEN_PAR1_1,
  74                    I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
  75
  76         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  77         I915_WRITE(GEN8_CHICKEN_DCPR_1,
  78                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
  79
  80         /* WaFbcTurnOffFbcWatermark:skl,bxt,kbl,cfl */
  81         /* WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl */
  82         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  83                    DISP_FBC_WM_DIS |
  84                    DISP_FBC_MEMORY_WAKE);
  85
  86         /* WaFbcHighMemBwCorruptionAvoidance:skl,bxt,kbl,cfl */
  87         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
  88                    ILK_DPFC_DISABLE_DUMMY0);
  89
  90         if (IS_SKYLAKE(dev_priv)) {
  91                 /* WaDisableDopClockGating */
  92                 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
  93                            & ~GEN7_DOP_CLOCK_GATE_ENABLE);
  94         }
  95 }
  96
  97 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
  98 {
  99         gen9_init_clock_gating(dev_priv);
 100
 101         /* WaDisableSDEUnitClockGating:bxt */
 102         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 103                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 104
 105         /*
 106          * FIXME:
 107          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
 108          */
 109         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 110                    GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
 111
 112         /*
 113          * Wa: Backlight PWM may stop in the asserted state, causing backlight
 114          * to stay fully on.
 115          */
 116         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 117                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 118 }
 119
 120 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
 121 {
 122         gen9_init_clock_gating(dev_priv);
 123
 124         /*
 125          * WaDisablePWMClockGating:glk
 126          * Backlight PWM may stop in the asserted state, causing backlight
 127          * to stay fully on.
 128          */
 129         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 130                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 131
 132         /* WaDDIIOTimeout:glk */
 133         if (IS_GLK_REVID(dev_priv, 0, GLK_REVID_A1)) {
 134                 u32 val = I915_READ(CHICKEN_MISC_2);
 135                 val &= ~(GLK_CL0_PWR_DOWN |
 136                          GLK_CL1_PWR_DOWN |
 137                          GLK_CL2_PWR_DOWN);
 138                 I915_WRITE(CHICKEN_MISC_2, val);
 139         }
 140
 141 }
 142
 143 static void i915_pineview_get_mem_freq(struct drm_i915_private *dev_priv)
 144 {
 145         u32 tmp;
 146
 147         tmp = I915_READ(CLKCFG);
 148
 149         switch (tmp & CLKCFG_FSB_MASK) {
 150         case CLKCFG_FSB_533:
 151                 dev_priv->fsb_freq = 533; /* 133*4 */
 152                 break;
 153         case CLKCFG_FSB_800:
 154                 dev_priv->fsb_freq = 800; /* 200*4 */
 155                 break;
 156         case CLKCFG_FSB_667:
 157                 dev_priv->fsb_freq =  667; /* 167*4 */
 158                 break;
 159         case CLKCFG_FSB_400:
 160                 dev_priv->fsb_freq = 400; /* 100*4 */
 161                 break;
 162         }
 163
 164         switch (tmp & CLKCFG_MEM_MASK) {
 165         case CLKCFG_MEM_533:
 166                 dev_priv->mem_freq = 533;
 167                 break;
 168         case CLKCFG_MEM_667:
 169                 dev_priv->mem_freq = 667;
 170                 break;
 171         case CLKCFG_MEM_800:
 172                 dev_priv->mem_freq = 800;
 173                 break;
 174         }
 175
 176         /* detect pineview DDR3 setting */
 177         tmp = I915_READ(CSHRDDR3CTL);
 178         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 179 }
 180
 181 static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv)
 182 {
 183         u16 ddrpll, csipll;
 184
 185         ddrpll = I915_READ16(DDRMPLL1);
 186         csipll = I915_READ16(CSIPLL0);
 187
 188         switch (ddrpll & 0xff) {
 189         case 0xc:
 190                 dev_priv->mem_freq = 800;
 191                 break;
 192         case 0x10:
 193                 dev_priv->mem_freq = 1066;
 194                 break;
 195         case 0x14:
 196                 dev_priv->mem_freq = 1333;
 197                 break;
 198         case 0x18:
 199                 dev_priv->mem_freq = 1600;
 200                 break;
 201         default:
 202                 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
 203                                  ddrpll & 0xff);
 204                 dev_priv->mem_freq = 0;
 205                 break;
 206         }
 207
 208         dev_priv->ips.r_t = dev_priv->mem_freq;
 209
 210         switch (csipll & 0x3ff) {
 211         case 0x00c:
 212                 dev_priv->fsb_freq = 3200;
 213                 break;
 214         case 0x00e:
 215                 dev_priv->fsb_freq = 3733;
 216                 break;
 217         case 0x010:
 218                 dev_priv->fsb_freq = 4266;
 219                 break;
 220         case 0x012:
 221                 dev_priv->fsb_freq = 4800;
 222                 break;
 223         case 0x014:
 224                 dev_priv->fsb_freq = 5333;
 225                 break;
 226         case 0x016:
 227                 dev_priv->fsb_freq = 5866;
 228                 break;
 229         case 0x018:
 230                 dev_priv->fsb_freq = 6400;
 231                 break;
 232         default:
 233                 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
 234                                  csipll & 0x3ff);
 235                 dev_priv->fsb_freq = 0;
 236                 break;
 237         }
 238
 239         if (dev_priv->fsb_freq == 3200) {
 240                 dev_priv->ips.c_m = 0;
 241         } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
 242                 dev_priv->ips.c_m = 1;
 243         } else {
 244                 dev_priv->ips.c_m = 2;
 245         }
 246 }
 247
 248 static const struct cxsr_latency cxsr_latency_table[] = {
 249         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 250         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 251         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 252         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 253         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 254
 255         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 256         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 257         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 258         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 259         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 260
 261         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 262         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 263         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 264         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 265         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 266
 267         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 268         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 269         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 270         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 271         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 272
 273         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 274         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 275         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 276         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 277         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 278
 279         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 280         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 281         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 282         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 283         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 284 };
 285
 286 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
 287                                                          bool is_ddr3,
 288                                                          int fsb,
 289                                                          int mem)
 290 {
 291         const struct cxsr_latency *latency;
 292         int i;
 293
 294         if (fsb == 0 || mem == 0)
 295                 return NULL;
 296
 297         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 298                 latency = &cxsr_latency_table[i];
 299                 if (is_desktop == latency->is_desktop &&
 300                     is_ddr3 == latency->is_ddr3 &&
 301                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 302                         return latency;
 303         }
 304
 305         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 306
 307         return NULL;
 308 }
 309
 310 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 311 {
 312         u32 val;
 313
 314         mutex_lock(&dev_priv->pcu_lock);
 315
 316         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 317         if (enable)
 318                 val &= ~FORCE_DDR_HIGH_FREQ;
 319         else
 320                 val |= FORCE_DDR_HIGH_FREQ;
 321         val &= ~FORCE_DDR_LOW_FREQ;
 322         val |= FORCE_DDR_FREQ_REQ_ACK;
 323         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 324
 325         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 326                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 327                 DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
 328
 329         mutex_unlock(&dev_priv->pcu_lock);
 330 }
 331
 332 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 333 {
 334         u32 val;
 335
 336         mutex_lock(&dev_priv->pcu_lock);
 337
 338         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
 339         if (enable)
 340                 val |= DSP_MAXFIFO_PM5_ENABLE;
 341         else
 342                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 343         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
 344
 345         mutex_unlock(&dev_priv->pcu_lock);
 346 }
 347
 348 #define FW_WM(value, plane) \
 349         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 350
 351 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 352 {
 353         bool was_enabled;
 354         u32 val;
 355
 356         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 357                 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 358                 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 359                 POSTING_READ(FW_BLC_SELF_VLV);
 360         } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
 361                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 362                 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 363                 POSTING_READ(FW_BLC_SELF);
 364         } else if (IS_PINEVIEW(dev_priv)) {
 365                 val = I915_READ(DSPFW3);
 366                 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 367                 if (enable)
 368                         val |= PINEVIEW_SELF_REFRESH_EN;
 369                 else
 370                         val &= ~PINEVIEW_SELF_REFRESH_EN;
 371                 I915_WRITE(DSPFW3, val);
 372                 POSTING_READ(DSPFW3);
 373         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 374                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 375                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 376                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 377                 I915_WRITE(FW_BLC_SELF, val);
 378                 POSTING_READ(FW_BLC_SELF);
 379         } else if (IS_I915GM(dev_priv)) {
 380                 /*
 381                  * FIXME can't find a bit like this for 915G, and
 382                  * and yet it does have the related watermark in
 383                  * FW_BLC_SELF. What's going on?
 384                  */
 385                 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
 386                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 387                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 388                 I915_WRITE(INSTPM, val);
 389                 POSTING_READ(INSTPM);
 390         } else {
 391                 return false;
 392         }
 393
 394         trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
 395
 396         DRM_DEBUG_KMS("memory self-refresh is %s (was %s)\n",
 397                       enableddisabled(enable),
 398                       enableddisabled(was_enabled));
 399
 400         return was_enabled;
 401 }
 402
 403 /**
 404  * intel_set_memory_cxsr - Configure CxSR state
 405  * @dev_priv: i915 device
 406  * @enable: Allow vs. disallow CxSR
 407  *
 408  * Allow or disallow the system to enter a special CxSR
 409  * (C-state self refresh) state. What typically happens in CxSR mode
 410  * is that several display FIFOs may get combined into a single larger
 411  * FIFO for a particular plane (so called max FIFO mode) to allow the
 412  * system to defer memory fetches longer, and the memory will enter
 413  * self refresh.
 414  *
 415  * Note that enabling CxSR does not guarantee that the system enter
 416  * this special mode, nor does it guarantee that the system stays
 417  * in that mode once entered. So this just allows/disallows the system
 418  * to autonomously utilize the CxSR mode. Other factors such as core
 419  * C-states will affect when/if the system actually enters/exits the
 420  * CxSR mode.
 421  *
 422  * Note that on VLV/CHV this actually only controls the max FIFO mode,
 423  * and the system is free to enter/exit memory self refresh at any time
 424  * even when the use of CxSR has been disallowed.
 425  *
 426  * While the system is actually in the CxSR/max FIFO mode, some plane
 427  * control registers will not get latched on vblank. Thus in order to
 428  * guarantee the system will respond to changes in the plane registers
 429  * we must always disallow CxSR prior to making changes to those registers.
 430  * Unfortunately the system will re-evaluate the CxSR conditions at
 431  * frame start which happens after vblank start (which is when the plane
 432  * registers would get latched), so we can't proceed with the plane update
 433  * during the same frame where we disallowed CxSR.
 434  *
 435  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 436  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 437  * the hardware w.r.t. HPLL SR when writing to plane registers.
 438  * Disallowing just CxSR is sufficient.
 439  */
 440 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 441 {
 442         bool ret;
 443
 444         mutex_lock(&dev_priv->wm.wm_mutex);
 445         ret = _intel_set_memory_cxsr(dev_priv, enable);
 446         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 447                 dev_priv->wm.vlv.cxsr = enable;
 448         else if (IS_G4X(dev_priv))
 449                 dev_priv->wm.g4x.cxsr = enable;
 450         mutex_unlock(&dev_priv->wm.wm_mutex);
 451
 452         return ret;
 453 }
 454
 455 /*
 456  * Latency for FIFO fetches is dependent on several factors:
 457  *   - memory configuration (speed, channels)
 458  *   - chipset
 459  *   - current MCH state
 460  * It can be fairly high in some situations, so here we assume a fairly
 461  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 462  * set this value too high, the FIFO will fetch frequently to stay full)
 463  * and power consumption (set it too low to save power and we might see
 464  * FIFO underruns and display "flicker").
 465  *
 466  * A value of 5us seems to be a good balance; safe for very low end
 467  * platforms but not overly aggressive on lower latency configs.
 468  */
 469 static const int pessimal_latency_ns = 5000;
 470
 471 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 472         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 473
 474 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 475 {
 476         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 477         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 478         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 479         enum pipe pipe = crtc->pipe;
 480         int sprite0_start, sprite1_start;
 481
 482         switch (pipe) {
 483                 uint32_t dsparb, dsparb2, dsparb3;
 484         case PIPE_A:
 485                 dsparb = I915_READ(DSPARB);
 486                 dsparb2 = I915_READ(DSPARB2);
 487                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 488                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 489                 break;
 490         case PIPE_B:
 491                 dsparb = I915_READ(DSPARB);
 492                 dsparb2 = I915_READ(DSPARB2);
 493                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 494                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 495                 break;
 496         case PIPE_C:
 497                 dsparb2 = I915_READ(DSPARB2);
 498                 dsparb3 = I915_READ(DSPARB3);
 499                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 500                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 501                 break;
 502         default:
 503                 MISSING_CASE(pipe);
 504                 return;
 505         }
 506
 507         fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 508         fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 509         fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 510         fifo_state->plane[PLANE_CURSOR] = 63;
 511 }
 512
 513 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
 514                               enum i9xx_plane_id i9xx_plane)
 515 {
 516         uint32_t dsparb = I915_READ(DSPARB);
 517         int size;
 518
 519         size = dsparb & 0x7f;
 520         if (i9xx_plane == PLANE_B)
 521                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 522
 523         DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
 524                       dsparb, plane_name(i9xx_plane), size);
 525
 526         return size;
 527 }
 528
 529 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
 530                               enum i9xx_plane_id i9xx_plane)
 531 {
 532         uint32_t dsparb = I915_READ(DSPARB);
 533         int size;
 534
 535         size = dsparb & 0x1ff;
 536         if (i9xx_plane == PLANE_B)
 537                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 538         size >>= 1; /* Convert to cachelines */
 539
 540         DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
 541                       dsparb, plane_name(i9xx_plane), size);
 542
 543         return size;
 544 }
 545
 546 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
 547                               enum i9xx_plane_id i9xx_plane)
 548 {
 549         uint32_t dsparb = I915_READ(DSPARB);
 550         int size;
 551
 552         size = dsparb & 0x7f;
 553         size >>= 2; /* Convert to cachelines */
 554
 555         DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
 556                       dsparb, plane_name(i9xx_plane), size);
 557
 558         return size;
 559 }
 560
 561 /* Pineview has different values for various configs */
 562 static const struct intel_watermark_params pineview_display_wm = {
 563         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 564         .max_wm = PINEVIEW_MAX_WM,
 565         .default_wm = PINEVIEW_DFT_WM,
 566         .guard_size = PINEVIEW_GUARD_WM,
 567         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 568 };
 569 static const struct intel_watermark_params pineview_display_hplloff_wm = {
 570         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 571         .max_wm = PINEVIEW_MAX_WM,
 572         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 573         .guard_size = PINEVIEW_GUARD_WM,
 574         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 575 };
 576 static const struct intel_watermark_params pineview_cursor_wm = {
 577         .fifo_size = PINEVIEW_CURSOR_FIFO,
 578         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 579         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 580         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 581         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 582 };
 583 static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
 584         .fifo_size = PINEVIEW_CURSOR_FIFO,
 585         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 586         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 587         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 588         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 589 };
 590 static const struct intel_watermark_params i965_cursor_wm_info = {
 591         .fifo_size = I965_CURSOR_FIFO,
 592         .max_wm = I965_CURSOR_MAX_WM,
 593         .default_wm = I965_CURSOR_DFT_WM,
 594         .guard_size = 2,
 595         .cacheline_size = I915_FIFO_LINE_SIZE,
 596 };
 597 static const struct intel_watermark_params i945_wm_info = {
 598         .fifo_size = I945_FIFO_SIZE,
 599         .max_wm = I915_MAX_WM,
 600         .default_wm = 1,
 601         .guard_size = 2,
 602         .cacheline_size = I915_FIFO_LINE_SIZE,
 603 };
 604 static const struct intel_watermark_params i915_wm_info = {
 605         .fifo_size = I915_FIFO_SIZE,
 606         .max_wm = I915_MAX_WM,
 607         .default_wm = 1,
 608         .guard_size = 2,
 609         .cacheline_size = I915_FIFO_LINE_SIZE,
 610 };
 611 static const struct intel_watermark_params i830_a_wm_info = {
 612         .fifo_size = I855GM_FIFO_SIZE,
 613         .max_wm = I915_MAX_WM,
 614         .default_wm = 1,
 615         .guard_size = 2,
 616         .cacheline_size = I830_FIFO_LINE_SIZE,
 617 };
 618 static const struct intel_watermark_params i830_bc_wm_info = {
 619         .fifo_size = I855GM_FIFO_SIZE,
 620         .max_wm = I915_MAX_WM/2,
 621         .default_wm = 1,
 622         .guard_size = 2,
 623         .cacheline_size = I830_FIFO_LINE_SIZE,
 624 };
 625 static const struct intel_watermark_params i845_wm_info = {
 626         .fifo_size = I830_FIFO_SIZE,
 627         .max_wm = I915_MAX_WM,
 628         .default_wm = 1,
 629         .guard_size = 2,
 630         .cacheline_size = I830_FIFO_LINE_SIZE,
 631 };
 632
 633 /**
 634  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 635  * @pixel_rate: Pipe pixel rate in kHz
 636  * @cpp: Plane bytes per pixel
 637  * @latency: Memory wakeup latency in 0.1us units
 638  *
 639  * Compute the watermark using the method 1 or "small buffer"
 640  * formula. The caller may additonally add extra cachelines
 641  * to account for TLB misses and clock crossings.
 642  *
 643  * This method is concerned with the short term drain rate
 644  * of the FIFO, ie. it does not account for blanking periods
 645  * which would effectively reduce the average drain rate across
 646  * a longer period. The name "small" refers to the fact the
 647  * FIFO is relatively small compared to the amount of data
 648  * fetched.
 649  *
 650  * The FIFO level vs. time graph might look something like:
 651  *
 652  *   |\   |\
 653  *   | \  | \
 654  * __---__---__ (- plane active, _ blanking)
 655  * -> time
 656  *
 657  * or perhaps like this:
 658  *
 659  *   |\|\  |\|\
 660  * __----__----__ (- plane active, _ blanking)
 661  * -> time
 662  *
 663  * Returns:
 664  * The watermark in bytes
 665  */
 666 static unsigned int intel_wm_method1(unsigned int pixel_rate,
 667                                      unsigned int cpp,
 668                                      unsigned int latency)
 669 {
 670         uint64_t ret;
 671
 672         ret = (uint64_t) pixel_rate * cpp * latency;
 673         ret = DIV_ROUND_UP_ULL(ret, 10000);
 674
 675         return ret;
 676 }
 677
 678 /**
 679  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 680  * @pixel_rate: Pipe pixel rate in kHz
 681  * @htotal: Pipe horizontal total
 682  * @width: Plane width in pixels
 683  * @cpp: Plane bytes per pixel
 684  * @latency: Memory wakeup latency in 0.1us units
 685  *
 686  * Compute the watermark using the method 2 or "large buffer"
 687  * formula. The caller may additonally add extra cachelines
 688  * to account for TLB misses and clock crossings.
 689  *
 690  * This method is concerned with the long term drain rate
 691  * of the FIFO, ie. it does account for blanking periods
 692  * which effectively reduce the average drain rate across
 693  * a longer period. The name "large" refers to the fact the
 694  * FIFO is relatively large compared to the amount of data
 695  * fetched.
 696  *
 697  * The FIFO level vs. time graph might look something like:
 698  *
 699  *    |\___       |\___
 700  *    |    \___   |    \___
 701  *    |        \  |        \
 702  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 703  * -> time
 704  *
 705  * Returns:
 706  * The watermark in bytes
 707  */
 708 static unsigned int intel_wm_method2(unsigned int pixel_rate,
 709                                      unsigned int htotal,
 710                                      unsigned int width,
 711                                      unsigned int cpp,
 712                                      unsigned int latency)
 713 {
 714         unsigned int ret;
 715
 716         /*
 717          * FIXME remove once all users are computing
 718          * watermarks in the correct place.
 719          */
 720         if (WARN_ON_ONCE(htotal == 0))
 721                 htotal = 1;
 722
 723         ret = (latency * pixel_rate) / (htotal * 10000);
 724         ret = (ret + 1) * width * cpp;
 725
 726         return ret;
 727 }
 728
 729 /**
 730  * intel_calculate_wm - calculate watermark level
 731  * @pixel_rate: pixel clock
 732  * @wm: chip FIFO params
 733  * @fifo_size: size of the FIFO buffer
 734  * @cpp: bytes per pixel
 735  * @latency_ns: memory latency for the platform
 736  *
 737  * Calculate the watermark level (the level at which the display plane will
 738  * start fetching from memory again).  Each chip has a different display
 739  * FIFO size and allocation, so the caller needs to figure that out and pass
 740  * in the correct intel_watermark_params structure.
 741  *
 742  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 743  * on the pixel size.  When it reaches the watermark level, it'll start
 744  * fetching FIFO line sized based chunks from memory until the FIFO fills
 745  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 746  * will occur, and a display engine hang could result.
 747  */
 748 static unsigned int intel_calculate_wm(int pixel_rate,
 749                                        const struct intel_watermark_params *wm,
 750                                        int fifo_size, int cpp,
 751                                        unsigned int latency_ns)
 752 {
 753         int entries, wm_size;
 754
 755         /*
 756          * Note: we need to make sure we don't overflow for various clock &
 757          * latency values.
 758          * clocks go from a few thousand to several hundred thousand.
 759          * latency is usually a few thousand
 760          */
 761         entries = intel_wm_method1(pixel_rate, cpp,
 762                                    latency_ns / 100);
 763         entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 764                 wm->guard_size;
 765         DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
 766
 767         wm_size = fifo_size - entries;
 768         DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
 769
 770         /* Don't promote wm_size to unsigned... */
 771         if (wm_size > wm->max_wm)
 772                 wm_size = wm->max_wm;
 773         if (wm_size <= 0)
 774                 wm_size = wm->default_wm;
 775
 776         /*
 777          * Bspec seems to indicate that the value shouldn't be lower than
 778          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 779          * Lets go for 8 which is the burst size since certain platforms
 780          * already use a hardcoded 8 (which is what the spec says should be
 781          * done).
 782          */
 783         if (wm_size <= 8)
 784                 wm_size = 8;
 785
 786         return wm_size;
 787 }
 788
 789 static bool is_disabling(int old, int new, int threshold)
 790 {
 791         return old >= threshold && new < threshold;
 792 }
 793
 794 static bool is_enabling(int old, int new, int threshold)
 795 {
 796         return old < threshold && new >= threshold;
 797 }
 798
 799 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
 800 {
 801         return dev_priv->wm.max_level + 1;
 802 }
 803
 804 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
 805                                    const struct intel_plane_state *plane_state)
 806 {
 807         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
 808
 809         /* FIXME check the 'enable' instead */
 810         if (!crtc_state->base.active)
 811                 return false;
 812
 813         /*
 814          * Treat cursor with fb as always visible since cursor updates
 815          * can happen faster than the vrefresh rate, and the current
 816          * watermark code doesn't handle that correctly. Cursor updates
 817          * which set/clear the fb or change the cursor size are going
 818          * to get throttled by intel_legacy_cursor_update() to work
 819          * around this problem with the watermark code.
 820          */
 821         if (plane->id == PLANE_CURSOR)
 822                 return plane_state->base.fb != NULL;
 823         else
 824                 return plane_state->base.visible;
 825 }
 826
 827 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 828 {
 829         struct intel_crtc *crtc, *enabled = NULL;
 830
 831         for_each_intel_crtc(&dev_priv->drm, crtc) {
 832                 if (intel_crtc_active(crtc)) {
 833                         if (enabled)
 834                                 return NULL;
 835                         enabled = crtc;
 836                 }
 837         }
 838
 839         return enabled;
 840 }
 841
 842 static void pineview_update_wm(struct intel_crtc *unused_crtc)
 843 {
 844         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
 845         struct intel_crtc *crtc;
 846         const struct cxsr_latency *latency;
 847         u32 reg;
 848         unsigned int wm;
 849
 850         latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
 851                                          dev_priv->is_ddr3,
 852                                          dev_priv->fsb_freq,
 853                                          dev_priv->mem_freq);
 854         if (!latency) {
 855                 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 856                 intel_set_memory_cxsr(dev_priv, false);
 857                 return;
 858         }
 859
 860         crtc = single_enabled_crtc(dev_priv);
 861         if (crtc) {
 862                 const struct drm_display_mode *adjusted_mode =
 863                         &crtc->config->base.adjusted_mode;
 864                 const struct drm_framebuffer *fb =
 865                         crtc->base.primary->state->fb;
 866                 int cpp = fb->format->cpp[0];
 867                 int clock = adjusted_mode->crtc_clock;
 868
 869                 /* Display SR */
 870                 wm = intel_calculate_wm(clock, &pineview_display_wm,
 871                                         pineview_display_wm.fifo_size,
 872                                         cpp, latency->display_sr);
 873                 reg = I915_READ(DSPFW1);
 874                 reg &= ~DSPFW_SR_MASK;
 875                 reg |= FW_WM(wm, SR);
 876                 I915_WRITE(DSPFW1, reg);
 877                 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
 878
 879                 /* cursor SR */
 880                 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
 881                                         pineview_display_wm.fifo_size,
 882                                         4, latency->cursor_sr);
 883                 reg = I915_READ(DSPFW3);
 884                 reg &= ~DSPFW_CURSOR_SR_MASK;
 885                 reg |= FW_WM(wm, CURSOR_SR);
 886                 I915_WRITE(DSPFW3, reg);
 887
 888                 /* Display HPLL off SR */
 889                 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
 890                                         pineview_display_hplloff_wm.fifo_size,
 891                                         cpp, latency->display_hpll_disable);
 892                 reg = I915_READ(DSPFW3);
 893                 reg &= ~DSPFW_HPLL_SR_MASK;
 894                 reg |= FW_WM(wm, HPLL_SR);
 895                 I915_WRITE(DSPFW3, reg);
 896
 897                 /* cursor HPLL off SR */
 898                 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
 899                                         pineview_display_hplloff_wm.fifo_size,
 900                                         4, latency->cursor_hpll_disable);
 901                 reg = I915_READ(DSPFW3);
 902                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 903                 reg |= FW_WM(wm, HPLL_CURSOR);
 904                 I915_WRITE(DSPFW3, reg);
 905                 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
 906
 907                 intel_set_memory_cxsr(dev_priv, true);
 908         } else {
 909                 intel_set_memory_cxsr(dev_priv, false);
 910         }
 911 }
 912
 913 /*
 914  * Documentation says:
 915  * "If the line size is small, the TLB fetches can get in the way of the
 916  *  data fetches, causing some lag in the pixel data return which is not
 917  *  accounted for in the above formulas. The following adjustment only
 918  *  needs to be applied if eight whole lines fit in the buffer at once.
 919  *  The WM is adjusted upwards by the difference between the FIFO size
 920  *  and the size of 8 whole lines. This adjustment is always performed
 921  *  in the actual pixel depth regardless of whether FBC is enabled or not."
 922  */
 923 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 924 {
 925         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 926
 927         return max(0, tlb_miss);
 928 }
 929
 930 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 931                                 const struct g4x_wm_values *wm)
 932 {
 933         enum pipe pipe;
 934
 935         for_each_pipe(dev_priv, pipe)
 936                 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 937
 938         I915_WRITE(DSPFW1,
 939                    FW_WM(wm->sr.plane, SR) |
 940                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 941                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 942                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 943         I915_WRITE(DSPFW2,
 944                    (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 945                    FW_WM(wm->sr.fbc, FBC_SR) |
 946                    FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
 947                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
 948                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 949                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 950         I915_WRITE(DSPFW3,
 951                    (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
 952                    FW_WM(wm->sr.cursor, CURSOR_SR) |
 953                    FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
 954                    FW_WM(wm->hpll.plane, HPLL_SR));
 955
 956         POSTING_READ(DSPFW1);
 957 }
 958
 959 #define FW_WM_VLV(value, plane) \
 960         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
 961
 962 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
 963                                 const struct vlv_wm_values *wm)
 964 {
 965         enum pipe pipe;
 966
 967         for_each_pipe(dev_priv, pipe) {
 968                 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 969
 970                 I915_WRITE(VLV_DDL(pipe),
 971                            (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
 972                            (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
 973                            (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
 974                            (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
 975         }
 976
 977         /*
 978          * Zero the (unused) WM1 watermarks, and also clear all the
 979          * high order bits so that there are no out of bounds values
 980          * present in the registers during the reprogramming.
 981          */
 982         I915_WRITE(DSPHOWM, 0);
 983         I915_WRITE(DSPHOWM1, 0);
 984         I915_WRITE(DSPFW4, 0);
 985         I915_WRITE(DSPFW5, 0);
 986         I915_WRITE(DSPFW6, 0);
 987
 988         I915_WRITE(DSPFW1,
 989                    FW_WM(wm->sr.plane, SR) |
 990                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 991                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 992                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 993         I915_WRITE(DSPFW2,
 994                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
 995                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 996                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 997         I915_WRITE(DSPFW3,
 998                    FW_WM(wm->sr.cursor, CURSOR_SR));
 999
1000         if (IS_CHERRYVIEW(dev_priv)) {
1001                 I915_WRITE(DSPFW7_CHV,
1002                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1003                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1004                 I915_WRITE(DSPFW8_CHV,
1005                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1006                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1007                 I915_WRITE(DSPFW9_CHV,
1008                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1009                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1010                 I915_WRITE(DSPHOWM,
1011                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1012                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1013                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1014                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1015                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1016                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1017                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1018                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1019                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1020                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1021         } else {
1022                 I915_WRITE(DSPFW7,
1023                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1024                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1025                 I915_WRITE(DSPHOWM,
1026                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1027                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1028                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1029                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1030                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1031                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1032                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1033         }
1034
1035         POSTING_READ(DSPFW1);
1036 }
1037
1038 #undef FW_WM_VLV
1039
1040 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1041 {
1042         /* all latencies in usec */
1043         dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1044         dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1045         dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1046
1047         dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1048 }
1049
1050 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1051 {
1052         /*
1053          * DSPCNTR[13] supposedly controls whether the
1054          * primary plane can use the FIFO space otherwise
1055          * reserved for the sprite plane. It's not 100% clear
1056          * what the actual FIFO size is, but it looks like we
1057          * can happily set both primary and sprite watermarks
1058          * up to 127 cachelines. So that would seem to mean
1059          * that either DSPCNTR[13] doesn't do anything, or that
1060          * the total FIFO is >= 256 cachelines in size. Either
1061          * way, we don't seem to have to worry about this
1062          * repartitioning as the maximum watermark value the
1063          * register can hold for each plane is lower than the
1064          * minimum FIFO size.
1065          */
1066         switch (plane_id) {
1067         case PLANE_CURSOR:
1068                 return 63;
1069         case PLANE_PRIMARY:
1070                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1071         case PLANE_SPRITE0:
1072                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1073         default:
1074                 MISSING_CASE(plane_id);
1075                 return 0;
1076         }
1077 }
1078
1079 static int g4x_fbc_fifo_size(int level)
1080 {
1081         switch (level) {
1082         case G4X_WM_LEVEL_SR:
1083                 return 7;
1084         case G4X_WM_LEVEL_HPLL:
1085                 return 15;
1086         default:
1087                 MISSING_CASE(level);
1088                 return 0;
1089         }
1090 }
1091
1092 static uint16_t g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1093                                const struct intel_plane_state *plane_state,
1094                                int level)
1095 {
1096         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1097         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1098         const struct drm_display_mode *adjusted_mode =
1099                 &crtc_state->base.adjusted_mode;
1100         unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1101         unsigned int clock, htotal, cpp, width, wm;
1102
1103         if (latency == 0)
1104                 return USHRT_MAX;
1105
1106         if (!intel_wm_plane_visible(crtc_state, plane_state))
1107                 return 0;
1108
1109         /*
1110          * Not 100% sure which way ELK should go here as the
1111          * spec only says CL/CTG should assume 32bpp and BW
1112          * doesn't need to. But as these things followed the
1113          * mobile vs. desktop lines on gen3 as well, let's
1114          * assume ELK doesn't need this.
1115          *
1116          * The spec also fails to list such a restriction for
1117          * the HPLL watermark, which seems a little strange.
1118          * Let's use 32bpp for the HPLL watermark as well.
1119          */
1120         if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1121             level != G4X_WM_LEVEL_NORMAL)
1122                 cpp = 4;
1123         else
1124                 cpp = plane_state->base.fb->format->cpp[0];
1125
1126         clock = adjusted_mode->crtc_clock;
1127         htotal = adjusted_mode->crtc_htotal;
1128
1129         if (plane->id == PLANE_CURSOR)
1130                 width = plane_state->base.crtc_w;
1131         else
1132                 width = drm_rect_width(&plane_state->base.dst);
1133
1134         if (plane->id == PLANE_CURSOR) {
1135                 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1136         } else if (plane->id == PLANE_PRIMARY &&
1137                    level == G4X_WM_LEVEL_NORMAL) {
1138                 wm = intel_wm_method1(clock, cpp, latency);
1139         } else {
1140                 unsigned int small, large;
1141
1142                 small = intel_wm_method1(clock, cpp, latency);
1143                 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1144
1145                 wm = min(small, large);
1146         }
1147
1148         wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1149                               width, cpp);
1150
1151         wm = DIV_ROUND_UP(wm, 64) + 2;
1152
1153         return min_t(unsigned int, wm, USHRT_MAX);
1154 }
1155
1156 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1157                                  int level, enum plane_id plane_id, u16 value)
1158 {
1159         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1160         bool dirty = false;
1161
1162         for (; level < intel_wm_num_levels(dev_priv); level++) {
1163                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1164
1165                 dirty |= raw->plane[plane_id] != value;
1166                 raw->plane[plane_id] = value;
1167         }
1168
1169         return dirty;
1170 }
1171
1172 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1173                                int level, u16 value)
1174 {
1175         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1176         bool dirty = false;
1177
1178         /* NORMAL level doesn't have an FBC watermark */
1179         level = max(level, G4X_WM_LEVEL_SR);
1180
1181         for (; level < intel_wm_num_levels(dev_priv); level++) {
1182                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1183
1184                 dirty |= raw->fbc != value;
1185                 raw->fbc = value;
1186         }
1187
1188         return dirty;
1189 }
1190
1191 static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
1192                                    const struct intel_plane_state *pstate,
1193                                    uint32_t pri_val);
1194
1195 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1196                                      const struct intel_plane_state *plane_state)
1197 {
1198         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1199         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1200         enum plane_id plane_id = plane->id;
1201         bool dirty = false;
1202         int level;
1203
1204         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1205                 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1206                 if (plane_id == PLANE_PRIMARY)
1207                         dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1208                 goto out;
1209         }
1210
1211         for (level = 0; level < num_levels; level++) {
1212                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1213                 int wm, max_wm;
1214
1215                 wm = g4x_compute_wm(crtc_state, plane_state, level);
1216                 max_wm = g4x_plane_fifo_size(plane_id, level);
1217
1218                 if (wm > max_wm)
1219                         break;
1220
1221                 dirty |= raw->plane[plane_id] != wm;
1222                 raw->plane[plane_id] = wm;
1223
1224                 if (plane_id != PLANE_PRIMARY ||
1225                     level == G4X_WM_LEVEL_NORMAL)
1226                         continue;
1227
1228                 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1229                                         raw->plane[plane_id]);
1230                 max_wm = g4x_fbc_fifo_size(level);
1231
1232                 /*
1233                  * FBC wm is not mandatory as we
1234                  * can always just disable its use.
1235                  */
1236                 if (wm > max_wm)
1237                         wm = USHRT_MAX;
1238
1239                 dirty |= raw->fbc != wm;
1240                 raw->fbc = wm;
1241         }
1242
1243         /* mark watermarks as invalid */
1244         dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1245
1246         if (plane_id == PLANE_PRIMARY)
1247                 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1248
1249  out:
1250         if (dirty) {
1251                 DRM_DEBUG_KMS("%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1252                               plane->base.name,
1253                               crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1254                               crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1255                               crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1256
1257                 if (plane_id == PLANE_PRIMARY)
1258                         DRM_DEBUG_KMS("FBC watermarks: SR=%d, HPLL=%d\n",
1259                                       crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1260                                       crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1261         }
1262
1263         return dirty;
1264 }
1265
1266 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1267                                       enum plane_id plane_id, int level)
1268 {
1269         const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1270
1271         return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1272 }
1273
1274 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1275                                      int level)
1276 {
1277         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1278
1279         if (level > dev_priv->wm.max_level)
1280                 return false;
1281
1282         return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1283                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1284                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1285 }
1286
1287 /* mark all levels starting from 'level' as invalid */
1288 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1289                                struct g4x_wm_state *wm_state, int level)
1290 {
1291         if (level <= G4X_WM_LEVEL_NORMAL) {
1292                 enum plane_id plane_id;
1293
1294                 for_each_plane_id_on_crtc(crtc, plane_id)
1295                         wm_state->wm.plane[plane_id] = USHRT_MAX;
1296         }
1297
1298         if (level <= G4X_WM_LEVEL_SR) {
1299                 wm_state->cxsr = false;
1300                 wm_state->sr.cursor = USHRT_MAX;
1301                 wm_state->sr.plane = USHRT_MAX;
1302                 wm_state->sr.fbc = USHRT_MAX;
1303         }
1304
1305         if (level <= G4X_WM_LEVEL_HPLL) {
1306                 wm_state->hpll_en = false;
1307                 wm_state->hpll.cursor = USHRT_MAX;
1308                 wm_state->hpll.plane = USHRT_MAX;
1309                 wm_state->hpll.fbc = USHRT_MAX;
1310         }
1311 }
1312
1313 static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1314 {
1315         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1316         struct intel_atomic_state *state =
1317                 to_intel_atomic_state(crtc_state->base.state);
1318         struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1319         int num_active_planes = hweight32(crtc_state->active_planes &
1320                                           ~BIT(PLANE_CURSOR));
1321         const struct g4x_pipe_wm *raw;
1322         const struct intel_plane_state *old_plane_state;
1323         const struct intel_plane_state *new_plane_state;
1324         struct intel_plane *plane;
1325         enum plane_id plane_id;
1326         int i, level;
1327         unsigned int dirty = 0;
1328
1329         for_each_oldnew_intel_plane_in_state(state, plane,
1330                                              old_plane_state,
1331                                              new_plane_state, i) {
1332                 if (new_plane_state->base.crtc != &crtc->base &&
1333                     old_plane_state->base.crtc != &crtc->base)
1334                         continue;
1335
1336                 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1337                         dirty |= BIT(plane->id);
1338         }
1339
1340         if (!dirty)
1341                 return 0;
1342
1343         level = G4X_WM_LEVEL_NORMAL;
1344         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1345                 goto out;
1346
1347         raw = &crtc_state->wm.g4x.raw[level];
1348         for_each_plane_id_on_crtc(crtc, plane_id)
1349                 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1350
1351         level = G4X_WM_LEVEL_SR;
1352
1353         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1354                 goto out;
1355
1356         raw = &crtc_state->wm.g4x.raw[level];
1357         wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1358         wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1359         wm_state->sr.fbc = raw->fbc;
1360
1361         wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1362
1363         level = G4X_WM_LEVEL_HPLL;
1364
1365         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1366                 goto out;
1367
1368         raw = &crtc_state->wm.g4x.raw[level];
1369         wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1370         wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1371         wm_state->hpll.fbc = raw->fbc;
1372
1373         wm_state->hpll_en = wm_state->cxsr;
1374
1375         level++;
1376
1377  out:
1378         if (level == G4X_WM_LEVEL_NORMAL)
1379                 return -EINVAL;
1380
1381         /* invalidate the higher levels */
1382         g4x_invalidate_wms(crtc, wm_state, level);
1383
1384         /*
1385          * Determine if the FBC watermark(s) can be used. IF
1386          * this isn't the case we prefer to disable the FBC
1387          ( watermark(s) rather than disable the SR/HPLL
1388          * level(s) entirely.
1389          */
1390         wm_state->fbc_en = level > G4X_WM_LEVEL_NORMAL;
1391
1392         if (level >= G4X_WM_LEVEL_SR &&
1393             wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1394                 wm_state->fbc_en = false;
1395         else if (level >= G4X_WM_LEVEL_HPLL &&
1396                  wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1397                 wm_state->fbc_en = false;
1398
1399         return 0;
1400 }
1401
1402 static int g4x_compute_intermediate_wm(struct drm_device *dev,
1403                                        struct intel_crtc *crtc,
1404                                        struct intel_crtc_state *new_crtc_state)
1405 {
1406         struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1407         const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1408         struct intel_atomic_state *intel_state =
1409                 to_intel_atomic_state(new_crtc_state->base.state);
1410         const struct intel_crtc_state *old_crtc_state =
1411                 intel_atomic_get_old_crtc_state(intel_state, crtc);
1412         const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1413         enum plane_id plane_id;
1414
1415         if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
1416                 *intermediate = *optimal;
1417
1418                 intermediate->cxsr = false;
1419                 intermediate->hpll_en = false;
1420                 goto out;
1421         }
1422
1423         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1424                 !new_crtc_state->disable_cxsr;
1425         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1426                 !new_crtc_state->disable_cxsr;
1427         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1428
1429         for_each_plane_id_on_crtc(crtc, plane_id) {
1430                 intermediate->wm.plane[plane_id] =
1431                         max(optimal->wm.plane[plane_id],
1432                             active->wm.plane[plane_id]);
1433
1434                 WARN_ON(intermediate->wm.plane[plane_id] >
1435                         g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1436         }
1437
1438         intermediate->sr.plane = max(optimal->sr.plane,
1439                                      active->sr.plane);
1440         intermediate->sr.cursor = max(optimal->sr.cursor,
1441                                       active->sr.cursor);
1442         intermediate->sr.fbc = max(optimal->sr.fbc,
1443                                    active->sr.fbc);
1444
1445         intermediate->hpll.plane = max(optimal->hpll.plane,
1446                                        active->hpll.plane);
1447         intermediate->hpll.cursor = max(optimal->hpll.cursor,
1448                                         active->hpll.cursor);
1449         intermediate->hpll.fbc = max(optimal->hpll.fbc,
1450                                      active->hpll.fbc);
1451
1452         WARN_ON((intermediate->sr.plane >
1453                  g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1454                  intermediate->sr.cursor >
1455                  g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1456                 intermediate->cxsr);
1457         WARN_ON((intermediate->sr.plane >
1458                  g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1459                  intermediate->sr.cursor >
1460                  g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1461                 intermediate->hpll_en);
1462
1463         WARN_ON(intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1464                 intermediate->fbc_en && intermediate->cxsr);
1465         WARN_ON(intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1466                 intermediate->fbc_en && intermediate->hpll_en);
1467
1468 out:
1469         /*
1470          * If our intermediate WM are identical to the final WM, then we can
1471          * omit the post-vblank programming; only update if it's different.
1472          */
1473         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1474                 new_crtc_state->wm.need_postvbl_update = true;
1475
1476         return 0;
1477 }
1478
1479 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1480                          struct g4x_wm_values *wm)
1481 {
1482         struct intel_crtc *crtc;
1483         int num_active_crtcs = 0;
1484
1485         wm->cxsr = true;
1486         wm->hpll_en = true;
1487         wm->fbc_en = true;
1488
1489         for_each_intel_crtc(&dev_priv->drm, crtc) {
1490                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1491
1492                 if (!crtc->active)
1493                         continue;
1494
1495                 if (!wm_state->cxsr)
1496                         wm->cxsr = false;
1497                 if (!wm_state->hpll_en)
1498                         wm->hpll_en = false;
1499                 if (!wm_state->fbc_en)
1500                         wm->fbc_en = false;
1501
1502                 num_active_crtcs++;
1503         }
1504
1505         if (num_active_crtcs != 1) {
1506                 wm->cxsr = false;
1507                 wm->hpll_en = false;
1508                 wm->fbc_en = false;
1509         }
1510
1511         for_each_intel_crtc(&dev_priv->drm, crtc) {
1512                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1513                 enum pipe pipe = crtc->pipe;
1514
1515                 wm->pipe[pipe] = wm_state->wm;
1516                 if (crtc->active && wm->cxsr)
1517                         wm->sr = wm_state->sr;
1518                 if (crtc->active && wm->hpll_en)
1519                         wm->hpll = wm_state->hpll;
1520         }
1521 }
1522
1523 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1524 {
1525         struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1526         struct g4x_wm_values new_wm = {};
1527
1528         g4x_merge_wm(dev_priv, &new_wm);
1529
1530         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1531                 return;
1532
1533         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1534                 _intel_set_memory_cxsr(dev_priv, false);
1535
1536         g4x_write_wm_values(dev_priv, &new_wm);
1537
1538         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1539                 _intel_set_memory_cxsr(dev_priv, true);
1540
1541         *old_wm = new_wm;
1542 }
1543
1544 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1545                                    struct intel_crtc_state *crtc_state)
1546 {
1547         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1548         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1549
1550         mutex_lock(&dev_priv->wm.wm_mutex);
1551         crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1552         g4x_program_watermarks(dev_priv);
1553         mutex_unlock(&dev_priv->wm.wm_mutex);
1554 }
1555
1556 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1557                                     struct intel_crtc_state *crtc_state)
1558 {
1559         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1560         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
1561
1562         if (!crtc_state->wm.need_postvbl_update)
1563                 return;
1564
1565         mutex_lock(&dev_priv->wm.wm_mutex);
1566         intel_crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1567         g4x_program_watermarks(dev_priv);
1568         mutex_unlock(&dev_priv->wm.wm_mutex);
1569 }
1570
1571 /* latency must be in 0.1us units. */
1572 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1573                                    unsigned int htotal,
1574                                    unsigned int width,
1575                                    unsigned int cpp,
1576                                    unsigned int latency)
1577 {
1578         unsigned int ret;
1579
1580         ret = intel_wm_method2(pixel_rate, htotal,
1581                                width, cpp, latency);
1582         ret = DIV_ROUND_UP(ret, 64);
1583
1584         return ret;
1585 }
1586
1587 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1588 {
1589         /* all latencies in usec */
1590         dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1591
1592         dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1593
1594         if (IS_CHERRYVIEW(dev_priv)) {
1595                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1596                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1597
1598                 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1599         }
1600 }
1601
1602 static uint16_t vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1603                                      const struct intel_plane_state *plane_state,
1604                                      int level)
1605 {
1606         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1607         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1608         const struct drm_display_mode *adjusted_mode =
1609                 &crtc_state->base.adjusted_mode;
1610         unsigned int clock, htotal, cpp, width, wm;
1611
1612         if (dev_priv->wm.pri_latency[level] == 0)
1613                 return USHRT_MAX;
1614
1615         if (!intel_wm_plane_visible(crtc_state, plane_state))
1616                 return 0;
1617
1618         cpp = plane_state->base.fb->format->cpp[0];
1619         clock = adjusted_mode->crtc_clock;
1620         htotal = adjusted_mode->crtc_htotal;
1621         width = crtc_state->pipe_src_w;
1622
1623         if (plane->id == PLANE_CURSOR) {
1624                 /*
1625                  * FIXME the formula gives values that are
1626                  * too big for the cursor FIFO, and hence we
1627                  * would never be able to use cursors. For
1628                  * now just hardcode the watermark.
1629                  */
1630                 wm = 63;
1631         } else {
1632                 wm = vlv_wm_method2(clock, htotal, width, cpp,
1633                                     dev_priv->wm.pri_latency[level] * 10);
1634         }
1635
1636         return min_t(unsigned int, wm, USHRT_MAX);
1637 }
1638
1639 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1640 {
1641         return (active_planes & (BIT(PLANE_SPRITE0) |
1642                                  BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1643 }
1644
1645 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1646 {
1647         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1648         const struct g4x_pipe_wm *raw =
1649                 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1650         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1651         unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1652         int num_active_planes = hweight32(active_planes);
1653         const int fifo_size = 511;
1654         int fifo_extra, fifo_left = fifo_size;
1655         int sprite0_fifo_extra = 0;
1656         unsigned int total_rate;
1657         enum plane_id plane_id;
1658
1659         /*
1660          * When enabling sprite0 after sprite1 has already been enabled
1661          * we tend to get an underrun unless sprite0 already has some
1662          * FIFO space allcoated. Hence we always allocate at least one
1663          * cacheline for sprite0 whenever sprite1 is enabled.
1664          *
1665          * All other plane enable sequences appear immune to this problem.
1666          */
1667         if (vlv_need_sprite0_fifo_workaround(active_planes))
1668                 sprite0_fifo_extra = 1;
1669
1670         total_rate = raw->plane[PLANE_PRIMARY] +
1671                 raw->plane[PLANE_SPRITE0] +
1672                 raw->plane[PLANE_SPRITE1] +
1673                 sprite0_fifo_extra;
1674
1675         if (total_rate > fifo_size)
1676                 return -EINVAL;
1677
1678         if (total_rate == 0)
1679                 total_rate = 1;
1680
1681         for_each_plane_id_on_crtc(crtc, plane_id) {
1682                 unsigned int rate;
1683
1684                 if ((active_planes & BIT(plane_id)) == 0) {
1685                         fifo_state->plane[plane_id] = 0;
1686                         continue;
1687                 }
1688
1689                 rate = raw->plane[plane_id];
1690                 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1691                 fifo_left -= fifo_state->plane[plane_id];
1692         }
1693
1694         fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1695         fifo_left -= sprite0_fifo_extra;
1696
1697         fifo_state->plane[PLANE_CURSOR] = 63;
1698
1699         fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1700
1701         /* spread the remainder evenly */
1702         for_each_plane_id_on_crtc(crtc, plane_id) {
1703                 int plane_extra;
1704
1705                 if (fifo_left == 0)
1706                         break;
1707
1708                 if ((active_planes & BIT(plane_id)) == 0)
1709                         continue;
1710
1711                 plane_extra = min(fifo_extra, fifo_left);
1712                 fifo_state->plane[plane_id] += plane_extra;
1713                 fifo_left -= plane_extra;
1714         }
1715
1716         WARN_ON(active_planes != 0 && fifo_left != 0);
1717
1718         /* give it all to the first plane if none are active */
1719         if (active_planes == 0) {
1720                 WARN_ON(fifo_left != fifo_size);
1721                 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1722         }
1723
1724         return 0;
1725 }
1726
1727 /* mark all levels starting from 'level' as invalid */
1728 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1729                                struct vlv_wm_state *wm_state, int level)
1730 {
1731         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1732
1733         for (; level < intel_wm_num_levels(dev_priv); level++) {
1734                 enum plane_id plane_id;
1735
1736                 for_each_plane_id_on_crtc(crtc, plane_id)
1737                         wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1738
1739                 wm_state->sr[level].cursor = USHRT_MAX;
1740                 wm_state->sr[level].plane = USHRT_MAX;
1741         }
1742 }
1743
1744 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1745 {
1746         if (wm > fifo_size)
1747                 return USHRT_MAX;
1748         else
1749                 return fifo_size - wm;
1750 }
1751
1752 /*
1753  * Starting from 'level' set all higher
1754  * levels to 'value' in the "raw" watermarks.
1755  */
1756 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1757                                  int level, enum plane_id plane_id, u16 value)
1758 {
1759         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1760         int num_levels = intel_wm_num_levels(dev_priv);
1761         bool dirty = false;
1762
1763         for (; level < num_levels; level++) {
1764                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1765
1766                 dirty |= raw->plane[plane_id] != value;
1767                 raw->plane[plane_id] = value;
1768         }
1769
1770         return dirty;
1771 }
1772
1773 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1774                                      const struct intel_plane_state *plane_state)
1775 {
1776         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1777         enum plane_id plane_id = plane->id;
1778         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1779         int level;
1780         bool dirty = false;
1781
1782         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1783                 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1784                 goto out;
1785         }
1786
1787         for (level = 0; level < num_levels; level++) {
1788                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1789                 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1790                 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1791
1792                 if (wm > max_wm)
1793                         break;
1794
1795                 dirty |= raw->plane[plane_id] != wm;
1796                 raw->plane[plane_id] = wm;
1797         }
1798
1799         /* mark all higher levels as invalid */
1800         dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1801
1802 out:
1803         if (dirty)
1804                 DRM_DEBUG_KMS("%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1805                               plane->base.name,
1806                               crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1807                               crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1808                               crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1809
1810         return dirty;
1811 }
1812
1813 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1814                                       enum plane_id plane_id, int level)
1815 {
1816         const struct g4x_pipe_wm *raw =
1817                 &crtc_state->wm.vlv.raw[level];
1818         const struct vlv_fifo_state *fifo_state =
1819                 &crtc_state->wm.vlv.fifo_state;
1820
1821         return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1822 }
1823
1824 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1825 {
1826         return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1827                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1828                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1829                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1830 }
1831
1832 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1833 {
1834         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1835         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1836         struct intel_atomic_state *state =
1837                 to_intel_atomic_state(crtc_state->base.state);
1838         struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1839         const struct vlv_fifo_state *fifo_state =
1840                 &crtc_state->wm.vlv.fifo_state;
1841         int num_active_planes = hweight32(crtc_state->active_planes &
1842                                           ~BIT(PLANE_CURSOR));
1843         bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->base);
1844         const struct intel_plane_state *old_plane_state;
1845         const struct intel_plane_state *new_plane_state;
1846         struct intel_plane *plane;
1847         enum plane_id plane_id;
1848         int level, ret, i;
1849         unsigned int dirty = 0;
1850
1851         for_each_oldnew_intel_plane_in_state(state, plane,
1852                                              old_plane_state,
1853                                              new_plane_state, i) {
1854                 if (new_plane_state->base.crtc != &crtc->base &&
1855                     old_plane_state->base.crtc != &crtc->base)
1856                         continue;
1857
1858                 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1859                         dirty |= BIT(plane->id);
1860         }
1861
1862         /*
1863          * DSPARB registers may have been reset due to the
1864          * power well being turned off. Make sure we restore
1865          * them to a consistent state even if no primary/sprite
1866          * planes are initially active.
1867          */
1868         if (needs_modeset)
1869                 crtc_state->fifo_changed = true;
1870
1871         if (!dirty)
1872                 return 0;
1873
1874         /* cursor changes don't warrant a FIFO recompute */
1875         if (dirty & ~BIT(PLANE_CURSOR)) {
1876                 const struct intel_crtc_state *old_crtc_state =
1877                         intel_atomic_get_old_crtc_state(state, crtc);
1878                 const struct vlv_fifo_state *old_fifo_state =
1879                         &old_crtc_state->wm.vlv.fifo_state;
1880
1881                 ret = vlv_compute_fifo(crtc_state);
1882                 if (ret)
1883                         return ret;
1884
1885                 if (needs_modeset ||
1886                     memcmp(old_fifo_state, fifo_state,
1887                            sizeof(*fifo_state)) != 0)
1888                         crtc_state->fifo_changed = true;
1889         }
1890
1891         /* initially allow all levels */
1892         wm_state->num_levels = intel_wm_num_levels(dev_priv);
1893         /*
1894          * Note that enabling cxsr with no primary/sprite planes
1895          * enabled can wedge the pipe. Hence we only allow cxsr
1896          * with exactly one enabled primary/sprite plane.
1897          */
1898         wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1899
1900         for (level = 0; level < wm_state->num_levels; level++) {
1901                 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1902                 const int sr_fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1;
1903
1904                 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1905                         break;
1906
1907                 for_each_plane_id_on_crtc(crtc, plane_id) {
1908                         wm_state->wm[level].plane[plane_id] =
1909                                 vlv_invert_wm_value(raw->plane[plane_id],
1910                                                     fifo_state->plane[plane_id]);
1911                 }
1912
1913                 wm_state->sr[level].plane =
1914                         vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1915                                                  raw->plane[PLANE_SPRITE0],
1916                                                  raw->plane[PLANE_SPRITE1]),
1917                                             sr_fifo_size);
1918
1919                 wm_state->sr[level].cursor =
1920                         vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1921                                             63);
1922         }
1923
1924         if (level == 0)
1925                 return -EINVAL;
1926
1927         /* limit to only levels we can actually handle */
1928         wm_state->num_levels = level;
1929
1930         /* invalidate the higher levels */
1931         vlv_invalidate_wms(crtc, wm_state, level);
1932
1933         return 0;
1934 }
1935
1936 #define VLV_FIFO(plane, value) \
1937         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1938
1939 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1940                                    struct intel_crtc_state *crtc_state)
1941 {
1942         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1943         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1944         const struct vlv_fifo_state *fifo_state =
1945                 &crtc_state->wm.vlv.fifo_state;
1946         int sprite0_start, sprite1_start, fifo_size;
1947
1948         if (!crtc_state->fifo_changed)
1949                 return;
1950
1951         sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1952         sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1953         fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1954
1955         WARN_ON(fifo_state->plane[PLANE_CURSOR] != 63);
1956         WARN_ON(fifo_size != 511);
1957
1958         trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1959
1960         /*
1961          * uncore.lock serves a double purpose here. It allows us to
1962          * use the less expensive I915_{READ,WRITE}_FW() functions, and
1963          * it protects the DSPARB registers from getting clobbered by
1964          * parallel updates from multiple pipes.
1965          *
1966          * intel_pipe_update_start() has already disabled interrupts
1967          * for us, so a plain spin_lock() is sufficient here.
1968          */
1969         spin_lock(&dev_priv->uncore.lock);
1970
1971         switch (crtc->pipe) {
1972                 uint32_t dsparb, dsparb2, dsparb3;
1973         case PIPE_A:
1974                 dsparb = I915_READ_FW(DSPARB);
1975                 dsparb2 = I915_READ_FW(DSPARB2);
1976
1977                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1978                             VLV_FIFO(SPRITEB, 0xff));
1979                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1980                            VLV_FIFO(SPRITEB, sprite1_start));
1981
1982                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1983                              VLV_FIFO(SPRITEB_HI, 0x1));
1984                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1985                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1986
1987                 I915_WRITE_FW(DSPARB, dsparb);
1988                 I915_WRITE_FW(DSPARB2, dsparb2);
1989                 break;
1990         case PIPE_B:
1991                 dsparb = I915_READ_FW(DSPARB);
1992                 dsparb2 = I915_READ_FW(DSPARB2);
1993
1994                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1995                             VLV_FIFO(SPRITED, 0xff));
1996                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1997                            VLV_FIFO(SPRITED, sprite1_start));
1998
1999                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2000                              VLV_FIFO(SPRITED_HI, 0xff));
2001                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2002                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2003
2004                 I915_WRITE_FW(DSPARB, dsparb);
2005                 I915_WRITE_FW(DSPARB2, dsparb2);
2006                 break;
2007         case PIPE_C:
2008                 dsparb3 = I915_READ_FW(DSPARB3);
2009                 dsparb2 = I915_READ_FW(DSPARB2);
2010
2011                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2012                              VLV_FIFO(SPRITEF, 0xff));
2013                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2014                             VLV_FIFO(SPRITEF, sprite1_start));
2015
2016                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2017                              VLV_FIFO(SPRITEF_HI, 0xff));
2018                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2019                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2020
2021                 I915_WRITE_FW(DSPARB3, dsparb3);
2022                 I915_WRITE_FW(DSPARB2, dsparb2);
2023                 break;
2024         default:
2025                 break;
2026         }
2027
2028         POSTING_READ_FW(DSPARB);
2029
2030         spin_unlock(&dev_priv->uncore.lock);
2031 }
2032
2033 #undef VLV_FIFO
2034
2035 static int vlv_compute_intermediate_wm(struct drm_device *dev,
2036                                        struct intel_crtc *crtc,
2037                                        struct intel_crtc_state *new_crtc_state)
2038 {
2039         struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2040         const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2041         struct intel_atomic_state *intel_state =
2042                 to_intel_atomic_state(new_crtc_state->base.state);
2043         const struct intel_crtc_state *old_crtc_state =
2044                 intel_atomic_get_old_crtc_state(intel_state, crtc);
2045         const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2046         int level;
2047
2048         if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
2049                 *intermediate = *optimal;
2050
2051                 intermediate->cxsr = false;
2052                 goto out;
2053         }
2054
2055         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2056         intermediate->cxsr = optimal->cxsr && active->cxsr &&
2057                 !new_crtc_state->disable_cxsr;
2058
2059         for (level = 0; level < intermediate->num_levels; level++) {
2060                 enum plane_id plane_id;
2061
2062                 for_each_plane_id_on_crtc(crtc, plane_id) {
2063                         intermediate->wm[level].plane[plane_id] =
2064                                 min(optimal->wm[level].plane[plane_id],
2065                                     active->wm[level].plane[plane_id]);
2066                 }
2067
2068                 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2069                                                     active->sr[level].plane);
2070                 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2071                                                      active->sr[level].cursor);
2072         }
2073
2074         vlv_invalidate_wms(crtc, intermediate, level);
2075
2076 out:
2077         /*
2078          * If our intermediate WM are identical to the final WM, then we can
2079          * omit the post-vblank programming; only update if it's different.
2080          */
2081         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2082                 new_crtc_state->wm.need_postvbl_update = true;
2083
2084         return 0;
2085 }
2086
2087 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2088                          struct vlv_wm_values *wm)
2089 {
2090         struct intel_crtc *crtc;
2091         int num_active_crtcs = 0;
2092
2093         wm->level = dev_priv->wm.max_level;
2094         wm->cxsr = true;
2095
2096         for_each_intel_crtc(&dev_priv->drm, crtc) {
2097                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2098
2099                 if (!crtc->active)
2100                         continue;
2101
2102                 if (!wm_state->cxsr)
2103                         wm->cxsr = false;
2104
2105                 num_active_crtcs++;
2106                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2107         }
2108
2109         if (num_active_crtcs != 1)
2110                 wm->cxsr = false;
2111
2112         if (num_active_crtcs > 1)
2113                 wm->level = VLV_WM_LEVEL_PM2;
2114
2115         for_each_intel_crtc(&dev_priv->drm, crtc) {
2116                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2117                 enum pipe pipe = crtc->pipe;
2118
2119                 wm->pipe[pipe] = wm_state->wm[wm->level];
2120                 if (crtc->active && wm->cxsr)
2121                         wm->sr = wm_state->sr[wm->level];
2122
2123                 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2124                 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2125                 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2126                 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2127         }
2128 }
2129
2130 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2131 {
2132         struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2133         struct vlv_wm_values new_wm = {};
2134
2135         vlv_merge_wm(dev_priv, &new_wm);
2136
2137         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2138                 return;
2139
2140         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2141                 chv_set_memory_dvfs(dev_priv, false);
2142
2143         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2144                 chv_set_memory_pm5(dev_priv, false);
2145
2146         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2147                 _intel_set_memory_cxsr(dev_priv, false);
2148
2149         vlv_write_wm_values(dev_priv, &new_wm);
2150
2151         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2152                 _intel_set_memory_cxsr(dev_priv, true);
2153
2154         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2155                 chv_set_memory_pm5(dev_priv, true);
2156
2157         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2158                 chv_set_memory_dvfs(dev_priv, true);
2159
2160         *old_wm = new_wm;
2161 }
2162
2163 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2164                                    struct intel_crtc_state *crtc_state)
2165 {
2166         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2167         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
2168
2169         mutex_lock(&dev_priv->wm.wm_mutex);
2170         crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2171         vlv_program_watermarks(dev_priv);
2172         mutex_unlock(&dev_priv->wm.wm_mutex);
2173 }
2174
2175 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2176                                     struct intel_crtc_state *crtc_state)
2177 {
2178         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2179         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
2180
2181         if (!crtc_state->wm.need_postvbl_update)
2182                 return;
2183
2184         mutex_lock(&dev_priv->wm.wm_mutex);
2185         intel_crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2186         vlv_program_watermarks(dev_priv);
2187         mutex_unlock(&dev_priv->wm.wm_mutex);
2188 }
2189
2190 static void i965_update_wm(struct intel_crtc *unused_crtc)
2191 {
2192         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2193         struct intel_crtc *crtc;
2194         int srwm = 1;
2195         int cursor_sr = 16;
2196         bool cxsr_enabled;
2197
2198         /* Calc sr entries for one plane configs */
2199         crtc = single_enabled_crtc(dev_priv);
2200         if (crtc) {
2201                 /* self-refresh has much higher latency */
2202                 static const int sr_latency_ns = 12000;
2203                 const struct drm_display_mode *adjusted_mode =
2204                         &crtc->config->base.adjusted_mode;
2205                 const struct drm_framebuffer *fb =
2206                         crtc->base.primary->state->fb;
2207                 int clock = adjusted_mode->crtc_clock;
2208                 int htotal = adjusted_mode->crtc_htotal;
2209                 int hdisplay = crtc->config->pipe_src_w;
2210                 int cpp = fb->format->cpp[0];
2211                 int entries;
2212
2213                 entries = intel_wm_method2(clock, htotal,
2214                                            hdisplay, cpp, sr_latency_ns / 100);
2215                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2216                 srwm = I965_FIFO_SIZE - entries;
2217                 if (srwm < 0)
2218                         srwm = 1;
2219                 srwm &= 0x1ff;
2220                 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
2221                               entries, srwm);
2222
2223                 entries = intel_wm_method2(clock, htotal,
2224                                            crtc->base.cursor->state->crtc_w, 4,
2225                                            sr_latency_ns / 100);
2226                 entries = DIV_ROUND_UP(entries,
2227                                        i965_cursor_wm_info.cacheline_size) +
2228                         i965_cursor_wm_info.guard_size;
2229
2230                 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2231                 if (cursor_sr > i965_cursor_wm_info.max_wm)
2232                         cursor_sr = i965_cursor_wm_info.max_wm;
2233
2234                 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
2235                               "cursor %d\n", srwm, cursor_sr);
2236
2237                 cxsr_enabled = true;
2238         } else {
2239                 cxsr_enabled = false;
2240                 /* Turn off self refresh if both pipes are enabled */
2241                 intel_set_memory_cxsr(dev_priv, false);
2242         }
2243
2244         DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2245                       srwm);
2246
2247         /* 965 has limitations... */
2248         I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
2249                    FW_WM(8, CURSORB) |
2250                    FW_WM(8, PLANEB) |
2251                    FW_WM(8, PLANEA));
2252         I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
2253                    FW_WM(8, PLANEC_OLD));
2254         /* update cursor SR watermark */
2255         I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2256
2257         if (cxsr_enabled)
2258                 intel_set_memory_cxsr(dev_priv, true);
2259 }
2260
2261 #undef FW_WM
2262
2263 static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2264 {
2265         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2266         const struct intel_watermark_params *wm_info;
2267         uint32_t fwater_lo;
2268         uint32_t fwater_hi;
2269         int cwm, srwm = 1;
2270         int fifo_size;
2271         int planea_wm, planeb_wm;
2272         struct intel_crtc *crtc, *enabled = NULL;
2273
2274         if (IS_I945GM(dev_priv))
2275                 wm_info = &i945_wm_info;
2276         else if (!IS_GEN2(dev_priv))
2277                 wm_info = &i915_wm_info;
2278         else
2279                 wm_info = &i830_a_wm_info;
2280
2281         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2282         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2283         if (intel_crtc_active(crtc)) {
2284                 const struct drm_display_mode *adjusted_mode =
2285                         &crtc->config->base.adjusted_mode;
2286                 const struct drm_framebuffer *fb =
2287                         crtc->base.primary->state->fb;
2288                 int cpp;
2289
2290                 if (IS_GEN2(dev_priv))
2291                         cpp = 4;
2292                 else
2293                         cpp = fb->format->cpp[0];
2294
2295                 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2296                                                wm_info, fifo_size, cpp,
2297                                                pessimal_latency_ns);
2298                 enabled = crtc;
2299         } else {
2300                 planea_wm = fifo_size - wm_info->guard_size;
2301                 if (planea_wm > (long)wm_info->max_wm)
2302                         planea_wm = wm_info->max_wm;
2303         }
2304
2305         if (IS_GEN2(dev_priv))
2306                 wm_info = &i830_bc_wm_info;
2307
2308         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2309         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2310         if (intel_crtc_active(crtc)) {
2311                 const struct drm_display_mode *adjusted_mode =
2312                         &crtc->config->base.adjusted_mode;
2313                 const struct drm_framebuffer *fb =
2314                         crtc->base.primary->state->fb;
2315                 int cpp;
2316
2317                 if (IS_GEN2(dev_priv))
2318                         cpp = 4;
2319                 else
2320                         cpp = fb->format->cpp[0];
2321
2322                 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2323                                                wm_info, fifo_size, cpp,
2324                                                pessimal_latency_ns);
2325                 if (enabled == NULL)
2326                         enabled = crtc;
2327                 else
2328                         enabled = NULL;
2329         } else {
2330                 planeb_wm = fifo_size - wm_info->guard_size;
2331                 if (planeb_wm > (long)wm_info->max_wm)
2332                         planeb_wm = wm_info->max_wm;
2333         }
2334
2335         DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2336
2337         if (IS_I915GM(dev_priv) && enabled) {
2338                 struct drm_i915_gem_object *obj;
2339
2340                 obj = intel_fb_obj(enabled->base.primary->state->fb);
2341
2342                 /* self-refresh seems busted with untiled */
2343                 if (!i915_gem_object_is_tiled(obj))
2344                         enabled = NULL;
2345         }
2346
2347         /*
2348          * Overlay gets an aggressive default since video jitter is bad.
2349          */
2350         cwm = 2;
2351
2352         /* Play safe and disable self-refresh before adjusting watermarks. */
2353         intel_set_memory_cxsr(dev_priv, false);
2354
2355         /* Calc sr entries for one plane configs */
2356         if (HAS_FW_BLC(dev_priv) && enabled) {
2357                 /* self-refresh has much higher latency */
2358                 static const int sr_latency_ns = 6000;
2359                 const struct drm_display_mode *adjusted_mode =
2360                         &enabled->config->base.adjusted_mode;
2361                 const struct drm_framebuffer *fb =
2362                         enabled->base.primary->state->fb;
2363                 int clock = adjusted_mode->crtc_clock;
2364                 int htotal = adjusted_mode->crtc_htotal;
2365                 int hdisplay = enabled->config->pipe_src_w;
2366                 int cpp;
2367                 int entries;
2368
2369                 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2370                         cpp = 4;
2371                 else
2372                         cpp = fb->format->cpp[0];
2373
2374                 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2375                                            sr_latency_ns / 100);
2376                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2377                 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
2378                 srwm = wm_info->fifo_size - entries;
2379                 if (srwm < 0)
2380                         srwm = 1;
2381
2382                 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2383                         I915_WRITE(FW_BLC_SELF,
2384                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2385                 else
2386                         I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
2387         }
2388
2389         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2390                       planea_wm, planeb_wm, cwm, srwm);
2391
2392         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2393         fwater_hi = (cwm & 0x1f);
2394
2395         /* Set request length to 8 cachelines per fetch */
2396         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2397         fwater_hi = fwater_hi | (1 << 8);
2398
2399         I915_WRITE(FW_BLC, fwater_lo);
2400         I915_WRITE(FW_BLC2, fwater_hi);
2401
2402         if (enabled)
2403                 intel_set_memory_cxsr(dev_priv, true);
2404 }
2405
2406 static void i845_update_wm(struct intel_crtc *unused_crtc)
2407 {
2408         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2409         struct intel_crtc *crtc;
2410         const struct drm_display_mode *adjusted_mode;
2411         uint32_t fwater_lo;
2412         int planea_wm;
2413
2414         crtc = single_enabled_crtc(dev_priv);
2415         if (crtc == NULL)
2416                 return;
2417
2418         adjusted_mode = &crtc->config->base.adjusted_mode;
2419         planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2420                                        &i845_wm_info,
2421                                        dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2422                                        4, pessimal_latency_ns);
2423         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
2424         fwater_lo |= (3<<8) | planea_wm;
2425
2426         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
2427
2428         I915_WRITE(FW_BLC, fwater_lo);
2429 }
2430
2431 /* latency must be in 0.1us units. */
2432 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2433                                    unsigned int cpp,
2434                                    unsigned int latency)
2435 {
2436         unsigned int ret;
2437
2438         ret = intel_wm_method1(pixel_rate, cpp, latency);
2439         ret = DIV_ROUND_UP(ret, 64) + 2;
2440
2441         return ret;
2442 }
2443
2444 /* latency must be in 0.1us units. */
2445 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2446                                    unsigned int htotal,
2447                                    unsigned int width,
2448                                    unsigned int cpp,
2449                                    unsigned int latency)
2450 {
2451         unsigned int ret;
2452
2453         ret = intel_wm_method2(pixel_rate, htotal,
2454                                width, cpp, latency);
2455         ret = DIV_ROUND_UP(ret, 64) + 2;
2456
2457         return ret;
2458 }
2459
2460 static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
2461                            uint8_t cpp)
2462 {
2463         /*
2464          * Neither of these should be possible since this function shouldn't be
2465          * called if the CRTC is off or the plane is invisible.  But let's be
2466          * extra paranoid to avoid a potential divide-by-zero if we screw up
2467          * elsewhere in the driver.
2468          */
2469         if (WARN_ON(!cpp))
2470                 return 0;
2471         if (WARN_ON(!horiz_pixels))
2472                 return 0;
2473
2474         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2475 }
2476
2477 struct ilk_wm_maximums {
2478         uint16_t pri;
2479         uint16_t spr;
2480         uint16_t cur;
2481         uint16_t fbc;
2482 };
2483
2484 /*
2485  * For both WM_PIPE and WM_LP.
2486  * mem_value must be in 0.1us units.
2487  */
2488 static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate,
2489                                    const struct intel_plane_state *pstate,
2490                                    uint32_t mem_value,
2491                                    bool is_lp)
2492 {
2493         uint32_t method1, method2;
2494         int cpp;
2495
2496         if (!intel_wm_plane_visible(cstate, pstate))
2497                 return 0;
2498
2499         cpp = pstate->base.fb->format->cpp[0];
2500
2501         method1 = ilk_wm_method1(cstate->pixel_rate, cpp, mem_value);
2502
2503         if (!is_lp)
2504                 return method1;
2505
2506         method2 = ilk_wm_method2(cstate->pixel_rate,
2507                                  cstate->base.adjusted_mode.crtc_htotal,
2508                                  drm_rect_width(&pstate->base.dst),
2509                                  cpp, mem_value);
2510
2511         return min(method1, method2);
2512 }
2513
2514 /*
2515  * For both WM_PIPE and WM_LP.
2516  * mem_value must be in 0.1us units.
2517  */
2518 static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate,
2519                                    const struct intel_plane_state *pstate,
2520                                    uint32_t mem_value)
2521 {
2522         uint32_t method1, method2;
2523         int cpp;
2524
2525         if (!intel_wm_plane_visible(cstate, pstate))
2526                 return 0;
2527
2528         cpp = pstate->base.fb->format->cpp[0];
2529
2530         method1 = ilk_wm_method1(cstate->pixel_rate, cpp, mem_value);
2531         method2 = ilk_wm_method2(cstate->pixel_rate,
2532                                  cstate->base.adjusted_mode.crtc_htotal,
2533                                  drm_rect_width(&pstate->base.dst),
2534                                  cpp, mem_value);
2535         return min(method1, method2);
2536 }
2537
2538 /*
2539  * For both WM_PIPE and WM_LP.
2540  * mem_value must be in 0.1us units.
2541  */
2542 static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate,
2543                                    const struct intel_plane_state *pstate,
2544                                    uint32_t mem_value)
2545 {
2546         int cpp;
2547
2548         if (!intel_wm_plane_visible(cstate, pstate))
2549                 return 0;
2550
2551         cpp = pstate->base.fb->format->cpp[0];
2552
2553         return ilk_wm_method2(cstate->pixel_rate,
2554                               cstate->base.adjusted_mode.crtc_htotal,
2555                               pstate->base.crtc_w, cpp, mem_value);
2556 }
2557
2558 /* Only for WM_LP. */
2559 static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
2560                                    const struct intel_plane_state *pstate,
2561                                    uint32_t pri_val)
2562 {
2563         int cpp;
2564
2565         if (!intel_wm_plane_visible(cstate, pstate))
2566                 return 0;
2567
2568         cpp = pstate->base.fb->format->cpp[0];
2569
2570         return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->base.dst), cpp);
2571 }
2572
2573 static unsigned int
2574 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2575 {
2576         if (INTEL_GEN(dev_priv) >= 8)
2577                 return 3072;
2578         else if (INTEL_GEN(dev_priv) >= 7)
2579                 return 768;
2580         else
2581                 return 512;
2582 }
2583
2584 static unsigned int
2585 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2586                      int level, bool is_sprite)
2587 {
2588         if (INTEL_GEN(dev_priv) >= 8)
2589                 /* BDW primary/sprite plane watermarks */
2590                 return level == 0 ? 255 : 2047;
2591         else if (INTEL_GEN(dev_priv) >= 7)
2592                 /* IVB/HSW primary/sprite plane watermarks */
2593                 return level == 0 ? 127 : 1023;
2594         else if (!is_sprite)
2595                 /* ILK/SNB primary plane watermarks */
2596                 return level == 0 ? 127 : 511;
2597         else
2598                 /* ILK/SNB sprite plane watermarks */
2599                 return level == 0 ? 63 : 255;
2600 }
2601
2602 static unsigned int
2603 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2604 {
2605         if (INTEL_GEN(dev_priv) >= 7)
2606                 return level == 0 ? 63 : 255;
2607         else
2608                 return level == 0 ? 31 : 63;
2609 }
2610
2611 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2612 {
2613         if (INTEL_GEN(dev_priv) >= 8)
2614                 return 31;
2615         else
2616                 return 15;
2617 }
2618
2619 /* Calculate the maximum primary/sprite plane watermark */
2620 static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
2621                                      int level,
2622                                      const struct intel_wm_config *config,
2623                                      enum intel_ddb_partitioning ddb_partitioning,
2624                                      bool is_sprite)
2625 {
2626         struct drm_i915_private *dev_priv = to_i915(dev);
2627         unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2628
2629         /* if sprites aren't enabled, sprites get nothing */
2630         if (is_sprite && !config->sprites_enabled)
2631                 return 0;
2632
2633         /* HSW allows LP1+ watermarks even with multiple pipes */
2634         if (level == 0 || config->num_pipes_active > 1) {
2635                 fifo_size /= INTEL_INFO(dev_priv)->num_pipes;
2636
2637                 /*
2638                  * For some reason the non self refresh
2639                  * FIFO size is only half of the self
2640                  * refresh FIFO size on ILK/SNB.
2641                  */
2642                 if (INTEL_GEN(dev_priv) <= 6)
2643                         fifo_size /= 2;
2644         }
2645
2646         if (config->sprites_enabled) {
2647                 /* level 0 is always calculated with 1:1 split */
2648                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2649                         if (is_sprite)
2650                                 fifo_size *= 5;
2651                         fifo_size /= 6;
2652                 } else {
2653                         fifo_size /= 2;
2654                 }
2655         }
2656
2657         /* clamp to max that the registers can hold */
2658         return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2659 }
2660
2661 /* Calculate the maximum cursor plane watermark */
2662 static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
2663                                       int level,
2664                                       const struct intel_wm_config *config)
2665 {
2666         /* HSW LP1+ watermarks w/ multiple pipes */
2667         if (level > 0 && config->num_pipes_active > 1)
2668                 return 64;
2669
2670         /* otherwise just report max that registers can hold */
2671         return ilk_cursor_wm_reg_max(to_i915(dev), level);
2672 }
2673
2674 static void ilk_compute_wm_maximums(const struct drm_device *dev,
2675                                     int level,
2676                                     const struct intel_wm_config *config,
2677                                     enum intel_ddb_partitioning ddb_partitioning,
2678                                     struct ilk_wm_maximums *max)
2679 {
2680         max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
2681         max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
2682         max->cur = ilk_cursor_wm_max(dev, level, config);
2683         max->fbc = ilk_fbc_wm_reg_max(to_i915(dev));
2684 }
2685
2686 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2687                                         int level,
2688                                         struct ilk_wm_maximums *max)
2689 {
2690         max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2691         max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2692         max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2693         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2694 }
2695
2696 static bool ilk_validate_wm_level(int level,
2697                                   const struct ilk_wm_maximums *max,
2698                                   struct intel_wm_level *result)
2699 {
2700         bool ret;
2701
2702         /* already determined to be invalid? */
2703         if (!result->enable)
2704                 return false;
2705
2706         result->enable = result->pri_val <= max->pri &&
2707                          result->spr_val <= max->spr &&
2708                          result->cur_val <= max->cur;
2709
2710         ret = result->enable;
2711
2712         /*
2713          * HACK until we can pre-compute everything,
2714          * and thus fail gracefully if LP0 watermarks
2715          * are exceeded...
2716          */
2717         if (level == 0 && !result->enable) {
2718                 if (result->pri_val > max->pri)
2719                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2720                                       level, result->pri_val, max->pri);
2721                 if (result->spr_val > max->spr)
2722                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2723                                       level, result->spr_val, max->spr);
2724                 if (result->cur_val > max->cur)
2725                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2726                                       level, result->cur_val, max->cur);
2727
2728                 result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
2729                 result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
2730                 result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
2731                 result->enable = true;
2732         }
2733
2734         return ret;
2735 }
2736
2737 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2738                                  const struct intel_crtc *intel_crtc,
2739                                  int level,
2740                                  struct intel_crtc_state *cstate,
2741                                  const struct intel_plane_state *pristate,
2742                                  const struct intel_plane_state *sprstate,
2743                                  const struct intel_plane_state *curstate,
2744                                  struct intel_wm_level *result)
2745 {
2746         uint16_t pri_latency = dev_priv->wm.pri_latency[level];
2747         uint16_t spr_latency = dev_priv->wm.spr_latency[level];
2748         uint16_t cur_latency = dev_priv->wm.cur_latency[level];
2749
2750         /* WM1+ latency values stored in 0.5us units */
2751         if (level > 0) {
2752                 pri_latency *= 5;
2753                 spr_latency *= 5;
2754                 cur_latency *= 5;
2755         }
2756
2757         if (pristate) {
2758                 result->pri_val = ilk_compute_pri_wm(cstate, pristate,
2759                                                      pri_latency, level);
2760                 result->fbc_val = ilk_compute_fbc_wm(cstate, pristate, result->pri_val);
2761         }
2762
2763         if (sprstate)
2764                 result->spr_val = ilk_compute_spr_wm(cstate, sprstate, spr_latency);
2765
2766         if (curstate)
2767                 result->cur_val = ilk_compute_cur_wm(cstate, curstate, cur_latency);
2768
2769         result->enable = true;
2770 }
2771
2772 static uint32_t
2773 hsw_compute_linetime_wm(const struct intel_crtc_state *cstate)
2774 {
2775         const struct intel_atomic_state *intel_state =
2776                 to_intel_atomic_state(cstate->base.state);
2777         const struct drm_display_mode *adjusted_mode =
2778                 &cstate->base.adjusted_mode;
2779         u32 linetime, ips_linetime;
2780
2781         if (!cstate->base.active)
2782                 return 0;
2783         if (WARN_ON(adjusted_mode->crtc_clock == 0))
2784                 return 0;
2785         if (WARN_ON(intel_state->cdclk.logical.cdclk == 0))
2786                 return 0;
2787
2788         /* The WM are computed with base on how long it takes to fill a single
2789          * row at the given clock rate, multiplied by 8.
2790          * */
2791         linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2792                                      adjusted_mode->crtc_clock);
2793         ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2794                                          intel_state->cdclk.logical.cdclk);
2795
2796         return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
2797                PIPE_WM_LINETIME_TIME(linetime);
2798 }
2799
2800 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2801                                   uint16_t wm[8])
2802 {
2803         if (INTEL_GEN(dev_priv) >= 9) {
2804                 uint32_t val;
2805                 int ret, i;
2806                 int level, max_level = ilk_wm_max_level(dev_priv);
2807
2808                 /* read the first set of memory latencies[0:3] */
2809                 val = 0; /* data0 to be programmed to 0 for first set */
2810                 mutex_lock(&dev_priv->pcu_lock);
2811                 ret = sandybridge_pcode_read(dev_priv,
2812                                              GEN9_PCODE_READ_MEM_LATENCY,
2813                                              &val);
2814                 mutex_unlock(&dev_priv->pcu_lock);
2815
2816                 if (ret) {
2817                         DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2818                         return;
2819                 }
2820
2821                 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2822                 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2823                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2824                 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2825                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2826                 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2827                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2828
2829                 /* read the second set of memory latencies[4:7] */
2830                 val = 1; /* data0 to be programmed to 1 for second set */
2831                 mutex_lock(&dev_priv->pcu_lock);
2832                 ret = sandybridge_pcode_read(dev_priv,
2833                                              GEN9_PCODE_READ_MEM_LATENCY,
2834                                              &val);
2835                 mutex_unlock(&dev_priv->pcu_lock);
2836                 if (ret) {
2837                         DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2838                         return;
2839                 }
2840
2841                 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2842                 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2843                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2844                 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2845                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2846                 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2847                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2848
2849                 /*
2850                  * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2851                  * need to be disabled. We make sure to sanitize the values out
2852                  * of the punit to satisfy this requirement.
2853                  */
2854                 for (level = 1; level <= max_level; level++) {
2855                         if (wm[level] == 0) {
2856                                 for (i = level + 1; i <= max_level; i++)
2857                                         wm[i] = 0;
2858                                 break;
2859                         }
2860                 }
2861
2862                 /*
2863                  * WaWmMemoryReadLatency:skl+,glk
2864                  *
2865                  * punit doesn't take into account the read latency so we need
2866                  * to add 2us to the various latency levels we retrieve from the
2867                  * punit when level 0 response data us 0us.
2868                  */
2869                 if (wm[0] == 0) {
2870                         wm[0] += 2;
2871                         for (level = 1; level <= max_level; level++) {
2872                                 if (wm[level] == 0)
2873                                         break;
2874                                 wm[level] += 2;
2875                         }
2876                 }
2877
2878                 /*
2879                  * WA Level-0 adjustment for 16GB DIMMs: SKL+
2880                  * If we could not get dimm info enable this WA to prevent from
2881                  * any underrun. If not able to get Dimm info assume 16GB dimm
2882                  * to avoid any underrun.
2883                  */
2884                 if (dev_priv->dram_info.is_16gb_dimm)
2885                         wm[0] += 1;
2886
2887         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2888                 uint64_t sskpd = I915_READ64(MCH_SSKPD);
2889
2890                 wm[0] = (sskpd >> 56) & 0xFF;
2891                 if (wm[0] == 0)
2892                         wm[0] = sskpd & 0xF;
2893                 wm[1] = (sskpd >> 4) & 0xFF;
2894                 wm[2] = (sskpd >> 12) & 0xFF;
2895                 wm[3] = (sskpd >> 20) & 0x1FF;
2896                 wm[4] = (sskpd >> 32) & 0x1FF;
2897         } else if (INTEL_GEN(dev_priv) >= 6) {
2898                 uint32_t sskpd = I915_READ(MCH_SSKPD);
2899
2900                 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2901                 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2902                 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2903                 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2904         } else if (INTEL_GEN(dev_priv) >= 5) {
2905                 uint32_t mltr = I915_READ(MLTR_ILK);
2906
2907                 /* ILK primary LP0 latency is 700 ns */
2908                 wm[0] = 7;
2909                 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2910                 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2911         } else {
2912                 MISSING_CASE(INTEL_DEVID(dev_priv));
2913         }
2914 }
2915
2916 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2917                                        uint16_t wm[5])
2918 {
2919         /* ILK sprite LP0 latency is 1300 ns */
2920         if (IS_GEN5(dev_priv))
2921                 wm[0] = 13;
2922 }
2923
2924 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2925                                        uint16_t wm[5])
2926 {
2927         /* ILK cursor LP0 latency is 1300 ns */
2928         if (IS_GEN5(dev_priv))
2929                 wm[0] = 13;
2930 }
2931
2932 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2933 {
2934         /* how many WM levels are we expecting */
2935         if (INTEL_GEN(dev_priv) >= 9)
2936                 return 7;
2937         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2938                 return 4;
2939         else if (INTEL_GEN(dev_priv) >= 6)
2940                 return 3;
2941         else
2942                 return 2;
2943 }
2944
2945 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2946                                    const char *name,
2947                                    const uint16_t wm[8])
2948 {
2949         int level, max_level = ilk_wm_max_level(dev_priv);
2950
2951         for (level = 0; level <= max_level; level++) {
2952                 unsigned int latency = wm[level];
2953
2954                 if (latency == 0) {
2955                         DRM_DEBUG_KMS("%s WM%d latency not provided\n",
2956                                       name, level);
2957                         continue;
2958                 }
2959
2960                 /*
2961                  * - latencies are in us on gen9.
2962                  * - before then, WM1+ latency values are in 0.5us units
2963                  */
2964                 if (INTEL_GEN(dev_priv) >= 9)
2965                         latency *= 10;
2966                 else if (level > 0)
2967                         latency *= 5;
2968
2969                 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
2970                               name, level, wm[level],
2971                               latency / 10, latency % 10);
2972         }
2973 }
2974
2975 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2976                                     uint16_t wm[5], uint16_t min)
2977 {
2978         int level, max_level = ilk_wm_max_level(dev_priv);
2979
2980         if (wm[0] >= min)
2981                 return false;
2982
2983         wm[0] = max(wm[0], min);
2984         for (level = 1; level <= max_level; level++)
2985                 wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
2986
2987         return true;
2988 }
2989
2990 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2991 {
2992         bool changed;
2993
2994         /*
2995          * The BIOS provided WM memory latency values are often
2996          * inadequate for high resolution displays. Adjust them.
2997          */
2998         changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
2999                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3000                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3001
3002         if (!changed)
3003                 return;
3004
3005         DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
3006         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3007         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3008         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3009 }
3010
3011 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3012 {
3013         intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3014
3015         memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3016                sizeof(dev_priv->wm.pri_latency));
3017         memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3018                sizeof(dev_priv->wm.pri_latency));
3019
3020         intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3021         intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3022
3023         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3024         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3025         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3026
3027         if (IS_GEN6(dev_priv))
3028                 snb_wm_latency_quirk(dev_priv);
3029 }
3030
3031 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3032 {
3033         intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3034         intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3035 }
3036
3037 static bool ilk_validate_pipe_wm(struct drm_device *dev,
3038                                  struct intel_pipe_wm *pipe_wm)
3039 {
3040         /* LP0 watermark maximums depend on this pipe alone */
3041         const struct intel_wm_config config = {
3042                 .num_pipes_active = 1,
3043                 .sprites_enabled = pipe_wm->sprites_enabled,
3044                 .sprites_scaled = pipe_wm->sprites_scaled,
3045         };
3046         struct ilk_wm_maximums max;
3047
3048         /* LP0 watermarks always use 1/2 DDB partitioning */
3049         ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
3050
3051         /* At least LP0 must be valid */
3052         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3053                 DRM_DEBUG_KMS("LP0 watermark invalid\n");
3054                 return false;
3055         }
3056
3057         return true;
3058 }
3059
3060 /* Compute new watermarks for the pipe */
3061 static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
3062 {
3063         struct drm_atomic_state *state = cstate->base.state;
3064         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
3065         struct intel_pipe_wm *pipe_wm;
3066         struct drm_device *dev = state->dev;
3067         const struct drm_i915_private *dev_priv = to_i915(dev);
3068         struct drm_plane *plane;
3069         const struct drm_plane_state *plane_state;
3070         const struct intel_plane_state *pristate = NULL;
3071         const struct intel_plane_state *sprstate = NULL;
3072         const struct intel_plane_state *curstate = NULL;
3073         int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3074         struct ilk_wm_maximums max;
3075
3076         pipe_wm = &cstate->wm.ilk.optimal;
3077
3078         drm_atomic_crtc_state_for_each_plane_state(plane, plane_state, &cstate->base) {
3079                 const struct intel_plane_state *ps = to_intel_plane_state(plane_state);
3080
3081                 if (plane->type == DRM_PLANE_TYPE_PRIMARY)
3082                         pristate = ps;
3083                 else if (plane->type == DRM_PLANE_TYPE_OVERLAY)
3084                         sprstate = ps;
3085                 else if (plane->type == DRM_PLANE_TYPE_CURSOR)
3086                         curstate = ps;
3087         }
3088
3089         pipe_wm->pipe_enabled = cstate->base.active;
3090         if (sprstate) {
3091                 pipe_wm->sprites_enabled = sprstate->base.visible;
3092                 pipe_wm->sprites_scaled = sprstate->base.visible &&
3093                         (drm_rect_width(&sprstate->base.dst) != drm_rect_width(&sprstate->base.src) >> 16 ||
3094                          drm_rect_height(&sprstate->base.dst) != drm_rect_height(&sprstate->base.src) >> 16);
3095         }
3096
3097         usable_level = max_level;
3098
3099         /* ILK/SNB: LP2+ watermarks only w/o sprites */
3100         if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3101                 usable_level = 1;
3102
3103         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3104         if (pipe_wm->sprites_scaled)
3105                 usable_level = 0;
3106
3107         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3108         ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate,
3109                              pristate, sprstate, curstate, &pipe_wm->wm[0]);
3110
3111         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3112                 pipe_wm->linetime = hsw_compute_linetime_wm(cstate);
3113
3114         if (!ilk_validate_pipe_wm(dev, pipe_wm))
3115                 return -EINVAL;
3116
3117         ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3118
3119         for (level = 1; level <= usable_level; level++) {
3120                 struct intel_wm_level *wm = &pipe_wm->wm[level];
3121
3122                 ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate,
3123                                      pristate, sprstate, curstate, wm);
3124
3125                 /*
3126                  * Disable any watermark level that exceeds the
3127                  * register maximums since such watermarks are
3128                  * always invalid.
3129                  */
3130                 if (!ilk_validate_wm_level(level, &max, wm)) {
3131                         memset(wm, 0, sizeof(*wm));
3132                         break;
3133                 }
3134         }
3135
3136         return 0;
3137 }
3138
3139 /*
3140  * Build a set of 'intermediate' watermark values that satisfy both the old
3141  * state and the new state.  These can be programmed to the hardware
3142  * immediately.
3143  */
3144 static int ilk_compute_intermediate_wm(struct drm_device *dev,
3145                                        struct intel_crtc *intel_crtc,
3146                                        struct intel_crtc_state *newstate)
3147 {
3148         struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3149         struct intel_atomic_state *intel_state =
3150                 to_intel_atomic_state(newstate->base.state);
3151         const struct intel_crtc_state *oldstate =
3152                 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3153         const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3154         int level, max_level = ilk_wm_max_level(to_i915(dev));
3155
3156         /*
3157          * Start with the final, target watermarks, then combine with the
3158          * currently active watermarks to get values that are safe both before
3159          * and after the vblank.
3160          */
3161         *a = newstate->wm.ilk.optimal;
3162         if (!newstate->base.active || drm_atomic_crtc_needs_modeset(&newstate->base))
3163                 return 0;
3164
3165         a->pipe_enabled |= b->pipe_enabled;
3166         a->sprites_enabled |= b->sprites_enabled;
3167         a->sprites_scaled |= b->sprites_scaled;
3168
3169         for (level = 0; level <= max_level; level++) {
3170                 struct intel_wm_level *a_wm = &a->wm[level];
3171                 const struct intel_wm_level *b_wm = &b->wm[level];
3172
3173                 a_wm->enable &= b_wm->enable;
3174                 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3175                 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3176                 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3177                 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3178         }
3179
3180         /*
3181          * We need to make sure that these merged watermark values are
3182          * actually a valid configuration themselves.  If they're not,
3183          * there's no safe way to transition from the old state to
3184          * the new state, so we need to fail the atomic transaction.
3185          */
3186         if (!ilk_validate_pipe_wm(dev, a))
3187                 return -EINVAL;
3188
3189         /*
3190          * If our intermediate WM are identical to the final WM, then we can
3191          * omit the post-vblank programming; only update if it's different.
3192          */
3193         if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3194                 newstate->wm.need_postvbl_update = true;
3195
3196         return 0;
3197 }
3198
3199 /*
3200  * Merge the watermarks from all active pipes for a specific level.
3201  */
3202 static void ilk_merge_wm_level(struct drm_device *dev,
3203                                int level,
3204                                struct intel_wm_level *ret_wm)
3205 {
3206         const struct intel_crtc *intel_crtc;
3207
3208         ret_wm->enable = true;
3209
3210         for_each_intel_crtc(dev, intel_crtc) {
3211                 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3212                 const struct intel_wm_level *wm = &active->wm[level];
3213
3214                 if (!active->pipe_enabled)
3215                         continue;
3216
3217                 /*
3218                  * The watermark values may have been used in the past,
3219                  * so we must maintain them in the registers for some
3220                  * time even if the level is now disabled.
3221                  */
3222                 if (!wm->enable)
3223                         ret_wm->enable = false;
3224
3225                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3226                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3227                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3228                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3229         }
3230 }
3231
3232 /*
3233  * Merge all low power watermarks for all active pipes.
3234  */
3235 static void ilk_wm_merge(struct drm_device *dev,
3236                          const struct intel_wm_config *config,
3237                          const struct ilk_wm_maximums *max,
3238                          struct intel_pipe_wm *merged)
3239 {
3240         struct drm_i915_private *dev_priv = to_i915(dev);
3241         int level, max_level = ilk_wm_max_level(dev_priv);
3242         int last_enabled_level = max_level;
3243
3244         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3245         if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3246             config->num_pipes_active > 1)
3247                 last_enabled_level = 0;
3248
3249         /* ILK: FBC WM must be disabled always */
3250         merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3251
3252         /* merge each WM1+ level */
3253         for (level = 1; level <= max_level; level++) {
3254                 struct intel_wm_level *wm = &merged->wm[level];
3255
3256                 ilk_merge_wm_level(dev, level, wm);
3257
3258                 if (level > last_enabled_level)
3259                         wm->enable = false;
3260                 else if (!ilk_validate_wm_level(level, max, wm))
3261                         /* make sure all following levels get disabled */
3262                         last_enabled_level = level - 1;
3263
3264                 /*
3265                  * The spec says it is preferred to disable
3266                  * FBC WMs instead of disabling a WM level.
3267                  */
3268                 if (wm->fbc_val > max->fbc) {
3269                         if (wm->enable)
3270                                 merged->fbc_wm_enabled = false;
3271                         wm->fbc_val = 0;
3272                 }
3273         }
3274
3275         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3276         /*
3277          * FIXME this is racy. FBC might get enabled later.
3278          * What we should check here is whether FBC can be
3279          * enabled sometime later.
3280          */
3281         if (IS_GEN5(dev_priv) && !merged->fbc_wm_enabled &&
3282             intel_fbc_is_active(dev_priv)) {
3283                 for (level = 2; level <= max_level; level++) {
3284                         struct intel_wm_level *wm = &merged->wm[level];
3285
3286                         wm->enable = false;
3287                 }
3288         }
3289 }
3290
3291 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3292 {
3293         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3294         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3295 }
3296
3297 /* The value we need to program into the WM_LPx latency field */
3298 static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
3299 {
3300         struct drm_i915_private *dev_priv = to_i915(dev);
3301
3302         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3303                 return 2 * level;
3304         else
3305                 return dev_priv->wm.pri_latency[level];
3306 }
3307
3308 static void ilk_compute_wm_results(struct drm_device *dev,
3309                                    const struct intel_pipe_wm *merged,
3310                                    enum intel_ddb_partitioning partitioning,
3311                                    struct ilk_wm_values *results)
3312 {
3313         struct drm_i915_private *dev_priv = to_i915(dev);
3314         struct intel_crtc *intel_crtc;
3315         int level, wm_lp;
3316
3317         results->enable_fbc_wm = merged->fbc_wm_enabled;
3318         results->partitioning = partitioning;
3319
3320         /* LP1+ register values */
3321         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3322                 const struct intel_wm_level *r;
3323
3324                 level = ilk_wm_lp_to_level(wm_lp, merged);
3325
3326                 r = &merged->wm[level];
3327
3328                 /*
3329                  * Maintain the watermark values even if the level is
3330                  * disabled. Doing otherwise could cause underruns.
3331                  */
3332                 results->wm_lp[wm_lp - 1] =
3333                         (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) |
3334                         (r->pri_val << WM1_LP_SR_SHIFT) |
3335                         r->cur_val;
3336
3337                 if (r->enable)
3338                         results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3339
3340                 if (INTEL_GEN(dev_priv) >= 8)
3341                         results->wm_lp[wm_lp - 1] |=
3342                                 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3343                 else
3344                         results->wm_lp[wm_lp - 1] |=
3345                                 r->fbc_val << WM1_LP_FBC_SHIFT;
3346
3347                 /*
3348                  * Always set WM1S_LP_EN when spr_val != 0, even if the
3349                  * level is disabled. Doing otherwise could cause underruns.
3350                  */
3351                 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3352                         WARN_ON(wm_lp != 1);
3353                         results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3354                 } else
3355                         results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3356         }
3357
3358         /* LP0 register values */
3359         for_each_intel_crtc(dev, intel_crtc) {
3360                 enum pipe pipe = intel_crtc->pipe;
3361                 const struct intel_wm_level *r =
3362                         &intel_crtc->wm.active.ilk.wm[0];
3363
3364                 if (WARN_ON(!r->enable))
3365                         continue;
3366
3367                 results->wm_linetime[pipe] = intel_crtc->wm.active.ilk.linetime;
3368
3369                 results->wm_pipe[pipe] =
3370                         (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3371                         (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3372                         r->cur_val;
3373         }
3374 }
3375
3376 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3377  * case both are at the same level. Prefer r1 in case they're the same. */
3378 static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
3379                                                   struct intel_pipe_wm *r1,
3380                                                   struct intel_pipe_wm *r2)
3381 {
3382         int level, max_level = ilk_wm_max_level(to_i915(dev));
3383         int level1 = 0, level2 = 0;
3384
3385         for (level = 1; level <= max_level; level++) {
3386                 if (r1->wm[level].enable)
3387                         level1 = level;
3388                 if (r2->wm[level].enable)
3389                         level2 = level;
3390         }
3391
3392         if (level1 == level2) {
3393                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3394                         return r2;
3395                 else
3396                         return r1;
3397         } else if (level1 > level2) {
3398                 return r1;
3399         } else {
3400                 return r2;
3401         }
3402 }
3403
3404 /* dirty bits used to track which watermarks need changes */
3405 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3406 #define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
3407 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3408 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3409 #define WM_DIRTY_FBC (1 << 24)
3410 #define WM_DIRTY_DDB (1 << 25)
3411
3412 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3413                                          const struct ilk_wm_values *old,
3414                                          const struct ilk_wm_values *new)
3415 {
3416         unsigned int dirty = 0;
3417         enum pipe pipe;
3418         int wm_lp;
3419
3420         for_each_pipe(dev_priv, pipe) {
3421                 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
3422                         dirty |= WM_DIRTY_LINETIME(pipe);
3423                         /* Must disable LP1+ watermarks too */
3424                         dirty |= WM_DIRTY_LP_ALL;
3425                 }
3426
3427                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3428                         dirty |= WM_DIRTY_PIPE(pipe);
3429                         /* Must disable LP1+ watermarks too */
3430                         dirty |= WM_DIRTY_LP_ALL;
3431                 }
3432         }
3433
3434         if (old->enable_fbc_wm != new->enable_fbc_wm) {
3435                 dirty |= WM_DIRTY_FBC;
3436                 /* Must disable LP1+ watermarks too */
3437                 dirty |= WM_DIRTY_LP_ALL;
3438         }
3439
3440         if (old->partitioning != new->partitioning) {
3441                 dirty |= WM_DIRTY_DDB;
3442                 /* Must disable LP1+ watermarks too */
3443                 dirty |= WM_DIRTY_LP_ALL;
3444         }
3445
3446         /* LP1+ watermarks already deemed dirty, no need to continue */
3447         if (dirty & WM_DIRTY_LP_ALL)
3448                 return dirty;
3449
3450         /* Find the lowest numbered LP1+ watermark in need of an update... */
3451         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3452                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3453                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3454                         break;
3455         }
3456
3457         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3458         for (; wm_lp <= 3; wm_lp++)
3459                 dirty |= WM_DIRTY_LP(wm_lp);
3460
3461         return dirty;
3462 }
3463
3464 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3465                                unsigned int dirty)
3466 {
3467         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3468         bool changed = false;
3469
3470         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3471                 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3472                 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3473                 changed = true;
3474         }
3475         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3476                 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3477                 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3478                 changed = true;
3479         }
3480         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3481                 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3482                 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3483                 changed = true;
3484         }
3485
3486         /*
3487          * Don't touch WM1S_LP_EN here.
3488          * Doing so could cause underruns.
3489          */
3490
3491         return changed;
3492 }
3493
3494 /*
3495  * The spec says we shouldn't write when we don't need, because every write
3496  * causes WMs to be re-evaluated, expending some power.
3497  */
3498 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3499                                 struct ilk_wm_values *results)
3500 {
3501         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3502         unsigned int dirty;
3503         uint32_t val;
3504
3505         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3506         if (!dirty)
3507                 return;
3508
3509         _ilk_disable_lp_wm(dev_priv, dirty);
3510
3511         if (dirty & WM_DIRTY_PIPE(PIPE_A))
3512                 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3513         if (dirty & WM_DIRTY_PIPE(PIPE_B))
3514                 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3515         if (dirty & WM_DIRTY_PIPE(PIPE_C))
3516                 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3517
3518         if (dirty & WM_DIRTY_LINETIME(PIPE_A))
3519                 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
3520         if (dirty & WM_DIRTY_LINETIME(PIPE_B))
3521                 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
3522         if (dirty & WM_DIRTY_LINETIME(PIPE_C))
3523                 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
3524
3525         if (dirty & WM_DIRTY_DDB) {
3526                 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3527                         val = I915_READ(WM_MISC);
3528                         if (results->partitioning == INTEL_DDB_PART_1_2)
3529                                 val &= ~WM_MISC_DATA_PARTITION_5_6;
3530                         else
3531                                 val |= WM_MISC_DATA_PARTITION_5_6;
3532                         I915_WRITE(WM_MISC, val);
3533                 } else {
3534                         val = I915_READ(DISP_ARB_CTL2);
3535                         if (results->partitioning == INTEL_DDB_PART_1_2)
3536                                 val &= ~DISP_DATA_PARTITION_5_6;
3537                         else
3538                                 val |= DISP_DATA_PARTITION_5_6;
3539                         I915_WRITE(DISP_ARB_CTL2, val);
3540                 }
3541         }
3542
3543         if (dirty & WM_DIRTY_FBC) {
3544                 val = I915_READ(DISP_ARB_CTL);
3545                 if (results->enable_fbc_wm)
3546                         val &= ~DISP_FBC_WM_DIS;
3547                 else
3548                         val |= DISP_FBC_WM_DIS;
3549                 I915_WRITE(DISP_ARB_CTL, val);
3550         }
3551
3552         if (dirty & WM_DIRTY_LP(1) &&
3553             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3554                 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3555
3556         if (INTEL_GEN(dev_priv) >= 7) {
3557                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3558                         I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3559                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3560                         I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3561         }
3562
3563         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3564                 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3565         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3566                 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3567         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3568                 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3569
3570         dev_priv->wm.hw = *results;
3571 }
3572
3573 bool ilk_disable_lp_wm(struct drm_device *dev)
3574 {
3575         struct drm_i915_private *dev_priv = to_i915(dev);
3576
3577         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3578 }
3579
3580 static u8 intel_enabled_dbuf_slices_num(struct drm_i915_private *dev_priv)
3581 {
3582         u8 enabled_slices;
3583
3584         /* Slice 1 will always be enabled */
3585         enabled_slices = 1;
3586
3587         /* Gen prior to GEN11 have only one DBuf slice */
3588         if (INTEL_GEN(dev_priv) < 11)
3589                 return enabled_slices;
3590
3591         if (I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE)
3592                 enabled_slices++;
3593
3594         return enabled_slices;
3595 }
3596
3597 /*
3598  * FIXME: We still don't have the proper code detect if we need to apply the WA,
3599  * so assume we'll always need it in order to avoid underruns.
3600  */
3601 static bool skl_needs_memory_bw_wa(struct intel_atomic_state *state)
3602 {
3603         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3604
3605         if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
3606                 return true;
3607
3608         return false;
3609 }
3610
3611 static bool
3612 intel_has_sagv(struct drm_i915_private *dev_priv)
3613 {
3614         if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) ||
3615             IS_CANNONLAKE(dev_priv))
3616                 return true;
3617
3618         if (IS_SKYLAKE(dev_priv) &&
3619             dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED)
3620                 return true;
3621
3622         return false;
3623 }
3624
3625 /*
3626  * SAGV dynamically adjusts the system agent voltage and clock frequencies
3627  * depending on power and performance requirements. The display engine access
3628  * to system memory is blocked during the adjustment time. Because of the
3629  * blocking time, having this enabled can cause full system hangs and/or pipe
3630  * underruns if we don't meet all of the following requirements:
3631  *
3632  *  - <= 1 pipe enabled
3633  *  - All planes can enable watermarks for latencies >= SAGV engine block time
3634  *  - We're not using an interlaced display configuration
3635  */
3636 int
3637 intel_enable_sagv(struct drm_i915_private *dev_priv)
3638 {
3639         int ret;
3640
3641         if (!intel_has_sagv(dev_priv))
3642                 return 0;
3643
3644         if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3645                 return 0;
3646
3647         DRM_DEBUG_KMS("Enabling the SAGV\n");
3648         mutex_lock(&dev_priv->pcu_lock);
3649
3650         ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3651                                       GEN9_SAGV_ENABLE);
3652
3653         /* We don't need to wait for the SAGV when enabling */
3654         mutex_unlock(&dev_priv->pcu_lock);
3655
3656         /*
3657          * Some skl systems, pre-release machines in particular,
3658          * don't actually have an SAGV.
3659          */
3660         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3661                 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3662                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3663                 return 0;
3664         } else if (ret < 0) {
3665                 DRM_ERROR("Failed to enable the SAGV\n");
3666                 return ret;
3667         }
3668
3669         dev_priv->sagv_status = I915_SAGV_ENABLED;
3670         return 0;
3671 }
3672
3673 int
3674 intel_disable_sagv(struct drm_i915_private *dev_priv)
3675 {
3676         int ret;
3677
3678         if (!intel_has_sagv(dev_priv))
3679                 return 0;
3680
3681         if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3682                 return 0;
3683
3684         DRM_DEBUG_KMS("Disabling the SAGV\n");
3685         mutex_lock(&dev_priv->pcu_lock);
3686
3687         /* bspec says to keep retrying for at least 1 ms */
3688         ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3689                                 GEN9_SAGV_DISABLE,
3690                                 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3691                                 1);
3692         mutex_unlock(&dev_priv->pcu_lock);
3693
3694         /*
3695          * Some skl systems, pre-release machines in particular,
3696          * don't actually have an SAGV.
3697          */
3698         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3699                 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3700                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3701                 return 0;
3702         } else if (ret < 0) {
3703                 DRM_ERROR("Failed to disable the SAGV (%d)\n", ret);
3704                 return ret;
3705         }
3706
3707         dev_priv->sagv_status = I915_SAGV_DISABLED;
3708         return 0;
3709 }
3710
3711 bool intel_can_enable_sagv(struct drm_atomic_state *state)
3712 {
3713         struct drm_device *dev = state->dev;
3714         struct drm_i915_private *dev_priv = to_i915(dev);
3715         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3716         struct intel_crtc *crtc;
3717         struct intel_plane *plane;
3718         struct intel_crtc_state *cstate;
3719         enum pipe pipe;
3720         int level, latency;
3721         int sagv_block_time_us;
3722
3723         if (!intel_has_sagv(dev_priv))
3724                 return false;
3725
3726         if (IS_GEN9(dev_priv))
3727                 sagv_block_time_us = 30;
3728         else if (IS_GEN10(dev_priv))
3729                 sagv_block_time_us = 20;
3730         else
3731                 sagv_block_time_us = 10;
3732
3733         /*
3734          * SKL+ workaround: bspec recommends we disable the SAGV when we have
3735          * more then one pipe enabled
3736          *
3737          * If there are no active CRTCs, no additional checks need be performed
3738          */
3739         if (hweight32(intel_state->active_crtcs) == 0)
3740                 return true;
3741         else if (hweight32(intel_state->active_crtcs) > 1)
3742                 return false;
3743
3744         /* Since we're now guaranteed to only have one active CRTC... */
3745         pipe = ffs(intel_state->active_crtcs) - 1;
3746         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
3747         cstate = to_intel_crtc_state(crtc->base.state);
3748
3749         if (crtc->base.state->adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3750                 return false;
3751
3752         for_each_intel_plane_on_crtc(dev, crtc, plane) {
3753                 struct skl_plane_wm *wm =
3754                         &cstate->wm.skl.optimal.planes[plane->id];
3755
3756                 /* Skip this plane if it's not enabled */
3757                 if (!wm->wm[0].plane_en)
3758                         continue;
3759
3760                 /* Find the highest enabled wm level for this plane */
3761                 for (level = ilk_wm_max_level(dev_priv);
3762                      !wm->wm[level].plane_en; --level)
3763                      { }
3764
3765                 latency = dev_priv->wm.skl_latency[level];
3766
3767                 if (skl_needs_memory_bw_wa(intel_state) &&
3768                     plane->base.state->fb->modifier ==
3769                     I915_FORMAT_MOD_X_TILED)
3770                         latency += 15;
3771
3772                 /*
3773                  * If any of the planes on this pipe don't enable wm levels that
3774                  * incur memory latencies higher than sagv_block_time_us we
3775                  * can't enable the SAGV.
3776                  */
3777                 if (latency < sagv_block_time_us)
3778                         return false;
3779         }
3780
3781         return true;
3782 }
3783
3784 static u16 intel_get_ddb_size(struct drm_i915_private *dev_priv,
3785                               const struct intel_crtc_state *cstate,
3786                               const unsigned int total_data_rate,
3787                               const int num_active,
3788                               struct skl_ddb_allocation *ddb)
3789 {
3790         const struct drm_display_mode *adjusted_mode;
3791         u64 total_data_bw;
3792         u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
3793
3794         WARN_ON(ddb_size == 0);
3795
3796         if (INTEL_GEN(dev_priv) < 11)
3797                 return ddb_size - 4; /* 4 blocks for bypass path allocation */
3798
3799         adjusted_mode = &cstate->base.adjusted_mode;
3800         total_data_bw = (u64)total_data_rate * drm_mode_vrefresh(adjusted_mode);
3801
3802         /*
3803          * 12GB/s is maximum BW supported by single DBuf slice.
3804          */
3805         if (total_data_bw >= GBps(12) || num_active > 1) {
3806                 ddb->enabled_slices = 2;
3807         } else {
3808                 ddb->enabled_slices = 1;
3809                 ddb_size /= 2;
3810         }
3811
3812         return ddb_size;
3813 }
3814
3815 static void
3816 skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
3817                                    const struct intel_crtc_state *cstate,
3818                                    const unsigned int total_data_rate,
3819                                    struct skl_ddb_allocation *ddb,
3820                                    struct skl_ddb_entry *alloc, /* out */
3821                                    int *num_active /* out */)
3822 {
3823         struct drm_atomic_state *state = cstate->base.state;
3824         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3825         struct drm_i915_private *dev_priv = to_i915(dev);
3826         struct drm_crtc *for_crtc = cstate->base.crtc;
3827         const struct drm_crtc_state *crtc_state;
3828         const struct drm_crtc *crtc;
3829         u32 pipe_width = 0, total_width = 0, width_before_pipe = 0;
3830         enum pipe for_pipe = to_intel_crtc(for_crtc)->pipe;
3831         u16 ddb_size;
3832         u32 i;
3833
3834         if (WARN_ON(!state) || !cstate->base.active) {
3835                 alloc->start = 0;
3836                 alloc->end = 0;
3837                 *num_active = hweight32(dev_priv->active_crtcs);
3838                 return;
3839         }
3840
3841         if (intel_state->active_pipe_changes)
3842                 *num_active = hweight32(intel_state->active_crtcs);
3843         else
3844                 *num_active = hweight32(dev_priv->active_crtcs);
3845
3846         ddb_size = intel_get_ddb_size(dev_priv, cstate, total_data_rate,
3847                                       *num_active, ddb);
3848
3849         /*
3850          * If the state doesn't change the active CRTC's or there is no
3851          * modeset request, then there's no need to recalculate;
3852          * the existing pipe allocation limits should remain unchanged.
3853          * Note that we're safe from racing commits since any racing commit
3854          * that changes the active CRTC list or do modeset would need to
3855          * grab _all_ crtc locks, including the one we currently hold.
3856          */
3857         if (!intel_state->active_pipe_changes && !intel_state->modeset) {
3858                 /*
3859                  * alloc may be cleared by clear_intel_crtc_state,
3860                  * copy from old state to be sure
3861                  */
3862                 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
3863                 return;
3864         }
3865
3866         /*
3867          * Watermark/ddb requirement highly depends upon width of the
3868          * framebuffer, So instead of allocating DDB equally among pipes
3869          * distribute DDB based on resolution/width of the display.
3870          */
3871         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
3872                 const struct drm_display_mode *adjusted_mode;
3873                 int hdisplay, vdisplay;
3874                 enum pipe pipe;
3875
3876                 if (!crtc_state->enable)
3877                         continue;
3878
3879                 pipe = to_intel_crtc(crtc)->pipe;
3880                 adjusted_mode = &crtc_state->adjusted_mode;
3881                 drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay);
3882                 total_width += hdisplay;
3883
3884                 if (pipe < for_pipe)
3885                         width_before_pipe += hdisplay;
3886                 else if (pipe == for_pipe)
3887                         pipe_width = hdisplay;
3888         }
3889
3890         alloc->start = ddb_size * width_before_pipe / total_width;
3891         alloc->end = ddb_size * (width_before_pipe + pipe_width) / total_width;
3892 }
3893
3894 static unsigned int skl_cursor_allocation(int num_active)
3895 {
3896         if (num_active == 1)
3897                 return 32;
3898
3899         return 8;
3900 }
3901
3902 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
3903                                        struct skl_ddb_entry *entry, u32 reg)
3904 {
3905         u16 mask;
3906
3907         if (INTEL_GEN(dev_priv) >= 11)
3908                 mask = ICL_DDB_ENTRY_MASK;
3909         else
3910                 mask = SKL_DDB_ENTRY_MASK;
3911         entry->start = reg & mask;
3912         entry->end = (reg >> DDB_ENTRY_END_SHIFT) & mask;
3913
3914         if (entry->end)
3915                 entry->end += 1;
3916 }
3917
3918 static void
3919 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
3920                            const enum pipe pipe,
3921                            const enum plane_id plane_id,
3922                            struct skl_ddb_allocation *ddb /* out */)
3923 {
3924         u32 val, val2 = 0;
3925         int fourcc, pixel_format;
3926
3927         /* Cursor doesn't support NV12/planar, so no extra calculation needed */
3928         if (plane_id == PLANE_CURSOR) {
3929                 val = I915_READ(CUR_BUF_CFG(pipe));
3930                 skl_ddb_entry_init_from_hw(dev_priv,
3931                                            &ddb->plane[pipe][plane_id], val);
3932                 return;
3933         }
3934
3935         val = I915_READ(PLANE_CTL(pipe, plane_id));
3936
3937         /* No DDB allocated for disabled planes */
3938         if (!(val & PLANE_CTL_ENABLE))
3939                 return;
3940
3941         pixel_format = val & PLANE_CTL_FORMAT_MASK;
3942         fourcc = skl_format_to_fourcc(pixel_format,
3943                                       val & PLANE_CTL_ORDER_RGBX,
3944                                       val & PLANE_CTL_ALPHA_MASK);
3945
3946         val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
3947         /*
3948          * FIXME: add proper NV12 support for ICL. Avoid reading unclaimed
3949          * registers for now.
3950          */
3951         if (INTEL_GEN(dev_priv) < 11)
3952                 val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
3953
3954         if (fourcc == DRM_FORMAT_NV12) {
3955                 skl_ddb_entry_init_from_hw(dev_priv,
3956                                            &ddb->plane[pipe][plane_id], val2);
3957                 skl_ddb_entry_init_from_hw(dev_priv,
3958                                            &ddb->uv_plane[pipe][plane_id], val);
3959         } else {
3960                 skl_ddb_entry_init_from_hw(dev_priv,
3961                                            &ddb->plane[pipe][plane_id], val);
3962         }
3963 }
3964
3965 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
3966                           struct skl_ddb_allocation *ddb /* out */)
3967 {
3968         struct intel_crtc *crtc;
3969
3970         memset(ddb, 0, sizeof(*ddb));
3971
3972         ddb->enabled_slices = intel_enabled_dbuf_slices_num(dev_priv);
3973
3974         for_each_intel_crtc(&dev_priv->drm, crtc) {
3975                 enum intel_display_power_domain power_domain;
3976                 enum plane_id plane_id;
3977                 enum pipe pipe = crtc->pipe;
3978
3979                 power_domain = POWER_DOMAIN_PIPE(pipe);
3980                 if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
3981                         continue;
3982
3983                 for_each_plane_id_on_crtc(crtc, plane_id)
3984                         skl_ddb_get_hw_plane_state(dev_priv, pipe,
3985                                                    plane_id, ddb);
3986
3987                 intel_display_power_put(dev_priv, power_domain);
3988         }
3989 }
3990
3991 /*
3992  * Determines the downscale amount of a plane for the purposes of watermark calculations.
3993  * The bspec defines downscale amount as:
3994  *
3995  * """
3996  * Horizontal down scale amount = maximum[1, Horizontal source size /
3997  *                                           Horizontal destination size]
3998  * Vertical down scale amount = maximum[1, Vertical source size /
3999  *                                         Vertical destination size]
4000  * Total down scale amount = Horizontal down scale amount *
4001  *                           Vertical down scale amount
4002  * """
4003  *
4004  * Return value is provided in 16.16 fixed point form to retain fractional part.
4005  * Caller should take care of dividing & rounding off the value.
4006  */
4007 static uint_fixed_16_16_t
4008 skl_plane_downscale_amount(const struct intel_crtc_state *cstate,
4009                            const struct intel_plane_state *pstate)
4010 {
4011         struct intel_plane *plane = to_intel_plane(pstate->base.plane);
4012         uint32_t src_w, src_h, dst_w, dst_h;
4013         uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4014         uint_fixed_16_16_t downscale_h, downscale_w;
4015
4016         if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
4017                 return u32_to_fixed16(0);
4018
4019         /* n.b., src is 16.16 fixed point, dst is whole integer */
4020         if (plane->id == PLANE_CURSOR) {
4021                 /*
4022                  * Cursors only support 0/180 degree rotation,
4023                  * hence no need to account for rotation here.
4024                  */
4025                 src_w = pstate->base.src_w >> 16;
4026                 src_h = pstate->base.src_h >> 16;
4027                 dst_w = pstate->base.crtc_w;
4028                 dst_h = pstate->base.crtc_h;
4029         } else {
4030                 /*
4031                  * Src coordinates are already rotated by 270 degrees for
4032                  * the 90/270 degree plane rotation cases (to match the
4033                  * GTT mapping), hence no need to account for rotation here.
4034                  */
4035                 src_w = drm_rect_width(&pstate->base.src) >> 16;
4036                 src_h = drm_rect_height(&pstate->base.src) >> 16;
4037                 dst_w = drm_rect_width(&pstate->base.dst);
4038                 dst_h = drm_rect_height(&pstate->base.dst);
4039         }
4040
4041         fp_w_ratio = div_fixed16(src_w, dst_w);
4042         fp_h_ratio = div_fixed16(src_h, dst_h);
4043         downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4044         downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4045
4046         return mul_fixed16(downscale_w, downscale_h);
4047 }
4048
4049 static uint_fixed_16_16_t
4050 skl_pipe_downscale_amount(const struct intel_crtc_state *crtc_state)
4051 {
4052         uint_fixed_16_16_t pipe_downscale = u32_to_fixed16(1);
4053
4054         if (!crtc_state->base.enable)
4055                 return pipe_downscale;
4056
4057         if (crtc_state->pch_pfit.enabled) {
4058                 uint32_t src_w, src_h, dst_w, dst_h;
4059                 uint32_t pfit_size = crtc_state->pch_pfit.size;
4060                 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4061                 uint_fixed_16_16_t downscale_h, downscale_w;
4062
4063                 src_w = crtc_state->pipe_src_w;
4064                 src_h = crtc_state->pipe_src_h;
4065                 dst_w = pfit_size >> 16;
4066                 dst_h = pfit_size & 0xffff;
4067
4068                 if (!dst_w || !dst_h)
4069                         return pipe_downscale;
4070
4071                 fp_w_ratio = div_fixed16(src_w, dst_w);
4072                 fp_h_ratio = div_fixed16(src_h, dst_h);
4073                 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4074                 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4075
4076                 pipe_downscale = mul_fixed16(downscale_w, downscale_h);
4077         }
4078
4079         return pipe_downscale;
4080 }
4081
4082 int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
4083                                   struct intel_crtc_state *cstate)
4084 {
4085         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4086         struct drm_crtc_state *crtc_state = &cstate->base;
4087         struct drm_atomic_state *state = crtc_state->state;
4088         struct drm_plane *plane;
4089         const struct drm_plane_state *pstate;
4090         struct intel_plane_state *intel_pstate;
4091         int crtc_clock, dotclk;
4092         uint32_t pipe_max_pixel_rate;
4093         uint_fixed_16_16_t pipe_downscale;
4094         uint_fixed_16_16_t max_downscale = u32_to_fixed16(1);
4095
4096         if (!cstate->base.enable)
4097                 return 0;
4098
4099         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
4100                 uint_fixed_16_16_t plane_downscale;
4101                 uint_fixed_16_16_t fp_9_div_8 = div_fixed16(9, 8);
4102                 int bpp;
4103
4104                 if (!intel_wm_plane_visible(cstate,
4105                                             to_intel_plane_state(pstate)))
4106                         continue;
4107
4108                 if (WARN_ON(!pstate->fb))
4109                         return -EINVAL;
4110
4111                 intel_pstate = to_intel_plane_state(pstate);
4112                 plane_downscale = skl_plane_downscale_amount(cstate,
4113                                                              intel_pstate);
4114                 bpp = pstate->fb->format->cpp[0] * 8;
4115                 if (bpp == 64)
4116                         plane_downscale = mul_fixed16(plane_downscale,
4117                                                       fp_9_div_8);
4118
4119                 max_downscale = max_fixed16(plane_downscale, max_downscale);
4120         }
4121         pipe_downscale = skl_pipe_downscale_amount(cstate);
4122
4123         pipe_downscale = mul_fixed16(pipe_downscale, max_downscale);
4124
4125         crtc_clock = crtc_state->adjusted_mode.crtc_clock;
4126         dotclk = to_intel_atomic_state(state)->cdclk.logical.cdclk;
4127
4128         if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
4129                 dotclk *= 2;
4130
4131         pipe_max_pixel_rate = div_round_up_u32_fixed16(dotclk, pipe_downscale);
4132
4133         if (pipe_max_pixel_rate < crtc_clock) {
4134                 DRM_DEBUG_KMS("Max supported pixel clock with scaling exceeded\n");
4135                 return -EINVAL;
4136         }
4137
4138         return 0;
4139 }
4140
4141 static unsigned int
4142 skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
4143                              const struct drm_plane_state *pstate,
4144                              const int plane)
4145 {
4146         struct intel_plane *intel_plane = to_intel_plane(pstate->plane);
4147         struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
4148         uint32_t data_rate;
4149         uint32_t width = 0, height = 0;
4150         struct drm_framebuffer *fb;
4151         u32 format;
4152         uint_fixed_16_16_t down_scale_amount;
4153
4154         if (!intel_pstate->base.visible)
4155                 return 0;
4156
4157         fb = pstate->fb;
4158         format = fb->format->format;
4159
4160         if (intel_plane->id == PLANE_CURSOR)
4161                 return 0;
4162         if (plane == 1 && format != DRM_FORMAT_NV12)
4163                 return 0;
4164
4165         /*
4166          * Src coordinates are already rotated by 270 degrees for
4167          * the 90/270 degree plane rotation cases (to match the
4168          * GTT mapping), hence no need to account for rotation here.
4169          */
4170         width = drm_rect_width(&intel_pstate->base.src) >> 16;
4171         height = drm_rect_height(&intel_pstate->base.src) >> 16;
4172
4173         /* UV plane does 1/2 pixel sub-sampling */
4174         if (plane == 1 && format == DRM_FORMAT_NV12) {
4175                 width /= 2;
4176                 height /= 2;
4177         }
4178
4179         data_rate = width * height * fb->format->cpp[plane];
4180
4181         down_scale_amount = skl_plane_downscale_amount(cstate, intel_pstate);
4182
4183         return mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4184 }
4185
4186 /*
4187  * We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching
4188  * a 8192x4096@32bpp framebuffer:
4189  *   3 * 4096 * 8192  * 4 < 2^32
4190  */
4191 static unsigned int
4192 skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate,
4193                                  unsigned int *plane_data_rate,
4194                                  unsigned int *uv_plane_data_rate)
4195 {
4196         struct drm_crtc_state *cstate = &intel_cstate->base;
4197         struct drm_atomic_state *state = cstate->state;
4198         struct drm_plane *plane;
4199         const struct drm_plane_state *pstate;
4200         unsigned int total_data_rate = 0;
4201
4202         if (WARN_ON(!state))
4203                 return 0;
4204
4205         /* Calculate and cache data rate for each plane */
4206         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, cstate) {
4207                 enum plane_id plane_id = to_intel_plane(plane)->id;
4208                 unsigned int rate;
4209
4210                 /* packed/y */
4211                 rate = skl_plane_relative_data_rate(intel_cstate,
4212                                                     pstate, 0);
4213                 plane_data_rate[plane_id] = rate;
4214
4215                 total_data_rate += rate;
4216
4217                 /* uv-plane */
4218                 rate = skl_plane_relative_data_rate(intel_cstate,
4219                                                     pstate, 1);
4220                 uv_plane_data_rate[plane_id] = rate;
4221
4222                 total_data_rate += rate;
4223         }
4224
4225         return total_data_rate;
4226 }
4227
4228 static uint16_t
4229 skl_ddb_min_alloc(const struct drm_plane_state *pstate, const int plane)
4230 {
4231         struct drm_framebuffer *fb = pstate->fb;
4232         struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
4233         uint32_t src_w, src_h;
4234         uint32_t min_scanlines = 8;
4235         uint8_t plane_bpp;
4236
4237         if (WARN_ON(!fb))
4238                 return 0;
4239
4240         /* For packed formats, and uv-plane, return 0 */
4241         if (plane == 1 && fb->format->format != DRM_FORMAT_NV12)
4242                 return 0;
4243
4244         /* For Non Y-tile return 8-blocks */
4245         if (fb->modifier != I915_FORMAT_MOD_Y_TILED &&
4246             fb->modifier != I915_FORMAT_MOD_Yf_TILED &&
4247             fb->modifier != I915_FORMAT_MOD_Y_TILED_CCS &&
4248             fb->modifier != I915_FORMAT_MOD_Yf_TILED_CCS)
4249                 return 8;
4250
4251         /*
4252          * Src coordinates are already rotated by 270 degrees for
4253          * the 90/270 degree plane rotation cases (to match the
4254          * GTT mapping), hence no need to account for rotation here.
4255          */
4256         src_w = drm_rect_width(&intel_pstate->base.src) >> 16;
4257         src_h = drm_rect_height(&intel_pstate->base.src) >> 16;
4258
4259         /* Halve UV plane width and height for NV12 */
4260         if (plane == 1) {
4261                 src_w /= 2;
4262                 src_h /= 2;
4263         }
4264
4265         plane_bpp = fb->format->cpp[plane];
4266
4267         if (drm_rotation_90_or_270(pstate->rotation)) {
4268                 switch (plane_bpp) {
4269                 case 1:
4270                         min_scanlines = 32;
4271                         break;
4272                 case 2:
4273                         min_scanlines = 16;
4274                         break;
4275                 case 4:
4276                         min_scanlines = 8;
4277                         break;
4278                 case 8:
4279                         min_scanlines = 4;
4280                         break;
4281                 default:
4282                         WARN(1, "Unsupported pixel depth %u for rotation",
4283                              plane_bpp);
4284                         min_scanlines = 32;
4285                 }
4286         }
4287
4288         return DIV_ROUND_UP((4 * src_w * plane_bpp), 512) * min_scanlines/4 + 3;
4289 }
4290
4291 static void
4292 skl_ddb_calc_min(const struct intel_crtc_state *cstate, int num_active,
4293                  uint16_t *minimum, uint16_t *uv_minimum)
4294 {
4295         const struct drm_plane_state *pstate;
4296         struct drm_plane *plane;
4297
4298         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, &cstate->base) {
4299                 enum plane_id plane_id = to_intel_plane(plane)->id;
4300
4301                 if (plane_id == PLANE_CURSOR)
4302                         continue;
4303
4304                 if (!pstate->visible)
4305                         continue;
4306
4307                 minimum[plane_id] = skl_ddb_min_alloc(pstate, 0);
4308                 uv_minimum[plane_id] = skl_ddb_min_alloc(pstate, 1);
4309         }
4310
4311         minimum[PLANE_CURSOR] = skl_cursor_allocation(num_active);
4312 }
4313
4314 static int
4315 skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
4316                       struct skl_ddb_allocation *ddb /* out */)
4317 {
4318         struct drm_atomic_state *state = cstate->base.state;
4319         struct drm_crtc *crtc = cstate->base.crtc;
4320         struct drm_device *dev = crtc->dev;
4321         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4322         enum pipe pipe = intel_crtc->pipe;
4323         struct skl_ddb_entry *alloc = &cstate->wm.skl.ddb;
4324         uint16_t alloc_size, start;
4325         uint16_t minimum[I915_MAX_PLANES] = {};
4326         uint16_t uv_minimum[I915_MAX_PLANES] = {};
4327         unsigned int total_data_rate;
4328         enum plane_id plane_id;
4329         int num_active;
4330         unsigned int plane_data_rate[I915_MAX_PLANES] = {};
4331         unsigned int uv_plane_data_rate[I915_MAX_PLANES] = {};
4332         uint16_t total_min_blocks = 0;
4333
4334         /* Clear the partitioning for disabled planes. */
4335         memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
4336         memset(ddb->uv_plane[pipe], 0, sizeof(ddb->uv_plane[pipe]));
4337
4338         if (WARN_ON(!state))
4339                 return 0;
4340
4341         if (!cstate->base.active) {
4342                 alloc->start = alloc->end = 0;
4343                 return 0;
4344         }
4345
4346         total_data_rate = skl_get_total_relative_data_rate(cstate,
4347                                                            plane_data_rate,
4348                                                            uv_plane_data_rate);
4349         skl_ddb_get_pipe_allocation_limits(dev, cstate, total_data_rate, ddb,
4350                                            alloc, &num_active);
4351         alloc_size = skl_ddb_entry_size(alloc);
4352         if (alloc_size == 0)
4353                 return 0;
4354
4355         skl_ddb_calc_min(cstate, num_active, minimum, uv_minimum);
4356
4357         /*
4358          * 1. Allocate the mininum required blocks for each active plane
4359          * and allocate the cursor, it doesn't require extra allocation
4360          * proportional to the data rate.
4361          */
4362
4363         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4364                 total_min_blocks += minimum[plane_id];
4365                 total_min_blocks += uv_minimum[plane_id];
4366         }
4367
4368         if (total_min_blocks > alloc_size) {
4369                 DRM_DEBUG_KMS("Requested display configuration exceeds system DDB limitations");
4370                 DRM_DEBUG_KMS("minimum required %d/%d\n", total_min_blocks,
4371                                                         alloc_size);
4372                 return -EINVAL;
4373         }
4374
4375         alloc_size -= total_min_blocks;
4376         ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - minimum[PLANE_CURSOR];
4377         ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
4378
4379         /*
4380          * 2. Distribute the remaining space in proportion to the amount of
4381          * data each plane needs to fetch from memory.
4382          *
4383          * FIXME: we may not allocate every single block here.
4384          */
4385         if (total_data_rate == 0)
4386                 return 0;
4387
4388         start = alloc->start;
4389         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4390                 unsigned int data_rate, uv_data_rate;
4391                 uint16_t plane_blocks, uv_plane_blocks;
4392
4393                 if (plane_id == PLANE_CURSOR)
4394                         continue;
4395
4396                 data_rate = plane_data_rate[plane_id];
4397
4398                 /*
4399                  * allocation for (packed formats) or (uv-plane part of planar format):
4400                  * promote the expression to 64 bits to avoid overflowing, the
4401                  * result is < available as data_rate / total_data_rate < 1
4402                  */
4403                 plane_blocks = minimum[plane_id];
4404                 plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
4405                                         total_data_rate);
4406
4407                 /* Leave disabled planes at (0,0) */
4408                 if (data_rate) {
4409                         ddb->plane[pipe][plane_id].start = start;
4410                         ddb->plane[pipe][plane_id].end = start + plane_blocks;
4411                 }
4412
4413                 start += plane_blocks;
4414
4415                 /* Allocate DDB for UV plane for planar format/NV12 */
4416                 uv_data_rate = uv_plane_data_rate[plane_id];
4417
4418                 uv_plane_blocks = uv_minimum[plane_id];
4419                 uv_plane_blocks += div_u64((uint64_t)alloc_size * uv_data_rate,
4420                                            total_data_rate);
4421
4422                 if (uv_data_rate) {
4423                         ddb->uv_plane[pipe][plane_id].start = start;
4424                         ddb->uv_plane[pipe][plane_id].end =
4425                                 start + uv_plane_blocks;
4426                 }
4427
4428                 start += uv_plane_blocks;
4429         }
4430
4431         return 0;
4432 }
4433
4434 /*
4435  * The max latency should be 257 (max the punit can code is 255 and we add 2us
4436  * for the read latency) and cpp should always be <= 8, so that
4437  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
4438  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
4439 */
4440 static uint_fixed_16_16_t
4441 skl_wm_method1(const struct drm_i915_private *dev_priv, uint32_t pixel_rate,
4442                uint8_t cpp, uint32_t latency, uint32_t dbuf_block_size)
4443 {
4444         uint32_t wm_intermediate_val;
4445         uint_fixed_16_16_t ret;
4446
4447         if (latency == 0)
4448                 return FP_16_16_MAX;
4449
4450         wm_intermediate_val = latency * pixel_rate * cpp;
4451         ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
4452
4453         if (INTEL_GEN(dev_priv) >= 10)
4454                 ret = add_fixed16_u32(ret, 1);
4455
4456         return ret;
4457 }
4458
4459 static uint_fixed_16_16_t skl_wm_method2(uint32_t pixel_rate,
4460                         uint32_t pipe_htotal,
4461                         uint32_t latency,
4462                         uint_fixed_16_16_t plane_blocks_per_line)
4463 {
4464         uint32_t wm_intermediate_val;
4465         uint_fixed_16_16_t ret;
4466
4467         if (latency == 0)
4468                 return FP_16_16_MAX;
4469
4470         wm_intermediate_val = latency * pixel_rate;
4471         wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
4472                                            pipe_htotal * 1000);
4473         ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
4474         return ret;
4475 }
4476
4477 static uint_fixed_16_16_t
4478 intel_get_linetime_us(struct intel_crtc_state *cstate)
4479 {
4480         uint32_t pixel_rate;
4481         uint32_t crtc_htotal;
4482         uint_fixed_16_16_t linetime_us;
4483
4484         if (!cstate->base.active)
4485                 return u32_to_fixed16(0);
4486
4487         pixel_rate = cstate->pixel_rate;
4488
4489         if (WARN_ON(pixel_rate == 0))
4490                 return u32_to_fixed16(0);
4491
4492         crtc_htotal = cstate->base.adjusted_mode.crtc_htotal;
4493         linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
4494
4495         return linetime_us;
4496 }
4497
4498 static uint32_t
4499 skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cstate,
4500                               const struct intel_plane_state *pstate)
4501 {
4502         uint64_t adjusted_pixel_rate;
4503         uint_fixed_16_16_t downscale_amount;
4504
4505         /* Shouldn't reach here on disabled planes... */
4506         if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
4507                 return 0;
4508
4509         /*
4510          * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
4511          * with additional adjustments for plane-specific scaling.
4512          */
4513         adjusted_pixel_rate = cstate->pixel_rate;
4514         downscale_amount = skl_plane_downscale_amount(cstate, pstate);
4515
4516         return mul_round_up_u32_fixed16(adjusted_pixel_rate,
4517                                             downscale_amount);
4518 }
4519
4520 static int
4521 skl_compute_plane_wm_params(const struct drm_i915_private *dev_priv,
4522                             struct intel_crtc_state *cstate,
4523                             const struct intel_plane_state *intel_pstate,
4524                             struct skl_wm_params *wp, int plane_id)
4525 {
4526         struct intel_plane *plane = to_intel_plane(intel_pstate->base.plane);
4527         const struct drm_plane_state *pstate = &intel_pstate->base;
4528         const struct drm_framebuffer *fb = pstate->fb;
4529         uint32_t interm_pbpl;
4530         struct intel_atomic_state *state =
4531                 to_intel_atomic_state(cstate->base.state);
4532         bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state);
4533
4534         if (!intel_wm_plane_visible(cstate, intel_pstate))
4535                 return 0;
4536
4537         /* only NV12 format has two planes */
4538         if (plane_id == 1 && fb->format->format != DRM_FORMAT_NV12) {
4539                 DRM_DEBUG_KMS("Non NV12 format have single plane\n");
4540                 return -EINVAL;
4541         }
4542
4543         wp->y_tiled = fb->modifier == I915_FORMAT_MOD_Y_TILED ||
4544                       fb->modifier == I915_FORMAT_MOD_Yf_TILED ||
4545                       fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4546                       fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4547         wp->x_tiled = fb->modifier == I915_FORMAT_MOD_X_TILED;
4548         wp->rc_surface = fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4549                          fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4550         wp->is_planar = fb->format->format == DRM_FORMAT_NV12;
4551
4552         if (plane->id == PLANE_CURSOR) {
4553                 wp->width = intel_pstate->base.crtc_w;
4554         } else {
4555                 /*
4556                  * Src coordinates are already rotated by 270 degrees for
4557                  * the 90/270 degree plane rotation cases (to match the
4558                  * GTT mapping), hence no need to account for rotation here.
4559                  */
4560                 wp->width = drm_rect_width(&intel_pstate->base.src) >> 16;
4561         }
4562
4563         if (plane_id == 1 && wp->is_planar)
4564                 wp->width /= 2;
4565
4566         wp->cpp = fb->format->cpp[plane_id];
4567         wp->plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate,
4568                                                              intel_pstate);
4569
4570         if (INTEL_GEN(dev_priv) >= 11 &&
4571             fb->modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 8)
4572                 wp->dbuf_block_size = 256;
4573         else
4574                 wp->dbuf_block_size = 512;
4575
4576         if (drm_rotation_90_or_270(pstate->rotation)) {
4577
4578                 switch (wp->cpp) {
4579                 case 1:
4580                         wp->y_min_scanlines = 16;
4581                         break;
4582                 case 2:
4583                         wp->y_min_scanlines = 8;
4584                         break;
4585                 case 4:
4586                         wp->y_min_scanlines = 4;
4587                         break;
4588                 default:
4589                         MISSING_CASE(wp->cpp);
4590                         return -EINVAL;
4591                 }
4592         } else {
4593                 wp->y_min_scanlines = 4;
4594         }
4595
4596         if (apply_memory_bw_wa)
4597                 wp->y_min_scanlines *= 2;
4598
4599         wp->plane_bytes_per_line = wp->width * wp->cpp;
4600         if (wp->y_tiled) {
4601                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
4602                                            wp->y_min_scanlines,
4603                                            wp->dbuf_block_size);
4604
4605                 if (INTEL_GEN(dev_priv) >= 10)
4606                         interm_pbpl++;
4607
4608                 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
4609                                                         wp->y_min_scanlines);
4610         } else if (wp->x_tiled && IS_GEN9(dev_priv)) {
4611                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
4612                                            wp->dbuf_block_size);
4613                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4614         } else {
4615                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
4616                                            wp->dbuf_block_size) + 1;
4617                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4618         }
4619
4620         wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
4621                                              wp->plane_blocks_per_line);
4622         wp->linetime_us = fixed16_to_u32_round_up(
4623                                         intel_get_linetime_us(cstate));
4624
4625         return 0;
4626 }
4627
4628 static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
4629                                 struct intel_crtc_state *cstate,
4630                                 const struct intel_plane_state *intel_pstate,
4631                                 uint16_t ddb_allocation,
4632                                 int level,
4633                                 const struct skl_wm_params *wp,
4634                                 const struct skl_wm_level *result_prev,
4635                                 struct skl_wm_level *result /* out */)
4636 {
4637         const struct drm_plane_state *pstate = &intel_pstate->base;
4638         uint32_t latency = dev_priv->wm.skl_latency[level];
4639         uint_fixed_16_16_t method1, method2;
4640         uint_fixed_16_16_t selected_result;
4641         uint32_t res_blocks, res_lines;
4642         struct intel_atomic_state *state =
4643                 to_intel_atomic_state(cstate->base.state);
4644         bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state);
4645         uint32_t min_disp_buf_needed;
4646
4647         if (latency == 0 ||
4648             !intel_wm_plane_visible(cstate, intel_pstate)) {
4649                 result->plane_en = false;
4650                 return 0;
4651         }
4652
4653         /* Display WA #1141: kbl,cfl */
4654         if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) ||
4655             IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0)) &&
4656             dev_priv->ipc_enabled)
4657                 latency += 4;
4658
4659         if (apply_memory_bw_wa && wp->x_tiled)
4660                 latency += 15;
4661
4662         method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
4663                                  wp->cpp, latency, wp->dbuf_block_size);
4664         method2 = skl_wm_method2(wp->plane_pixel_rate,
4665                                  cstate->base.adjusted_mode.crtc_htotal,
4666                                  latency,
4667                                  wp->plane_blocks_per_line);
4668
4669         if (wp->y_tiled) {
4670                 selected_result = max_fixed16(method2, wp->y_tile_minimum);
4671         } else {
4672                 if ((wp->cpp * cstate->base.adjusted_mode.crtc_htotal /
4673                      wp->dbuf_block_size < 1) &&
4674                      (wp->plane_bytes_per_line / wp->dbuf_block_size < 1))
4675                         selected_result = method2;
4676                 else if (ddb_allocation >=
4677                          fixed16_to_u32_round_up(wp->plane_blocks_per_line))
4678                         selected_result = min_fixed16(method1, method2);
4679                 else if (latency >= wp->linetime_us)
4680                         selected_result = min_fixed16(method1, method2);
4681                 else
4682                         selected_result = method1;
4683         }
4684
4685         res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
4686         res_lines = div_round_up_fixed16(selected_result,
4687                                          wp->plane_blocks_per_line);
4688
4689         /* Display WA #1125: skl,bxt,kbl,glk */
4690         if (level == 0 && wp->rc_surface)
4691                 res_blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
4692
4693         /* Display WA #1126: skl,bxt,kbl,glk */
4694         if (level >= 1 && level <= 7) {
4695                 if (wp->y_tiled) {
4696                         res_blocks += fixed16_to_u32_round_up(
4697                                                         wp->y_tile_minimum);
4698                         res_lines += wp->y_min_scanlines;
4699                 } else {
4700                         res_blocks++;
4701                 }
4702
4703                 /*
4704                  * Make sure result blocks for higher latency levels are atleast
4705                  * as high as level below the current level.
4706                  * Assumption in DDB algorithm optimization for special cases.
4707                  * Also covers Display WA #1125 for RC.
4708                  */
4709                 if (result_prev->plane_res_b > res_blocks)
4710                         res_blocks = result_prev->plane_res_b;
4711         }
4712
4713         if (INTEL_GEN(dev_priv) >= 11) {
4714                 if (wp->y_tiled) {
4715                         uint32_t extra_lines;
4716                         uint_fixed_16_16_t fp_min_disp_buf_needed;
4717
4718                         if (res_lines % wp->y_min_scanlines == 0)
4719                                 extra_lines = wp->y_min_scanlines;
4720                         else
4721                                 extra_lines = wp->y_min_scanlines * 2 -
4722                                               res_lines % wp->y_min_scanlines;
4723
4724                         fp_min_disp_buf_needed = mul_u32_fixed16(res_lines +
4725                                                 extra_lines,
4726                                                 wp->plane_blocks_per_line);
4727                         min_disp_buf_needed = fixed16_to_u32_round_up(
4728                                                 fp_min_disp_buf_needed);
4729                 } else {
4730                         min_disp_buf_needed = DIV_ROUND_UP(res_blocks * 11, 10);
4731                 }
4732         } else {
4733                 min_disp_buf_needed = res_blocks;
4734         }
4735
4736         if ((level > 0 && res_lines > 31) ||
4737             res_blocks >= ddb_allocation ||
4738             min_disp_buf_needed >= ddb_allocation) {
4739                 result->plane_en = false;
4740
4741                 /*
4742                  * If there are no valid level 0 watermarks, then we can't
4743                  * support this display configuration.
4744                  */
4745                 if (level) {
4746                         return 0;
4747                 } else {
4748                         struct drm_plane *plane = pstate->plane;
4749
4750                         DRM_DEBUG_KMS("Requested display configuration exceeds system watermark limitations\n");
4751                         DRM_DEBUG_KMS("[PLANE:%d:%s] blocks required = %u/%u, lines required = %u/31\n",
4752                                       plane->base.id, plane->name,
4753                                       res_blocks, ddb_allocation, res_lines);
4754                         return -EINVAL;
4755                 }
4756         }
4757
4758         /*
4759          * Display WA #826 (SKL:ALL, BXT:ALL) & #1059 (CNL:A)
4760          * disable wm level 1-7 on NV12 planes
4761          */
4762         if (wp->is_planar && level >= 1 &&
4763             (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv) ||
4764              IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0))) {
4765                 result->plane_en = false;
4766                 return 0;
4767         }
4768
4769         /* The number of lines are ignored for the level 0 watermark. */
4770         result->plane_res_b = res_blocks;
4771         result->plane_res_l = res_lines;
4772         result->plane_en = true;
4773
4774         return 0;
4775 }
4776
4777 static int
4778 skl_compute_wm_levels(const struct drm_i915_private *dev_priv,
4779                       struct skl_ddb_allocation *ddb,
4780                       struct intel_crtc_state *cstate,
4781                       const struct intel_plane_state *intel_pstate,
4782                       const struct skl_wm_params *wm_params,
4783                       struct skl_plane_wm *wm,
4784                       int plane_id)
4785 {
4786         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
4787         struct drm_plane *plane = intel_pstate->base.plane;
4788         struct intel_plane *intel_plane = to_intel_plane(plane);
4789         uint16_t ddb_blocks;
4790         enum pipe pipe = intel_crtc->pipe;
4791         int level, max_level = ilk_wm_max_level(dev_priv);
4792         enum plane_id intel_plane_id = intel_plane->id;
4793         int ret;
4794
4795         if (WARN_ON(!intel_pstate->base.fb))
4796                 return -EINVAL;
4797
4798         ddb_blocks = plane_id ?
4799                      skl_ddb_entry_size(&ddb->uv_plane[pipe][intel_plane_id]) :
4800                      skl_ddb_entry_size(&ddb->plane[pipe][intel_plane_id]);
4801
4802         for (level = 0; level <= max_level; level++) {
4803                 struct skl_wm_level *result = plane_id ? &wm->uv_wm[level] :
4804                                                           &wm->wm[level];
4805                 struct skl_wm_level *result_prev;
4806
4807                 if (level)
4808                         result_prev = plane_id ? &wm->uv_wm[level - 1] :
4809                                                   &wm->wm[level - 1];
4810                 else
4811                         result_prev = plane_id ? &wm->uv_wm[0] : &wm->wm[0];
4812
4813                 ret = skl_compute_plane_wm(dev_priv,
4814                                            cstate,
4815                                            intel_pstate,
4816                                            ddb_blocks,
4817                                            level,
4818                                            wm_params,
4819                                            result_prev,
4820                                            result);
4821                 if (ret)
4822                         return ret;
4823         }
4824
4825         if (intel_pstate->base.fb->format->format == DRM_FORMAT_NV12)
4826                 wm->is_planar = true;
4827
4828         return 0;
4829 }
4830
4831 static uint32_t
4832 skl_compute_linetime_wm(struct intel_crtc_state *cstate)
4833 {
4834         struct drm_atomic_state *state = cstate->base.state;
4835         struct drm_i915_private *dev_priv = to_i915(state->dev);
4836         uint_fixed_16_16_t linetime_us;
4837         uint32_t linetime_wm;
4838
4839         linetime_us = intel_get_linetime_us(cstate);
4840
4841         if (is_fixed16_zero(linetime_us))
4842                 return 0;
4843
4844         linetime_wm = fixed16_to_u32_round_up(mul_u32_fixed16(8, linetime_us));
4845
4846         /* Display WA #1135: bxt:ALL GLK:ALL */
4847         if ((IS_BROXTON(dev_priv) || IS_GEMINILAKE(dev_priv)) &&
4848             dev_priv->ipc_enabled)
4849                 linetime_wm /= 2;
4850
4851         return linetime_wm;
4852 }
4853
4854 static void skl_compute_transition_wm(struct intel_crtc_state *cstate,
4855                                       struct skl_wm_params *wp,
4856                                       struct skl_wm_level *wm_l0,
4857                                       uint16_t ddb_allocation,
4858                                       struct skl_wm_level *trans_wm /* out */)
4859 {
4860         struct drm_device *dev = cstate->base.crtc->dev;
4861         const struct drm_i915_private *dev_priv = to_i915(dev);
4862         uint16_t trans_min, trans_y_tile_min;
4863         const uint16_t trans_amount = 10; /* This is configurable amount */
4864         uint16_t trans_offset_b, res_blocks;
4865
4866         if (!cstate->base.active)
4867                 goto exit;
4868
4869         /* Transition WM are not recommended by HW team for GEN9 */
4870         if (INTEL_GEN(dev_priv) <= 9)
4871                 goto exit;
4872
4873         /* Transition WM don't make any sense if ipc is disabled */
4874         if (!dev_priv->ipc_enabled)
4875                 goto exit;
4876
4877         trans_min = 0;
4878         if (INTEL_GEN(dev_priv) >= 10)
4879                 trans_min = 4;
4880
4881         trans_offset_b = trans_min + trans_amount;
4882
4883         if (wp->y_tiled) {
4884                 trans_y_tile_min = (uint16_t) mul_round_up_u32_fixed16(2,
4885                                                         wp->y_tile_minimum);
4886                 res_blocks = max(wm_l0->plane_res_b, trans_y_tile_min) +
4887                                 trans_offset_b;
4888         } else {
4889                 res_blocks = wm_l0->plane_res_b + trans_offset_b;
4890
4891                 /* WA BUG:1938466 add one block for non y-tile planes */
4892                 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0))
4893                         res_blocks += 1;
4894
4895         }
4896
4897         res_blocks += 1;
4898
4899         if (res_blocks < ddb_allocation) {
4900                 trans_wm->plane_res_b = res_blocks;
4901                 trans_wm->plane_en = true;
4902                 return;
4903         }
4904
4905 exit:
4906         trans_wm->plane_en = false;
4907 }
4908
4909 static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
4910                              struct skl_ddb_allocation *ddb,
4911                              struct skl_pipe_wm *pipe_wm)
4912 {
4913         struct drm_device *dev = cstate->base.crtc->dev;
4914         struct drm_crtc_state *crtc_state = &cstate->base;
4915         const struct drm_i915_private *dev_priv = to_i915(dev);
4916         struct drm_plane *plane;
4917         const struct drm_plane_state *pstate;
4918         struct skl_plane_wm *wm;
4919         int ret;
4920
4921         /*
4922          * We'll only calculate watermarks for planes that are actually
4923          * enabled, so make sure all other planes are set as disabled.
4924          */
4925         memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
4926
4927         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
4928                 const struct intel_plane_state *intel_pstate =
4929                                                 to_intel_plane_state(pstate);
4930                 enum plane_id plane_id = to_intel_plane(plane)->id;
4931                 struct skl_wm_params wm_params;
4932                 enum pipe pipe = to_intel_crtc(cstate->base.crtc)->pipe;
4933                 uint16_t ddb_blocks;
4934
4935                 wm = &pipe_wm->planes[plane_id];
4936                 ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][plane_id]);
4937
4938                 ret = skl_compute_plane_wm_params(dev_priv, cstate,
4939                                                   intel_pstate, &wm_params, 0);
4940                 if (ret)
4941                         return ret;
4942
4943                 ret = skl_compute_wm_levels(dev_priv, ddb, cstate,
4944                                             intel_pstate, &wm_params, wm, 0);
4945                 if (ret)
4946                         return ret;
4947
4948                 skl_compute_transition_wm(cstate, &wm_params, &wm->wm[0],
4949                                           ddb_blocks, &wm->trans_wm);
4950
4951                 /* uv plane watermarks must also be validated for NV12/Planar */
4952                 if (wm_params.is_planar) {
4953                         memset(&wm_params, 0, sizeof(struct skl_wm_params));
4954                         wm->is_planar = true;
4955
4956                         ret = skl_compute_plane_wm_params(dev_priv, cstate,
4957                                                           intel_pstate,
4958                                                           &wm_params, 1);
4959                         if (ret)
4960                                 return ret;
4961
4962                         ret = skl_compute_wm_levels(dev_priv, ddb, cstate,
4963                                                     intel_pstate, &wm_params,
4964                                                     wm, 1);
4965                         if (ret)
4966                                 return ret;
4967                 }
4968         }
4969
4970         pipe_wm->linetime = skl_compute_linetime_wm(cstate);
4971
4972         return 0;
4973 }
4974
4975 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
4976                                 i915_reg_t reg,
4977                                 const struct skl_ddb_entry *entry)
4978 {
4979         if (entry->end)
4980                 I915_WRITE(reg, (entry->end - 1) << 16 | entry->start);
4981         else
4982                 I915_WRITE(reg, 0);
4983 }
4984
4985 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
4986                                i915_reg_t reg,
4987                                const struct skl_wm_level *level)
4988 {
4989         uint32_t val = 0;
4990
4991         if (level->plane_en) {
4992                 val |= PLANE_WM_EN;
4993                 val |= level->plane_res_b;
4994                 val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
4995         }
4996
4997         I915_WRITE(reg, val);
4998 }
4999
5000 static void skl_write_plane_wm(struct intel_crtc *intel_crtc,
5001                                const struct skl_plane_wm *wm,
5002                                const struct skl_ddb_allocation *ddb,
5003                                enum plane_id plane_id)
5004 {
5005         struct drm_crtc *crtc = &intel_crtc->base;
5006         struct drm_device *dev = crtc->dev;
5007         struct drm_i915_private *dev_priv = to_i915(dev);
5008         int level, max_level = ilk_wm_max_level(dev_priv);
5009         enum pipe pipe = intel_crtc->pipe;
5010
5011         for (level = 0; level <= max_level; level++) {
5012                 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5013                                    &wm->wm[level]);
5014         }
5015         skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5016                            &wm->trans_wm);
5017
5018         skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
5019                             &ddb->plane[pipe][plane_id]);
5020         /* FIXME: add proper NV12 support for ICL. */
5021         if (INTEL_GEN(dev_priv) >= 11)
5022                 return skl_ddb_entry_write(dev_priv,
5023                                            PLANE_BUF_CFG(pipe, plane_id),
5024                                            &ddb->plane[pipe][plane_id]);
5025         if (wm->is_planar) {
5026                 skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
5027                                     &ddb->uv_plane[pipe][plane_id]);
5028                 skl_ddb_entry_write(dev_priv,
5029                                     PLANE_NV12_BUF_CFG(pipe, plane_id),
5030                                     &ddb->plane[pipe][plane_id]);
5031         } else {
5032                 skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
5033                                     &ddb->plane[pipe][plane_id]);
5034                 I915_WRITE(PLANE_NV12_BUF_CFG(pipe, plane_id), 0x0);
5035         }
5036 }
5037
5038 static void skl_write_cursor_wm(struct intel_crtc *intel_crtc,
5039                                 const struct skl_plane_wm *wm,
5040                                 const struct skl_ddb_allocation *ddb)
5041 {
5042         struct drm_crtc *crtc = &intel_crtc->base;
5043         struct drm_device *dev = crtc->dev;
5044         struct drm_i915_private *dev_priv = to_i915(dev);
5045         int level, max_level = ilk_wm_max_level(dev_priv);
5046         enum pipe pipe = intel_crtc->pipe;
5047
5048         for (level = 0; level <= max_level; level++) {
5049                 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5050                                    &wm->wm[level]);
5051         }
5052         skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
5053
5054         skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
5055                             &ddb->plane[pipe][PLANE_CURSOR]);
5056 }
5057
5058 bool skl_wm_level_equals(const struct skl_wm_level *l1,
5059                          const struct skl_wm_level *l2)
5060 {
5061         if (l1->plane_en != l2->plane_en)
5062                 return false;
5063
5064         /* If both planes aren't enabled, the rest shouldn't matter */
5065         if (!l1->plane_en)
5066                 return true;
5067
5068         return (l1->plane_res_l == l2->plane_res_l &&
5069                 l1->plane_res_b == l2->plane_res_b);
5070 }
5071
5072 static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5073                                            const struct skl_ddb_entry *b)
5074 {
5075         return a->start < b->end && b->start < a->end;
5076 }
5077
5078 bool skl_ddb_allocation_overlaps(struct drm_i915_private *dev_priv,
5079                                  const struct skl_ddb_entry **entries,
5080                                  const struct skl_ddb_entry *ddb,
5081                                  int ignore)
5082 {
5083         enum pipe pipe;
5084
5085         for_each_pipe(dev_priv, pipe) {
5086                 if (pipe != ignore && entries[pipe] &&
5087                     skl_ddb_entries_overlap(ddb, entries[pipe]))
5088                         return true;
5089         }
5090
5091         return false;
5092 }
5093
5094 static int skl_update_pipe_wm(struct drm_crtc_state *cstate,
5095                               const struct skl_pipe_wm *old_pipe_wm,
5096                               struct skl_pipe_wm *pipe_wm, /* out */
5097                               struct skl_ddb_allocation *ddb, /* out */
5098                               bool *changed /* out */)
5099 {
5100         struct intel_crtc_state *intel_cstate = to_intel_crtc_state(cstate);
5101         int ret;
5102
5103         ret = skl_build_pipe_wm(intel_cstate, ddb, pipe_wm);
5104         if (ret)
5105                 return ret;
5106
5107         if (!memcmp(old_pipe_wm, pipe_wm, sizeof(*pipe_wm)))
5108                 *changed = false;
5109         else
5110                 *changed = true;
5111
5112         return 0;
5113 }
5114
5115 static uint32_t
5116 pipes_modified(struct drm_atomic_state *state)
5117 {
5118         struct drm_crtc *crtc;
5119         struct drm_crtc_state *cstate;
5120         uint32_t i, ret = 0;
5121
5122         for_each_new_crtc_in_state(state, crtc, cstate, i)
5123                 ret |= drm_crtc_mask(crtc);
5124
5125         return ret;
5126 }
5127
5128 static int
5129 skl_ddb_add_affected_planes(struct intel_crtc_state *cstate)
5130 {
5131         struct drm_atomic_state *state = cstate->base.state;
5132         struct drm_device *dev = state->dev;
5133         struct drm_crtc *crtc = cstate->base.crtc;
5134         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5135         struct drm_i915_private *dev_priv = to_i915(dev);
5136         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5137         struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
5138         struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
5139         struct drm_plane_state *plane_state;
5140         struct drm_plane *plane;
5141         enum pipe pipe = intel_crtc->pipe;
5142
5143         drm_for_each_plane_mask(plane, dev, cstate->base.plane_mask) {
5144                 enum plane_id plane_id = to_intel_plane(plane)->id;
5145
5146                 if (skl_ddb_entry_equal(&cur_ddb->plane[pipe][plane_id],
5147                                         &new_ddb->plane[pipe][plane_id]) &&
5148                     skl_ddb_entry_equal(&cur_ddb->uv_plane[pipe][plane_id],
5149                                         &new_ddb->uv_plane[pipe][plane_id]))
5150                         continue;
5151
5152                 plane_state = drm_atomic_get_plane_state(state, plane);
5153                 if (IS_ERR(plane_state))
5154                         return PTR_ERR(plane_state);
5155         }
5156
5157         return 0;
5158 }
5159
5160 static int
5161 skl_compute_ddb(struct drm_atomic_state *state)
5162 {
5163         const struct drm_i915_private *dev_priv = to_i915(state->dev);
5164         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5165         struct skl_ddb_allocation *ddb = &intel_state->wm_results.ddb;
5166         struct intel_crtc *crtc;
5167         struct intel_crtc_state *cstate;
5168         int ret, i;
5169
5170         memcpy(ddb, &dev_priv->wm.skl_hw.ddb, sizeof(*ddb));
5171
5172         for_each_new_intel_crtc_in_state(intel_state, crtc, cstate, i) {
5173                 ret = skl_allocate_pipe_ddb(cstate, ddb);
5174                 if (ret)
5175                         return ret;
5176
5177                 ret = skl_ddb_add_affected_planes(cstate);
5178                 if (ret)
5179                         return ret;
5180         }
5181
5182         return 0;
5183 }
5184
5185 static void
5186 skl_print_wm_changes(const struct drm_atomic_state *state)
5187 {
5188         const struct drm_device *dev = state->dev;
5189         const struct drm_i915_private *dev_priv = to_i915(dev);
5190         const struct intel_atomic_state *intel_state =
5191                 to_intel_atomic_state(state);
5192         const struct drm_crtc *crtc;
5193         const struct drm_crtc_state *cstate;
5194         const struct intel_plane *intel_plane;
5195         const struct skl_ddb_allocation *old_ddb = &dev_priv->wm.skl_hw.ddb;
5196         const struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
5197         int i;
5198
5199         for_each_new_crtc_in_state(state, crtc, cstate, i) {
5200                 const struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5201                 enum pipe pipe = intel_crtc->pipe;
5202
5203                 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
5204                         enum plane_id plane_id = intel_plane->id;
5205                         const struct skl_ddb_entry *old, *new;
5206
5207                         old = &old_ddb->plane[pipe][plane_id];
5208                         new = &new_ddb->plane[pipe][plane_id];
5209
5210                         if (skl_ddb_entry_equal(old, new))
5211                                 continue;
5212
5213                         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] ddb (%d - %d) -> (%d - %d)\n",
5214                                          intel_plane->base.base.id,
5215                                          intel_plane->base.name,
5216                                          old->start, old->end,
5217                                          new->start, new->end);
5218                 }
5219         }
5220 }
5221
5222 static int
5223 skl_ddb_add_affected_pipes(struct drm_atomic_state *state, bool *changed)
5224 {
5225         struct drm_device *dev = state->dev;
5226         const struct drm_i915_private *dev_priv = to_i915(dev);
5227         const struct drm_crtc *crtc;
5228         const struct drm_crtc_state *cstate;
5229         struct intel_crtc *intel_crtc;
5230         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5231         uint32_t realloc_pipes = pipes_modified(state);
5232         int ret, i;
5233
5234         /*
5235          * When we distrust bios wm we always need to recompute to set the
5236          * expected DDB allocations for each CRTC.
5237          */
5238         if (dev_priv->wm.distrust_bios_wm)
5239                 (*changed) = true;
5240
5241         /*
5242          * If this transaction isn't actually touching any CRTC's, don't
5243          * bother with watermark calculation.  Note that if we pass this
5244          * test, we're guaranteed to hold at least one CRTC state mutex,
5245          * which means we can safely use values like dev_priv->active_crtcs
5246          * since any racing commits that want to update them would need to
5247          * hold _all_ CRTC state mutexes.
5248          */
5249         for_each_new_crtc_in_state(state, crtc, cstate, i)
5250                 (*changed) = true;
5251
5252         if (!*changed)
5253                 return 0;
5254
5255         /*
5256          * If this is our first atomic update following hardware readout,
5257          * we can't trust the DDB that the BIOS programmed for us.  Let's
5258          * pretend that all pipes switched active status so that we'll
5259          * ensure a full DDB recompute.
5260          */
5261         if (dev_priv->wm.distrust_bios_wm) {
5262                 ret = drm_modeset_lock(&dev->mode_config.connection_mutex,
5263                                        state->acquire_ctx);
5264                 if (ret)
5265                         return ret;
5266
5267                 intel_state->active_pipe_changes = ~0;
5268
5269                 /*
5270                  * We usually only initialize intel_state->active_crtcs if we
5271                  * we're doing a modeset; make sure this field is always
5272                  * initialized during the sanitization process that happens
5273                  * on the first commit too.
5274                  */
5275                 if (!intel_state->modeset)
5276                         intel_state->active_crtcs = dev_priv->active_crtcs;
5277         }
5278
5279         /*
5280          * If the modeset changes which CRTC's are active, we need to
5281          * recompute the DDB allocation for *all* active pipes, even
5282          * those that weren't otherwise being modified in any way by this
5283          * atomic commit.  Due to the shrinking of the per-pipe allocations
5284          * when new active CRTC's are added, it's possible for a pipe that
5285          * we were already using and aren't changing at all here to suddenly
5286          * become invalid if its DDB needs exceeds its new allocation.
5287          *
5288          * Note that if we wind up doing a full DDB recompute, we can't let
5289          * any other display updates race with this transaction, so we need
5290          * to grab the lock on *all* CRTC's.
5291          */
5292         if (intel_state->active_pipe_changes || intel_state->modeset) {
5293                 realloc_pipes = ~0;
5294                 intel_state->wm_results.dirty_pipes = ~0;
5295         }
5296
5297         /*
5298          * We're not recomputing for the pipes not included in the commit, so
5299          * make sure we start with the current state.
5300          */
5301         for_each_intel_crtc_mask(dev, intel_crtc, realloc_pipes) {
5302                 struct intel_crtc_state *cstate;
5303
5304                 cstate = intel_atomic_get_crtc_state(state, intel_crtc);
5305                 if (IS_ERR(cstate))
5306                         return PTR_ERR(cstate);
5307         }
5308
5309         return 0;
5310 }
5311
5312 static int
5313 skl_compute_wm(struct drm_atomic_state *state)
5314 {
5315         struct drm_crtc *crtc;
5316         struct drm_crtc_state *cstate;
5317         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5318         struct skl_ddb_values *results = &intel_state->wm_results;
5319         struct skl_pipe_wm *pipe_wm;
5320         bool changed = false;
5321         int ret, i;
5322
5323         /* Clear all dirty flags */
5324         results->dirty_pipes = 0;
5325
5326         ret = skl_ddb_add_affected_pipes(state, &changed);
5327         if (ret || !changed)
5328                 return ret;
5329
5330         ret = skl_compute_ddb(state);
5331         if (ret)
5332                 return ret;
5333
5334         /*
5335          * Calculate WM's for all pipes that are part of this transaction.
5336          * Note that the DDB allocation above may have added more CRTC's that
5337          * weren't otherwise being modified (and set bits in dirty_pipes) if
5338          * pipe allocations had to change.
5339          *
5340          * FIXME:  Now that we're doing this in the atomic check phase, we
5341          * should allow skl_update_pipe_wm() to return failure in cases where
5342          * no suitable watermark values can be found.
5343          */
5344         for_each_new_crtc_in_state(state, crtc, cstate, i) {
5345                 struct intel_crtc_state *intel_cstate =
5346                         to_intel_crtc_state(cstate);
5347                 const struct skl_pipe_wm *old_pipe_wm =
5348                         &to_intel_crtc_state(crtc->state)->wm.skl.optimal;
5349
5350                 pipe_wm = &intel_cstate->wm.skl.optimal;
5351                 ret = skl_update_pipe_wm(cstate, old_pipe_wm, pipe_wm,
5352                                          &results->ddb, &changed);
5353                 if (ret)
5354                         return ret;
5355
5356                 if (changed)
5357                         results->dirty_pipes |= drm_crtc_mask(crtc);
5358
5359                 if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0)
5360                         /* This pipe's WM's did not change */
5361                         continue;
5362
5363                 intel_cstate->update_wm_pre = true;
5364         }
5365
5366         skl_print_wm_changes(state);
5367
5368         return 0;
5369 }
5370
5371 static void skl_atomic_update_crtc_wm(struct intel_atomic_state *state,
5372                                       struct intel_crtc_state *cstate)
5373 {
5374         struct intel_crtc *crtc = to_intel_crtc(cstate->base.crtc);
5375         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5376         struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal;
5377         const struct skl_ddb_allocation *ddb = &state->wm_results.ddb;
5378         enum pipe pipe = crtc->pipe;
5379         enum plane_id plane_id;
5380
5381         if (!(state->wm_results.dirty_pipes & drm_crtc_mask(&crtc->base)))
5382                 return;
5383
5384         I915_WRITE(PIPE_WM_LINETIME(pipe), pipe_wm->linetime);
5385
5386         for_each_plane_id_on_crtc(crtc, plane_id) {
5387                 if (plane_id != PLANE_CURSOR)
5388                         skl_write_plane_wm(crtc, &pipe_wm->planes[plane_id],
5389                                            ddb, plane_id);
5390                 else
5391                         skl_write_cursor_wm(crtc, &pipe_wm->planes[plane_id],
5392                                             ddb);
5393         }
5394 }
5395
5396 static void skl_initial_wm(struct intel_atomic_state *state,
5397                            struct intel_crtc_state *cstate)
5398 {
5399         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5400         struct drm_device *dev = intel_crtc->base.dev;
5401         struct drm_i915_private *dev_priv = to_i915(dev);
5402         struct skl_ddb_values *results = &state->wm_results;
5403         struct skl_ddb_values *hw_vals = &dev_priv->wm.skl_hw;
5404         enum pipe pipe = intel_crtc->pipe;
5405
5406         if ((results->dirty_pipes & drm_crtc_mask(&intel_crtc->base)) == 0)
5407                 return;
5408
5409         mutex_lock(&dev_priv->wm.wm_mutex);
5410
5411         if (cstate->base.active_changed)
5412                 skl_atomic_update_crtc_wm(state, cstate);
5413
5414         memcpy(hw_vals->ddb.uv_plane[pipe], results->ddb.uv_plane[pipe],
5415                sizeof(hw_vals->ddb.uv_plane[pipe]));
5416         memcpy(hw_vals->ddb.plane[pipe], results->ddb.plane[pipe],
5417                sizeof(hw_vals->ddb.plane[pipe]));
5418
5419         mutex_unlock(&dev_priv->wm.wm_mutex);
5420 }
5421
5422 static void ilk_compute_wm_config(struct drm_device *dev,
5423                                   struct intel_wm_config *config)
5424 {
5425         struct intel_crtc *crtc;
5426
5427         /* Compute the currently _active_ config */
5428         for_each_intel_crtc(dev, crtc) {
5429                 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
5430
5431                 if (!wm->pipe_enabled)
5432                         continue;
5433
5434                 config->sprites_enabled |= wm->sprites_enabled;
5435                 config->sprites_scaled |= wm->sprites_scaled;
5436                 config->num_pipes_active++;
5437         }
5438 }
5439
5440 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
5441 {
5442         struct drm_device *dev = &dev_priv->drm;
5443         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
5444         struct ilk_wm_maximums max;
5445         struct intel_wm_config config = {};
5446         struct ilk_wm_values results = {};
5447         enum intel_ddb_partitioning partitioning;
5448
5449         ilk_compute_wm_config(dev, &config);
5450
5451         ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
5452         ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
5453
5454         /* 5/6 split only in single pipe config on IVB+ */
5455         if (INTEL_GEN(dev_priv) >= 7 &&
5456             config.num_pipes_active == 1 && config.sprites_enabled) {
5457                 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
5458                 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
5459
5460                 best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
5461         } else {
5462                 best_lp_wm = &lp_wm_1_2;
5463         }
5464
5465         partitioning = (best_lp_wm == &lp_wm_1_2) ?
5466                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
5467
5468         ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
5469
5470         ilk_write_wm_values(dev_priv, &results);
5471 }
5472
5473 static void ilk_initial_watermarks(struct intel_atomic_state *state,
5474                                    struct intel_crtc_state *cstate)
5475 {
5476         struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
5477         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5478
5479         mutex_lock(&dev_priv->wm.wm_mutex);
5480         intel_crtc->wm.active.ilk = cstate->wm.ilk.intermediate;
5481         ilk_program_watermarks(dev_priv);
5482         mutex_unlock(&dev_priv->wm.wm_mutex);
5483 }
5484
5485 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
5486                                     struct intel_crtc_state *cstate)
5487 {
5488         struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
5489         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5490
5491         mutex_lock(&dev_priv->wm.wm_mutex);
5492         if (cstate->wm.need_postvbl_update) {
5493                 intel_crtc->wm.active.ilk = cstate->wm.ilk.optimal;
5494                 ilk_program_watermarks(dev_priv);
5495         }
5496         mutex_unlock(&dev_priv->wm.wm_mutex);
5497 }
5498
5499 static inline void skl_wm_level_from_reg_val(uint32_t val,
5500                                              struct skl_wm_level *level)
5501 {
5502         level->plane_en = val & PLANE_WM_EN;
5503         level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
5504         level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
5505                 PLANE_WM_LINES_MASK;
5506 }
5507
5508 void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc,
5509                               struct skl_pipe_wm *out)
5510 {
5511         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
5512         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5513         enum pipe pipe = intel_crtc->pipe;
5514         int level, max_level;
5515         enum plane_id plane_id;
5516         uint32_t val;
5517
5518         max_level = ilk_wm_max_level(dev_priv);
5519
5520         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
5521                 struct skl_plane_wm *wm = &out->planes[plane_id];
5522
5523                 for (level = 0; level <= max_level; level++) {
5524                         if (plane_id != PLANE_CURSOR)
5525                                 val = I915_READ(PLANE_WM(pipe, plane_id, level));
5526                         else
5527                                 val = I915_READ(CUR_WM(pipe, level));
5528
5529                         skl_wm_level_from_reg_val(val, &wm->wm[level]);
5530                 }
5531
5532                 if (plane_id != PLANE_CURSOR)
5533                         val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
5534                 else
5535                         val = I915_READ(CUR_WM_TRANS(pipe));
5536
5537                 skl_wm_level_from_reg_val(val, &wm->trans_wm);
5538         }
5539
5540         if (!intel_crtc->active)
5541                 return;
5542
5543         out->linetime = I915_READ(PIPE_WM_LINETIME(pipe));
5544 }
5545
5546 void skl_wm_get_hw_state(struct drm_device *dev)
5547 {
5548         struct drm_i915_private *dev_priv = to_i915(dev);
5549         struct skl_ddb_values *hw = &dev_priv->wm.skl_hw;
5550         struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
5551         struct drm_crtc *crtc;
5552         struct intel_crtc *intel_crtc;
5553         struct intel_crtc_state *cstate;
5554
5555         skl_ddb_get_hw_state(dev_priv, ddb);
5556         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
5557                 intel_crtc = to_intel_crtc(crtc);
5558                 cstate = to_intel_crtc_state(crtc->state);
5559
5560                 skl_pipe_wm_get_hw_state(crtc, &cstate->wm.skl.optimal);
5561
5562                 if (intel_crtc->active)
5563                         hw->dirty_pipes |= drm_crtc_mask(crtc);
5564         }
5565
5566         if (dev_priv->active_crtcs) {
5567                 /* Fully recompute DDB on first atomic commit */
5568                 dev_priv->wm.distrust_bios_wm = true;
5569         } else {
5570                 /*
5571                  * Easy/common case; just sanitize DDB now if everything off
5572                  * Keep dbuf slice info intact
5573                  */
5574                 memset(ddb->plane, 0, sizeof(ddb->plane));
5575                 memset(ddb->uv_plane, 0, sizeof(ddb->uv_plane));
5576         }
5577 }
5578
5579 static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
5580 {
5581         struct drm_device *dev = crtc->dev;
5582         struct drm_i915_private *dev_priv = to_i915(dev);
5583         struct ilk_wm_values *hw = &dev_priv->wm.hw;
5584         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5585         struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
5586         struct intel_pipe_wm *active = &cstate->wm.ilk.optimal;
5587         enum pipe pipe = intel_crtc->pipe;
5588         static const i915_reg_t wm0_pipe_reg[] = {
5589                 [PIPE_A] = WM0_PIPEA_ILK,
5590                 [PIPE_B] = WM0_PIPEB_ILK,
5591                 [PIPE_C] = WM0_PIPEC_IVB,
5592         };
5593
5594         hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
5595         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
5596                 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
5597
5598         memset(active, 0, sizeof(*active));
5599
5600         active->pipe_enabled = intel_crtc->active;
5601
5602         if (active->pipe_enabled) {
5603                 u32 tmp = hw->wm_pipe[pipe];
5604
5605                 /*
5606                  * For active pipes LP0 watermark is marked as
5607                  * enabled, and LP1+ watermaks as disabled since
5608                  * we can't really reverse compute them in case
5609                  * multiple pipes are active.
5610                  */
5611                 active->wm[0].enable = true;
5612                 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
5613                 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
5614                 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
5615                 active->linetime = hw->wm_linetime[pipe];
5616         } else {
5617                 int level, max_level = ilk_wm_max_level(dev_priv);
5618
5619                 /*
5620                  * For inactive pipes, all watermark levels
5621                  * should be marked as enabled but zeroed,
5622                  * which is what we'd compute them to.
5623                  */
5624                 for (level = 0; level <= max_level; level++)
5625                         active->wm[level].enable = true;
5626         }
5627
5628         intel_crtc->wm.active.ilk = *active;
5629 }
5630
5631 #define _FW_WM(value, plane) \
5632         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
5633 #define _FW_WM_VLV(value, plane) \
5634         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
5635
5636 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
5637                                struct g4x_wm_values *wm)
5638 {
5639         uint32_t tmp;
5640
5641         tmp = I915_READ(DSPFW1);
5642         wm->sr.plane = _FW_WM(tmp, SR);
5643         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5644         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
5645         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
5646
5647         tmp = I915_READ(DSPFW2);
5648         wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
5649         wm->sr.fbc = _FW_WM(tmp, FBC_SR);
5650         wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
5651         wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
5652         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5653         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
5654
5655         tmp = I915_READ(DSPFW3);
5656         wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
5657         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
5658         wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
5659         wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
5660 }
5661
5662 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
5663                                struct vlv_wm_values *wm)
5664 {
5665         enum pipe pipe;
5666         uint32_t tmp;
5667
5668         for_each_pipe(dev_priv, pipe) {
5669                 tmp = I915_READ(VLV_DDL(pipe));
5670
5671                 wm->ddl[pipe].plane[PLANE_PRIMARY] =
5672                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5673                 wm->ddl[pipe].plane[PLANE_CURSOR] =
5674                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5675                 wm->ddl[pipe].plane[PLANE_SPRITE0] =
5676                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5677                 wm->ddl[pipe].plane[PLANE_SPRITE1] =
5678                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5679         }
5680
5681         tmp = I915_READ(DSPFW1);
5682         wm->sr.plane = _FW_WM(tmp, SR);
5683         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5684         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
5685         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
5686
5687         tmp = I915_READ(DSPFW2);
5688         wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
5689         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5690         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
5691
5692         tmp = I915_READ(DSPFW3);
5693         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
5694
5695         if (IS_CHERRYVIEW(dev_priv)) {
5696                 tmp = I915_READ(DSPFW7_CHV);
5697                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
5698                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
5699
5700                 tmp = I915_READ(DSPFW8_CHV);
5701                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
5702                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
5703
5704                 tmp = I915_READ(DSPFW9_CHV);
5705                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
5706                 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
5707
5708                 tmp = I915_READ(DSPHOWM);
5709                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
5710                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
5711                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
5712                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
5713                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
5714                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
5715                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
5716                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
5717                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
5718                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
5719         } else {
5720                 tmp = I915_READ(DSPFW7);
5721                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
5722                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
5723
5724                 tmp = I915_READ(DSPHOWM);
5725                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
5726                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
5727                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
5728                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
5729                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
5730                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
5731                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
5732         }
5733 }
5734
5735 #undef _FW_WM
5736 #undef _FW_WM_VLV
5737
5738 void g4x_wm_get_hw_state(struct drm_device *dev)
5739 {
5740         struct drm_i915_private *dev_priv = to_i915(dev);
5741         struct g4x_wm_values *wm = &dev_priv->wm.g4x;
5742         struct intel_crtc *crtc;
5743
5744         g4x_read_wm_values(dev_priv, wm);
5745
5746         wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
5747
5748         for_each_intel_crtc(dev, crtc) {
5749                 struct intel_crtc_state *crtc_state =
5750                         to_intel_crtc_state(crtc->base.state);
5751                 struct g4x_wm_state *active = &crtc->wm.active.g4x;
5752                 struct g4x_pipe_wm *raw;
5753                 enum pipe pipe = crtc->pipe;
5754                 enum plane_id plane_id;
5755                 int level, max_level;
5756
5757                 active->cxsr = wm->cxsr;
5758                 active->hpll_en = wm->hpll_en;
5759                 active->fbc_en = wm->fbc_en;
5760
5761                 active->sr = wm->sr;
5762                 active->hpll = wm->hpll;
5763
5764                 for_each_plane_id_on_crtc(crtc, plane_id) {
5765                         active->wm.plane[plane_id] =
5766                                 wm->pipe[pipe].plane[plane_id];
5767                 }
5768
5769                 if (wm->cxsr && wm->hpll_en)
5770                         max_level = G4X_WM_LEVEL_HPLL;
5771                 else if (wm->cxsr)
5772                         max_level = G4X_WM_LEVEL_SR;
5773                 else
5774                         max_level = G4X_WM_LEVEL_NORMAL;
5775
5776                 level = G4X_WM_LEVEL_NORMAL;
5777                 raw = &crtc_state->wm.g4x.raw[level];
5778                 for_each_plane_id_on_crtc(crtc, plane_id)
5779                         raw->plane[plane_id] = active->wm.plane[plane_id];
5780
5781                 if (++level > max_level)
5782                         goto out;
5783
5784                 raw = &crtc_state->wm.g4x.raw[level];
5785                 raw->plane[PLANE_PRIMARY] = active->sr.plane;
5786                 raw->plane[PLANE_CURSOR] = active->sr.cursor;
5787                 raw->plane[PLANE_SPRITE0] = 0;
5788                 raw->fbc = active->sr.fbc;
5789
5790                 if (++level > max_level)
5791                         goto out;
5792
5793                 raw = &crtc_state->wm.g4x.raw[level];
5794                 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
5795                 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
5796                 raw->plane[PLANE_SPRITE0] = 0;
5797                 raw->fbc = active->hpll.fbc;
5798
5799         out:
5800                 for_each_plane_id_on_crtc(crtc, plane_id)
5801                         g4x_raw_plane_wm_set(crtc_state, level,
5802                                              plane_id, USHRT_MAX);
5803                 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
5804
5805                 crtc_state->wm.g4x.optimal = *active;
5806                 crtc_state->wm.g4x.intermediate = *active;
5807
5808                 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
5809                               pipe_name(pipe),
5810                               wm->pipe[pipe].plane[PLANE_PRIMARY],
5811                               wm->pipe[pipe].plane[PLANE_CURSOR],
5812                               wm->pipe[pipe].plane[PLANE_SPRITE0]);
5813         }
5814
5815         DRM_DEBUG_KMS("Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
5816                       wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
5817         DRM_DEBUG_KMS("Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
5818                       wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
5819         DRM_DEBUG_KMS("Initial SR=%s HPLL=%s FBC=%s\n",
5820                       yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
5821 }
5822
5823 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
5824 {
5825         struct intel_plane *plane;
5826         struct intel_crtc *crtc;
5827
5828         mutex_lock(&dev_priv->wm.wm_mutex);
5829
5830         for_each_intel_plane(&dev_priv->drm, plane) {
5831                 struct intel_crtc *crtc =
5832                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
5833                 struct intel_crtc_state *crtc_state =
5834                         to_intel_crtc_state(crtc->base.state);
5835                 struct intel_plane_state *plane_state =
5836                         to_intel_plane_state(plane->base.state);
5837                 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
5838                 enum plane_id plane_id = plane->id;
5839                 int level;
5840
5841                 if (plane_state->base.visible)
5842                         continue;
5843
5844                 for (level = 0; level < 3; level++) {
5845                         struct g4x_pipe_wm *raw =
5846                                 &crtc_state->wm.g4x.raw[level];
5847
5848                         raw->plane[plane_id] = 0;
5849                         wm_state->wm.plane[plane_id] = 0;
5850                 }
5851
5852                 if (plane_id == PLANE_PRIMARY) {
5853                         for (level = 0; level < 3; level++) {
5854                                 struct g4x_pipe_wm *raw =
5855                                         &crtc_state->wm.g4x.raw[level];
5856                                 raw->fbc = 0;
5857                         }
5858
5859                         wm_state->sr.fbc = 0;
5860                         wm_state->hpll.fbc = 0;
5861                         wm_state->fbc_en = false;
5862                 }
5863         }
5864
5865         for_each_intel_crtc(&dev_priv->drm, crtc) {
5866                 struct intel_crtc_state *crtc_state =
5867                         to_intel_crtc_state(crtc->base.state);
5868
5869                 crtc_state->wm.g4x.intermediate =
5870                         crtc_state->wm.g4x.optimal;
5871                 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
5872         }
5873
5874         g4x_program_watermarks(dev_priv);
5875
5876         mutex_unlock(&dev_priv->wm.wm_mutex);
5877 }
5878
5879 void vlv_wm_get_hw_state(struct drm_device *dev)
5880 {
5881         struct drm_i915_private *dev_priv = to_i915(dev);
5882         struct vlv_wm_values *wm = &dev_priv->wm.vlv;
5883         struct intel_crtc *crtc;
5884         u32 val;
5885
5886         vlv_read_wm_values(dev_priv, wm);
5887
5888         wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
5889         wm->level = VLV_WM_LEVEL_PM2;
5890
5891         if (IS_CHERRYVIEW(dev_priv)) {
5892                 mutex_lock(&dev_priv->pcu_lock);
5893
5894                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
5895                 if (val & DSP_MAXFIFO_PM5_ENABLE)
5896                         wm->level = VLV_WM_LEVEL_PM5;
5897
5898                 /*
5899                  * If DDR DVFS is disabled in the BIOS, Punit
5900                  * will never ack the request. So if that happens
5901                  * assume we don't have to enable/disable DDR DVFS
5902                  * dynamically. To test that just set the REQ_ACK
5903                  * bit to poke the Punit, but don't change the
5904                  * HIGH/LOW bits so that we don't actually change
5905                  * the current state.
5906                  */
5907                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
5908                 val |= FORCE_DDR_FREQ_REQ_ACK;
5909                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
5910
5911                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
5912                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
5913                         DRM_DEBUG_KMS("Punit not acking DDR DVFS request, "
5914                                       "assuming DDR DVFS is disabled\n");
5915                         dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
5916                 } else {
5917                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
5918                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
5919                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
5920                 }
5921
5922                 mutex_unlock(&dev_priv->pcu_lock);
5923         }
5924
5925         for_each_intel_crtc(dev, crtc) {
5926                 struct intel_crtc_state *crtc_state =
5927                         to_intel_crtc_state(crtc->base.state);
5928                 struct vlv_wm_state *active = &crtc->wm.active.vlv;
5929                 const struct vlv_fifo_state *fifo_state =
5930                         &crtc_state->wm.vlv.fifo_state;
5931                 enum pipe pipe = crtc->pipe;
5932                 enum plane_id plane_id;
5933                 int level;
5934
5935                 vlv_get_fifo_size(crtc_state);
5936
5937                 active->num_levels = wm->level + 1;
5938                 active->cxsr = wm->cxsr;
5939
5940                 for (level = 0; level < active->num_levels; level++) {
5941                         struct g4x_pipe_wm *raw =
5942                                 &crtc_state->wm.vlv.raw[level];
5943
5944                         active->sr[level].plane = wm->sr.plane;
5945                         active->sr[level].cursor = wm->sr.cursor;
5946
5947                         for_each_plane_id_on_crtc(crtc, plane_id) {
5948                                 active->wm[level].plane[plane_id] =
5949                                         wm->pipe[pipe].plane[plane_id];
5950
5951                                 raw->plane[plane_id] =
5952                                         vlv_invert_wm_value(active->wm[level].plane[plane_id],
5953                                                             fifo_state->plane[plane_id]);
5954                         }
5955                 }
5956
5957                 for_each_plane_id_on_crtc(crtc, plane_id)
5958                         vlv_raw_plane_wm_set(crtc_state, level,
5959                                              plane_id, USHRT_MAX);
5960                 vlv_invalidate_wms(crtc, active, level);
5961
5962                 crtc_state->wm.vlv.optimal = *active;
5963                 crtc_state->wm.vlv.intermediate = *active;
5964
5965                 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
5966                               pipe_name(pipe),
5967                               wm->pipe[pipe].plane[PLANE_PRIMARY],
5968                               wm->pipe[pipe].plane[PLANE_CURSOR],
5969                               wm->pipe[pipe].plane[PLANE_SPRITE0],
5970                               wm->pipe[pipe].plane[PLANE_SPRITE1]);
5971         }
5972
5973         DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
5974                       wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
5975 }
5976
5977 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
5978 {
5979         struct intel_plane *plane;
5980         struct intel_crtc *crtc;
5981
5982         mutex_lock(&dev_priv->wm.wm_mutex);
5983
5984         for_each_intel_plane(&dev_priv->drm, plane) {
5985                 struct intel_crtc *crtc =
5986                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
5987                 struct intel_crtc_state *crtc_state =
5988                         to_intel_crtc_state(crtc->base.state);
5989                 struct intel_plane_state *plane_state =
5990                         to_intel_plane_state(plane->base.state);
5991                 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
5992                 const struct vlv_fifo_state *fifo_state =
5993                         &crtc_state->wm.vlv.fifo_state;
5994                 enum plane_id plane_id = plane->id;
5995                 int level;
5996
5997                 if (plane_state->base.visible)
5998                         continue;
5999
6000                 for (level = 0; level < wm_state->num_levels; level++) {
6001                         struct g4x_pipe_wm *raw =
6002                                 &crtc_state->wm.vlv.raw[level];
6003
6004                         raw->plane[plane_id] = 0;
6005
6006                         wm_state->wm[level].plane[plane_id] =
6007                                 vlv_invert_wm_value(raw->plane[plane_id],
6008                                                     fifo_state->plane[plane_id]);
6009                 }
6010         }
6011
6012         for_each_intel_crtc(&dev_priv->drm, crtc) {
6013                 struct intel_crtc_state *crtc_state =
6014                         to_intel_crtc_state(crtc->base.state);
6015
6016                 crtc_state->wm.vlv.intermediate =
6017                         crtc_state->wm.vlv.optimal;
6018                 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6019         }
6020
6021         vlv_program_watermarks(dev_priv);
6022
6023         mutex_unlock(&dev_priv->wm.wm_mutex);
6024 }
6025
6026 /*
6027  * FIXME should probably kill this and improve
6028  * the real watermark readout/sanitation instead
6029  */
6030 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6031 {
6032         I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6033         I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6034         I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6035
6036         /*
6037          * Don't touch WM1S_LP_EN here.
6038          * Doing so could cause underruns.
6039          */
6040 }
6041
6042 void ilk_wm_get_hw_state(struct drm_device *dev)
6043 {
6044         struct drm_i915_private *dev_priv = to_i915(dev);
6045         struct ilk_wm_values *hw = &dev_priv->wm.hw;
6046         struct drm_crtc *crtc;
6047
6048         ilk_init_lp_watermarks(dev_priv);
6049
6050         for_each_crtc(dev, crtc)
6051                 ilk_pipe_wm_get_hw_state(crtc);
6052
6053         hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
6054         hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
6055         hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
6056
6057         hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
6058         if (INTEL_GEN(dev_priv) >= 7) {
6059                 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
6060                 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
6061         }
6062
6063         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6064                 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6065                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6066         else if (IS_IVYBRIDGE(dev_priv))
6067                 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6068                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6069
6070         hw->enable_fbc_wm =
6071                 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6072 }
6073
6074 /**
6075  * intel_update_watermarks - update FIFO watermark values based on current modes
6076  * @crtc: the #intel_crtc on which to compute the WM
6077  *
6078  * Calculate watermark values for the various WM regs based on current mode
6079  * and plane configuration.
6080  *
6081  * There are several cases to deal with here:
6082  *   - normal (i.e. non-self-refresh)
6083  *   - self-refresh (SR) mode
6084  *   - lines are large relative to FIFO size (buffer can hold up to 2)
6085  *   - lines are small relative to FIFO size (buffer can hold more than 2
6086  *     lines), so need to account for TLB latency
6087  *
6088  *   The normal calculation is:
6089  *     watermark = dotclock * bytes per pixel * latency
6090  *   where latency is platform & configuration dependent (we assume pessimal
6091  *   values here).
6092  *
6093  *   The SR calculation is:
6094  *     watermark = (trunc(latency/line time)+1) * surface width *
6095  *       bytes per pixel
6096  *   where
6097  *     line time = htotal / dotclock
6098  *     surface width = hdisplay for normal plane and 64 for cursor
6099  *   and latency is assumed to be high, as above.
6100  *
6101  * The final value programmed to the register should always be rounded up,
6102  * and include an extra 2 entries to account for clock crossings.
6103  *
6104  * We don't use the sprite, so we can ignore that.  And on Crestline we have
6105  * to set the non-SR watermarks to 8.
6106  */
6107 void intel_update_watermarks(struct intel_crtc *crtc)
6108 {
6109         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6110
6111         if (dev_priv->display.update_wm)
6112                 dev_priv->display.update_wm(crtc);
6113 }
6114
6115 void intel_enable_ipc(struct drm_i915_private *dev_priv)
6116 {
6117         u32 val;
6118
6119         /* Display WA #0477 WaDisableIPC: skl */
6120         if (IS_SKYLAKE(dev_priv))
6121                 dev_priv->ipc_enabled = false;
6122
6123         /* Display WA #1141: SKL:all KBL:all CFL */
6124         if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) &&
6125             !dev_priv->dram_info.symmetric_memory)
6126                 dev_priv->ipc_enabled = false;
6127
6128         val = I915_READ(DISP_ARB_CTL2);
6129
6130         if (dev_priv->ipc_enabled)
6131                 val |= DISP_IPC_ENABLE;
6132         else
6133                 val &= ~DISP_IPC_ENABLE;
6134
6135         I915_WRITE(DISP_ARB_CTL2, val);
6136 }
6137
6138 void intel_init_ipc(struct drm_i915_private *dev_priv)
6139 {
6140         dev_priv->ipc_enabled = false;
6141         if (!HAS_IPC(dev_priv))
6142                 return;
6143
6144         dev_priv->ipc_enabled = true;
6145         intel_enable_ipc(dev_priv);
6146 }
6147
6148 /*
6149  * Lock protecting IPS related data structures
6150  */
6151 DEFINE_SPINLOCK(mchdev_lock);
6152
6153 /* Global for IPS driver to get at the current i915 device. Protected by
6154  * mchdev_lock. */
6155 static struct drm_i915_private *i915_mch_dev;
6156
6157 bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val)
6158 {
6159         u16 rgvswctl;
6160
6161         lockdep_assert_held(&mchdev_lock);
6162
6163         rgvswctl = I915_READ16(MEMSWCTL);
6164         if (rgvswctl & MEMCTL_CMD_STS) {
6165                 DRM_DEBUG("gpu busy, RCS change rejected\n");
6166                 return false; /* still busy with another command */
6167         }
6168
6169         rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
6170                 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
6171         I915_WRITE16(MEMSWCTL, rgvswctl);
6172         POSTING_READ16(MEMSWCTL);
6173
6174         rgvswctl |= MEMCTL_CMD_STS;
6175         I915_WRITE16(MEMSWCTL, rgvswctl);
6176
6177         return true;
6178 }
6179
6180 static void ironlake_enable_drps(struct drm_i915_private *dev_priv)
6181 {
6182         u32 rgvmodectl;
6183         u8 fmax, fmin, fstart, vstart;
6184
6185         spin_lock_irq(&mchdev_lock);
6186
6187         rgvmodectl = I915_READ(MEMMODECTL);
6188
6189         /* Enable temp reporting */
6190         I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
6191         I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
6192
6193         /* 100ms RC evaluation intervals */
6194         I915_WRITE(RCUPEI, 100000);
6195         I915_WRITE(RCDNEI, 100000);
6196
6197         /* Set max/min thresholds to 90ms and 80ms respectively */
6198         I915_WRITE(RCBMAXAVG, 90000);
6199         I915_WRITE(RCBMINAVG, 80000);
6200
6201         I915_WRITE(MEMIHYST, 1);
6202
6203         /* Set up min, max, and cur for interrupt handling */
6204         fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
6205         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
6206         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
6207                 MEMMODE_FSTART_SHIFT;
6208
6209         vstart = (I915_READ(PXVFREQ(fstart)) & PXVFREQ_PX_MASK) >>
6210                 PXVFREQ_PX_SHIFT;
6211
6212         dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
6213         dev_priv->ips.fstart = fstart;
6214
6215         dev_priv->ips.max_delay = fstart;
6216         dev_priv->ips.min_delay = fmin;
6217         dev_priv->ips.cur_delay = fstart;
6218
6219         DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
6220                          fmax, fmin, fstart);
6221
6222         I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
6223
6224         /*
6225          * Interrupts will be enabled in ironlake_irq_postinstall
6226          */
6227
6228         I915_WRITE(VIDSTART, vstart);
6229         POSTING_READ(VIDSTART);
6230
6231         rgvmodectl |= MEMMODE_SWMODE_EN;
6232         I915_WRITE(MEMMODECTL, rgvmodectl);
6233
6234         if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
6235                 DRM_ERROR("stuck trying to change perf mode\n");
6236         mdelay(1);
6237
6238         ironlake_set_drps(dev_priv, fstart);
6239
6240         dev_priv->ips.last_count1 = I915_READ(DMIEC) +
6241                 I915_READ(DDREC) + I915_READ(CSIEC);
6242         dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
6243         dev_priv->ips.last_count2 = I915_READ(GFXEC);
6244         dev_priv->ips.last_time2 = ktime_get_raw_ns();
6245
6246         spin_unlock_irq(&mchdev_lock);
6247 }
6248
6249 static void ironlake_disable_drps(struct drm_i915_private *dev_priv)
6250 {
6251         u16 rgvswctl;
6252
6253         spin_lock_irq(&mchdev_lock);
6254
6255         rgvswctl = I915_READ16(MEMSWCTL);
6256
6257         /* Ack interrupts, disable EFC interrupt */
6258         I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
6259         I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
6260         I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
6261         I915_WRITE(DEIIR, DE_PCU_EVENT);
6262         I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
6263
6264         /* Go back to the starting frequency */
6265         ironlake_set_drps(dev_priv, dev_priv->ips.fstart);
6266         mdelay(1);
6267         rgvswctl |= MEMCTL_CMD_STS;
6268         I915_WRITE(MEMSWCTL, rgvswctl);
6269         mdelay(1);
6270
6271         spin_unlock_irq(&mchdev_lock);
6272 }
6273
6274 /* There's a funny hw issue where the hw returns all 0 when reading from
6275  * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
6276  * ourselves, instead of doing a rmw cycle (which might result in us clearing
6277  * all limits and the gpu stuck at whatever frequency it is at atm).
6278  */
6279 static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
6280 {
6281         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6282         u32 limits;
6283
6284         /* Only set the down limit when we've reached the lowest level to avoid
6285          * getting more interrupts, otherwise leave this clear. This prevents a
6286          * race in the hw when coming out of rc6: There's a tiny window where
6287          * the hw runs at the minimal clock before selecting the desired
6288          * frequency, if the down threshold expires in that window we will not
6289          * receive a down interrupt. */
6290         if (INTEL_GEN(dev_priv) >= 9) {
6291                 limits = (rps->max_freq_softlimit) << 23;
6292                 if (val <= rps->min_freq_softlimit)
6293                         limits |= (rps->min_freq_softlimit) << 14;
6294         } else {
6295                 limits = rps->max_freq_softlimit << 24;
6296                 if (val <= rps->min_freq_softlimit)
6297                         limits |= rps->min_freq_softlimit << 16;
6298         }
6299
6300         return limits;
6301 }
6302
6303 static void rps_set_power(struct drm_i915_private *dev_priv, int new_power)
6304 {
6305         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6306         u32 threshold_up = 0, threshold_down = 0; /* in % */
6307         u32 ei_up = 0, ei_down = 0;
6308
6309         lockdep_assert_held(&rps->power.mutex);
6310
6311         if (new_power == rps->power.mode)
6312                 return;
6313
6314         /* Note the units here are not exactly 1us, but 1280ns. */
6315         switch (new_power) {
6316         case LOW_POWER:
6317                 /* Upclock if more than 95% busy over 16ms */
6318                 ei_up = 16000;
6319                 threshold_up = 95;
6320
6321                 /* Downclock if less than 85% busy over 32ms */
6322                 ei_down = 32000;
6323                 threshold_down = 85;
6324                 break;
6325
6326         case BETWEEN:
6327                 /* Upclock if more than 90% busy over 13ms */
6328                 ei_up = 13000;
6329                 threshold_up = 90;
6330
6331                 /* Downclock if less than 75% busy over 32ms */
6332                 ei_down = 32000;
6333                 threshold_down = 75;
6334                 break;
6335
6336         case HIGH_POWER:
6337                 /* Upclock if more than 85% busy over 10ms */
6338                 ei_up = 10000;
6339                 threshold_up = 85;
6340
6341                 /* Downclock if less than 60% busy over 32ms */
6342                 ei_down = 32000;
6343                 threshold_down = 60;
6344                 break;
6345         }
6346
6347         /* When byt can survive without system hang with dynamic
6348          * sw freq adjustments, this restriction can be lifted.
6349          */
6350         if (IS_VALLEYVIEW(dev_priv))
6351                 goto skip_hw_write;
6352
6353         I915_WRITE(GEN6_RP_UP_EI,
6354                    GT_INTERVAL_FROM_US(dev_priv, ei_up));
6355         I915_WRITE(GEN6_RP_UP_THRESHOLD,
6356                    GT_INTERVAL_FROM_US(dev_priv,
6357                                        ei_up * threshold_up / 100));
6358
6359         I915_WRITE(GEN6_RP_DOWN_EI,
6360                    GT_INTERVAL_FROM_US(dev_priv, ei_down));
6361         I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
6362                    GT_INTERVAL_FROM_US(dev_priv,
6363                                        ei_down * threshold_down / 100));
6364
6365         I915_WRITE(GEN6_RP_CONTROL,
6366                    GEN6_RP_MEDIA_TURBO |
6367                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
6368                    GEN6_RP_MEDIA_IS_GFX |
6369                    GEN6_RP_ENABLE |
6370                    GEN6_RP_UP_BUSY_AVG |
6371                    GEN6_RP_DOWN_IDLE_AVG);
6372
6373 skip_hw_write:
6374         rps->power.mode = new_power;
6375         rps->power.up_threshold = threshold_up;
6376         rps->power.down_threshold = threshold_down;
6377 }
6378
6379 static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
6380 {
6381         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6382         int new_power;
6383
6384         new_power = rps->power.mode;
6385         switch (rps->power.mode) {
6386         case LOW_POWER:
6387                 if (val > rps->efficient_freq + 1 &&
6388                     val > rps->cur_freq)
6389                         new_power = BETWEEN;
6390                 break;
6391
6392         case BETWEEN:
6393                 if (val <= rps->efficient_freq &&
6394                     val < rps->cur_freq)
6395                         new_power = LOW_POWER;
6396                 else if (val >= rps->rp0_freq &&
6397                          val > rps->cur_freq)
6398                         new_power = HIGH_POWER;
6399                 break;
6400
6401         case HIGH_POWER:
6402                 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
6403                     val < rps->cur_freq)
6404                         new_power = BETWEEN;
6405                 break;
6406         }
6407         /* Max/min bins are special */
6408         if (val <= rps->min_freq_softlimit)
6409                 new_power = LOW_POWER;
6410         if (val >= rps->max_freq_softlimit)
6411                 new_power = HIGH_POWER;
6412
6413         mutex_lock(&rps->power.mutex);
6414         if (rps->power.interactive)
6415                 new_power = HIGH_POWER;
6416         rps_set_power(dev_priv, new_power);
6417         mutex_unlock(&rps->power.mutex);
6418 }
6419
6420 void intel_rps_mark_interactive(struct drm_i915_private *i915, bool interactive)
6421 {
6422         struct intel_rps *rps = &i915->gt_pm.rps;
6423
6424         if (INTEL_GEN(i915) < 6)
6425                 return;
6426
6427         mutex_lock(&rps->power.mutex);
6428         if (interactive) {
6429                 if (!rps->power.interactive++ && READ_ONCE(i915->gt.awake))
6430                         rps_set_power(i915, HIGH_POWER);
6431         } else {
6432                 GEM_BUG_ON(!rps->power.interactive);
6433                 rps->power.interactive--;
6434         }
6435         mutex_unlock(&rps->power.mutex);
6436 }
6437
6438 static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
6439 {
6440         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6441         u32 mask = 0;
6442
6443         /* We use UP_EI_EXPIRED interupts for both up/down in manual mode */
6444         if (val > rps->min_freq_softlimit)
6445                 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
6446         if (val < rps->max_freq_softlimit)
6447                 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
6448
6449         mask &= dev_priv->pm_rps_events;
6450
6451         return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
6452 }
6453
6454 /* gen6_set_rps is called to update the frequency request, but should also be
6455  * called when the range (min_delay and max_delay) is modified so that we can
6456  * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
6457 static int gen6_set_rps(struct drm_i915_private *dev_priv, u8 val)
6458 {
6459         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6460
6461         /* min/max delay may still have been modified so be sure to
6462          * write the limits value.
6463          */
6464         if (val != rps->cur_freq) {
6465                 gen6_set_rps_thresholds(dev_priv, val);
6466
6467                 if (INTEL_GEN(dev_priv) >= 9)
6468                         I915_WRITE(GEN6_RPNSWREQ,
6469                                    GEN9_FREQUENCY(val));
6470                 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6471                         I915_WRITE(GEN6_RPNSWREQ,
6472                                    HSW_FREQUENCY(val));
6473                 else
6474                         I915_WRITE(GEN6_RPNSWREQ,
6475                                    GEN6_FREQUENCY(val) |
6476                                    GEN6_OFFSET(0) |
6477                                    GEN6_AGGRESSIVE_TURBO);
6478         }
6479
6480         /* Make sure we continue to get interrupts
6481          * until we hit the minimum or maximum frequencies.
6482          */
6483         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
6484         I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
6485
6486         rps->cur_freq = val;
6487         trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
6488
6489         return 0;
6490 }
6491
6492 static int valleyview_set_rps(struct drm_i915_private *dev_priv, u8 val)
6493 {
6494         int err;
6495
6496         if (WARN_ONCE(IS_CHERRYVIEW(dev_priv) && (val & 1),
6497                       "Odd GPU freq value\n"))
6498                 val &= ~1;
6499
6500         I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
6501
6502         if (val != dev_priv->gt_pm.rps.cur_freq) {
6503                 err = vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
6504                 if (err)
6505                         return err;
6506
6507                 gen6_set_rps_thresholds(dev_priv, val);
6508         }
6509
6510         dev_priv->gt_pm.rps.cur_freq = val;
6511         trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
6512
6513         return 0;
6514 }
6515
6516 /* vlv_set_rps_idle: Set the frequency to idle, if Gfx clocks are down
6517  *
6518  * * If Gfx is Idle, then
6519  * 1. Forcewake Media well.
6520  * 2. Request idle freq.
6521  * 3. Release Forcewake of Media well.
6522 */
6523 static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
6524 {
6525         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6526         u32 val = rps->idle_freq;
6527         int err;
6528
6529         if (rps->cur_freq <= val)
6530                 return;
6531
6532         /* The punit delays the write of the frequency and voltage until it
6533          * determines the GPU is awake. During normal usage we don't want to
6534          * waste power changing the frequency if the GPU is sleeping (rc6).
6535          * However, the GPU and driver is now idle and we do not want to delay
6536          * switching to minimum voltage (reducing power whilst idle) as we do
6537          * not expect to be woken in the near future and so must flush the
6538          * change by waking the device.
6539          *
6540          * We choose to take the media powerwell (either would do to trick the
6541          * punit into committing the voltage change) as that takes a lot less
6542          * power than the render powerwell.
6543          */
6544         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA);
6545         err = valleyview_set_rps(dev_priv, val);
6546         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA);
6547
6548         if (err)
6549                 DRM_ERROR("Failed to set RPS for idle\n");
6550 }
6551
6552 void gen6_rps_busy(struct drm_i915_private *dev_priv)
6553 {
6554         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6555
6556         mutex_lock(&dev_priv->pcu_lock);
6557         if (rps->enabled) {
6558                 u8 freq;
6559
6560                 if (dev_priv->pm_rps_events & GEN6_PM_RP_UP_EI_EXPIRED)
6561                         gen6_rps_reset_ei(dev_priv);
6562                 I915_WRITE(GEN6_PMINTRMSK,
6563                            gen6_rps_pm_mask(dev_priv, rps->cur_freq));
6564
6565                 gen6_enable_rps_interrupts(dev_priv);
6566
6567                 /* Use the user's desired frequency as a guide, but for better
6568                  * performance, jump directly to RPe as our starting frequency.
6569                  */
6570                 freq = max(rps->cur_freq,
6571                            rps->efficient_freq);
6572
6573                 if (intel_set_rps(dev_priv,
6574                                   clamp(freq,
6575                                         rps->min_freq_softlimit,
6576                                         rps->max_freq_softlimit)))
6577                         DRM_DEBUG_DRIVER("Failed to set idle frequency\n");
6578         }
6579         mutex_unlock(&dev_priv->pcu_lock);
6580 }
6581
6582 void gen6_rps_idle(struct drm_i915_private *dev_priv)
6583 {
6584         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6585
6586         /* Flush our bottom-half so that it does not race with us
6587          * setting the idle frequency and so that it is bounded by
6588          * our rpm wakeref. And then disable the interrupts to stop any
6589          * futher RPS reclocking whilst we are asleep.
6590          */
6591         gen6_disable_rps_interrupts(dev_priv);
6592
6593         mutex_lock(&dev_priv->pcu_lock);
6594         if (rps->enabled) {
6595                 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
6596                         vlv_set_rps_idle(dev_priv);
6597                 else
6598                         gen6_set_rps(dev_priv, rps->idle_freq);
6599                 rps->last_adj = 0;
6600                 I915_WRITE(GEN6_PMINTRMSK,
6601                            gen6_sanitize_rps_pm_mask(dev_priv, ~0));
6602         }
6603         mutex_unlock(&dev_priv->pcu_lock);
6604 }
6605
6606 void gen6_rps_boost(struct i915_request *rq,
6607                     struct intel_rps_client *rps_client)
6608 {
6609         struct intel_rps *rps = &rq->i915->gt_pm.rps;
6610         unsigned long flags;
6611         bool boost;
6612
6613         /* This is intentionally racy! We peek at the state here, then
6614          * validate inside the RPS worker.
6615          */
6616         if (!rps->enabled)
6617                 return;
6618
6619         if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))
6620                 return;
6621
6622         /* Serializes with i915_request_retire() */
6623         boost = false;
6624         spin_lock_irqsave(&rq->lock, flags);
6625         if (!rq->waitboost && !dma_fence_is_signaled_locked(&rq->fence)) {
6626                 boost = !atomic_fetch_inc(&rps->num_waiters);
6627                 rq->waitboost = true;
6628         }
6629         spin_unlock_irqrestore(&rq->lock, flags);
6630         if (!boost)
6631                 return;
6632
6633         if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
6634                 schedule_work(&rps->work);
6635
6636         atomic_inc(rps_client ? &rps_client->boosts : &rps->boosts);
6637 }
6638
6639 int intel_set_rps(struct drm_i915_private *dev_priv, u8 val)
6640 {
6641         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6642         int err;
6643
6644         lockdep_assert_held(&dev_priv->pcu_lock);
6645         GEM_BUG_ON(val > rps->max_freq);
6646         GEM_BUG_ON(val < rps->min_freq);
6647
6648         if (!rps->enabled) {
6649                 rps->cur_freq = val;
6650                 return 0;
6651         }
6652
6653         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
6654                 err = valleyview_set_rps(dev_priv, val);
6655         else
6656                 err = gen6_set_rps(dev_priv, val);
6657
6658         return err;
6659 }
6660
6661 static void gen9_disable_rc6(struct drm_i915_private *dev_priv)
6662 {
6663         I915_WRITE(GEN6_RC_CONTROL, 0);
6664         I915_WRITE(GEN9_PG_ENABLE, 0);
6665 }
6666
6667 static void gen9_disable_rps(struct drm_i915_private *dev_priv)
6668 {
6669         I915_WRITE(GEN6_RP_CONTROL, 0);
6670 }
6671
6672 static void gen6_disable_rc6(struct drm_i915_private *dev_priv)
6673 {
6674         I915_WRITE(GEN6_RC_CONTROL, 0);
6675 }
6676
6677 static void gen6_disable_rps(struct drm_i915_private *dev_priv)
6678 {
6679         I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
6680         I915_WRITE(GEN6_RP_CONTROL, 0);
6681 }
6682
6683 static void cherryview_disable_rc6(struct drm_i915_private *dev_priv)
6684 {
6685         I915_WRITE(GEN6_RC_CONTROL, 0);
6686 }
6687
6688 static void cherryview_disable_rps(struct drm_i915_private *dev_priv)
6689 {
6690         I915_WRITE(GEN6_RP_CONTROL, 0);
6691 }
6692
6693 static void valleyview_disable_rc6(struct drm_i915_private *dev_priv)
6694 {
6695         /* We're doing forcewake before Disabling RC6,
6696          * This what the BIOS expects when going into suspend */
6697         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6698
6699         I915_WRITE(GEN6_RC_CONTROL, 0);
6700
6701         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6702 }
6703
6704 static void valleyview_disable_rps(struct drm_i915_private *dev_priv)
6705 {
6706         I915_WRITE(GEN6_RP_CONTROL, 0);
6707 }
6708
6709 static bool bxt_check_bios_rc6_setup(struct drm_i915_private *dev_priv)
6710 {
6711         bool enable_rc6 = true;
6712         unsigned long rc6_ctx_base;
6713         u32 rc_ctl;
6714         int rc_sw_target;
6715
6716         rc_ctl = I915_READ(GEN6_RC_CONTROL);
6717         rc_sw_target = (I915_READ(GEN6_RC_STATE) & RC_SW_TARGET_STATE_MASK) >>
6718                        RC_SW_TARGET_STATE_SHIFT;
6719         DRM_DEBUG_DRIVER("BIOS enabled RC states: "
6720                          "HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n",
6721                          onoff(rc_ctl & GEN6_RC_CTL_HW_ENABLE),
6722                          onoff(rc_ctl & GEN6_RC_CTL_RC6_ENABLE),
6723                          rc_sw_target);
6724
6725         if (!(I915_READ(RC6_LOCATION) & RC6_CTX_IN_DRAM)) {
6726                 DRM_DEBUG_DRIVER("RC6 Base location not set properly.\n");
6727                 enable_rc6 = false;
6728         }
6729
6730         /*
6731          * The exact context size is not known for BXT, so assume a page size
6732          * for this check.
6733          */
6734         rc6_ctx_base = I915_READ(RC6_CTX_BASE) & RC6_CTX_BASE_MASK;
6735         if (!((rc6_ctx_base >= dev_priv->dsm_reserved.start) &&
6736               (rc6_ctx_base + PAGE_SIZE < dev_priv->dsm_reserved.end))) {
6737                 DRM_DEBUG_DRIVER("RC6 Base address not as expected.\n");
6738                 enable_rc6 = false;
6739         }
6740
6741         if (!(((I915_READ(PWRCTX_MAXCNT_RCSUNIT) & IDLE_TIME_MASK) > 1) &&
6742               ((I915_READ(PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1) &&
6743               ((I915_READ(PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1) &&
6744               ((I915_READ(PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1))) {
6745                 DRM_DEBUG_DRIVER("Engine Idle wait time not set properly.\n");
6746                 enable_rc6 = false;
6747         }
6748
6749         if (!I915_READ(GEN8_PUSHBUS_CONTROL) ||
6750             !I915_READ(GEN8_PUSHBUS_ENABLE) ||
6751             !I915_READ(GEN8_PUSHBUS_SHIFT)) {
6752                 DRM_DEBUG_DRIVER("Pushbus not setup properly.\n");
6753                 enable_rc6 = false;
6754         }
6755
6756         if (!I915_READ(GEN6_GFXPAUSE)) {
6757                 DRM_DEBUG_DRIVER("GFX pause not setup properly.\n");
6758                 enable_rc6 = false;
6759         }
6760
6761         if (!I915_READ(GEN8_MISC_CTRL0)) {
6762                 DRM_DEBUG_DRIVER("GPM control not setup properly.\n");
6763                 enable_rc6 = false;
6764         }
6765
6766         return enable_rc6;
6767 }
6768
6769 static bool sanitize_rc6(struct drm_i915_private *i915)
6770 {
6771         struct intel_device_info *info = mkwrite_device_info(i915);
6772
6773         /* Powersaving is controlled by the host when inside a VM */
6774         if (intel_vgpu_active(i915))
6775                 info->has_rc6 = 0;
6776
6777         if (info->has_rc6 &&
6778             IS_GEN9_LP(i915) && !bxt_check_bios_rc6_setup(i915)) {
6779                 DRM_INFO("RC6 disabled by BIOS\n");
6780                 info->has_rc6 = 0;
6781         }
6782
6783         /*
6784          * We assume that we do not have any deep rc6 levels if we don't have
6785          * have the previous rc6 level supported, i.e. we use HAS_RC6()
6786          * as the initial coarse check for rc6 in general, moving on to
6787          * progressively finer/deeper levels.
6788          */
6789         if (!info->has_rc6 && info->has_rc6p)
6790                 info->has_rc6p = 0;
6791
6792         return info->has_rc6;
6793 }
6794
6795 static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
6796 {
6797         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6798
6799         /* All of these values are in units of 50MHz */
6800
6801         /* static values from HW: RP0 > RP1 > RPn (min_freq) */
6802         if (IS_GEN9_LP(dev_priv)) {
6803                 u32 rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
6804                 rps->rp0_freq = (rp_state_cap >> 16) & 0xff;
6805                 rps->rp1_freq = (rp_state_cap >>  8) & 0xff;
6806                 rps->min_freq = (rp_state_cap >>  0) & 0xff;
6807         } else {
6808                 u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
6809                 rps->rp0_freq = (rp_state_cap >>  0) & 0xff;
6810                 rps->rp1_freq = (rp_state_cap >>  8) & 0xff;
6811                 rps->min_freq = (rp_state_cap >> 16) & 0xff;
6812         }
6813         /* hw_max = RP0 until we check for overclocking */
6814         rps->max_freq = rps->rp0_freq;
6815
6816         rps->efficient_freq = rps->rp1_freq;
6817         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) ||
6818             IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
6819                 u32 ddcc_status = 0;
6820
6821                 if (sandybridge_pcode_read(dev_priv,
6822                                            HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
6823                                            &ddcc_status) == 0)
6824                         rps->efficient_freq =
6825                                 clamp_t(u8,
6826                                         ((ddcc_status >> 8) & 0xff),
6827                                         rps->min_freq,
6828                                         rps->max_freq);
6829         }
6830
6831         if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
6832                 /* Store the frequency values in 16.66 MHZ units, which is
6833                  * the natural hardware unit for SKL
6834                  */
6835                 rps->rp0_freq *= GEN9_FREQ_SCALER;
6836                 rps->rp1_freq *= GEN9_FREQ_SCALER;
6837                 rps->min_freq *= GEN9_FREQ_SCALER;
6838                 rps->max_freq *= GEN9_FREQ_SCALER;
6839                 rps->efficient_freq *= GEN9_FREQ_SCALER;
6840         }
6841 }
6842
6843 static void reset_rps(struct drm_i915_private *dev_priv,
6844                       int (*set)(struct drm_i915_private *, u8))
6845 {
6846         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6847         u8 freq = rps->cur_freq;
6848
6849         /* force a reset */
6850         rps->power.mode = -1;
6851         rps->cur_freq = -1;
6852
6853         if (set(dev_priv, freq))
6854                 DRM_ERROR("Failed to reset RPS to initial values\n");
6855 }
6856
6857 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
6858 static void gen9_enable_rps(struct drm_i915_private *dev_priv)
6859 {
6860         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6861
6862         /* Program defaults and thresholds for RPS */
6863         if (IS_GEN9(dev_priv))
6864                 I915_WRITE(GEN6_RC_VIDEO_FREQ,
6865                         GEN9_FREQUENCY(dev_priv->gt_pm.rps.rp1_freq));
6866
6867         /* 1 second timeout*/
6868         I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
6869                 GT_INTERVAL_FROM_US(dev_priv, 1000000));
6870
6871         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
6872
6873         /* Leaning on the below call to gen6_set_rps to program/setup the
6874          * Up/Down EI & threshold registers, as well as the RP_CONTROL,
6875          * RP_INTERRUPT_LIMITS & RPNSWREQ registers */
6876         reset_rps(dev_priv, gen6_set_rps);
6877
6878         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6879 }
6880
6881 static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
6882 {
6883         struct intel_engine_cs *engine;
6884         enum intel_engine_id id;
6885         u32 rc6_mode;
6886
6887         /* 1a: Software RC state - RC0 */
6888         I915_WRITE(GEN6_RC_STATE, 0);
6889
6890         /* 1b: Get forcewake during program sequence. Although the driver
6891          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6892         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6893
6894         /* 2a: Disable RC states. */
6895         I915_WRITE(GEN6_RC_CONTROL, 0);
6896
6897         /* 2b: Program RC6 thresholds.*/
6898         if (INTEL_GEN(dev_priv) >= 10) {
6899                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
6900                 I915_WRITE(GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
6901         } else if (IS_SKYLAKE(dev_priv)) {
6902                 /*
6903                  * WaRsDoubleRc6WrlWithCoarsePowerGating:skl Doubling WRL only
6904                  * when CPG is enabled
6905                  */
6906                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
6907         } else {
6908                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
6909         }
6910
6911         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
6912         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
6913         for_each_engine(engine, dev_priv, id)
6914                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
6915
6916         if (HAS_GUC(dev_priv))
6917                 I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
6918
6919         I915_WRITE(GEN6_RC_SLEEP, 0);
6920
6921         /*
6922          * 2c: Program Coarse Power Gating Policies.
6923          *
6924          * Bspec's guidance is to use 25us (really 25 * 1280ns) here. What we
6925          * use instead is a more conservative estimate for the maximum time
6926          * it takes us to service a CS interrupt and submit a new ELSP - that
6927          * is the time which the GPU is idle waiting for the CPU to select the
6928          * next request to execute. If the idle hysteresis is less than that
6929          * interrupt service latency, the hardware will automatically gate
6930          * the power well and we will then incur the wake up cost on top of
6931          * the service latency. A similar guide from intel_pstate is that we
6932          * do not want the enable hysteresis to less than the wakeup latency.
6933          *
6934          * igt/gem_exec_nop/sequential provides a rough estimate for the
6935          * service latency, and puts it around 10us for Broadwell (and other
6936          * big core) and around 40us for Broxton (and other low power cores).
6937          * [Note that for legacy ringbuffer submission, this is less than 1us!]
6938          * However, the wakeup latency on Broxton is closer to 100us. To be
6939          * conservative, we have to factor in a context switch on top (due
6940          * to ksoftirqd).
6941          */
6942         I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250);
6943         I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 250);
6944
6945         /* 3a: Enable RC6 */
6946         I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
6947
6948         /* WaRsUseTimeoutMode:cnl (pre-prod) */
6949         if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_C0))
6950                 rc6_mode = GEN7_RC_CTL_TO_MODE;
6951         else
6952                 rc6_mode = GEN6_RC_CTL_EI_MODE(1);
6953
6954         I915_WRITE(GEN6_RC_CONTROL,
6955                    GEN6_RC_CTL_HW_ENABLE |
6956                    GEN6_RC_CTL_RC6_ENABLE |
6957                    rc6_mode);
6958
6959         /*
6960          * 3b: Enable Coarse Power Gating only when RC6 is enabled.
6961          * WaRsDisableCoarsePowerGating:skl,cnl - Render/Media PG need to be disabled with RC6.
6962          */
6963         if (NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
6964                 I915_WRITE(GEN9_PG_ENABLE, 0);
6965         else
6966                 I915_WRITE(GEN9_PG_ENABLE,
6967                            GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE);
6968
6969         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6970 }
6971
6972 static void gen8_enable_rc6(struct drm_i915_private *dev_priv)
6973 {
6974         struct intel_engine_cs *engine;
6975         enum intel_engine_id id;
6976
6977         /* 1a: Software RC state - RC0 */
6978         I915_WRITE(GEN6_RC_STATE, 0);
6979
6980         /* 1b: Get forcewake during program sequence. Although the driver
6981          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6982         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6983
6984         /* 2a: Disable RC states. */
6985         I915_WRITE(GEN6_RC_CONTROL, 0);
6986
6987         /* 2b: Program RC6 thresholds.*/
6988         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
6989         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
6990         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
6991         for_each_engine(engine, dev_priv, id)
6992                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
6993         I915_WRITE(GEN6_RC_SLEEP, 0);
6994         I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
6995
6996         /* 3: Enable RC6 */
6997
6998         I915_WRITE(GEN6_RC_CONTROL,
6999                    GEN6_RC_CTL_HW_ENABLE |
7000                    GEN7_RC_CTL_TO_MODE |
7001                    GEN6_RC_CTL_RC6_ENABLE);
7002
7003         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7004 }
7005
7006 static void gen8_enable_rps(struct drm_i915_private *dev_priv)
7007 {
7008         struct intel_rps *rps = &dev_priv->gt_pm.rps;
7009
7010         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7011
7012         /* 1 Program defaults and thresholds for RPS*/
7013         I915_WRITE(GEN6_RPNSWREQ,
7014                    HSW_FREQUENCY(rps->rp1_freq));
7015         I915_WRITE(GEN6_RC_VIDEO_FREQ,
7016                    HSW_FREQUENCY(rps->rp1_freq));
7017         /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
7018         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
7019
7020         /* Docs recommend 900MHz, and 300 MHz respectively */
7021         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
7022                    rps->max_freq_softlimit << 24 |
7023                    rps->min_freq_softlimit << 16);
7024
7025         I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
7026         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
7027         I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
7028         I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
7029
7030         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7031
7032         /* 2: Enable RPS */
7033         I915_WRITE(GEN6_RP_CONTROL,
7034                    GEN6_RP_MEDIA_TURBO |
7035                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
7036                    GEN6_RP_MEDIA_IS_GFX |
7037                    GEN6_RP_ENABLE |
7038                    GEN6_RP_UP_BUSY_AVG |
7039                    GEN6_RP_DOWN_IDLE_AVG);
7040
7041         reset_rps(dev_priv, gen6_set_rps);
7042
7043         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7044 }
7045
7046 static void gen6_enable_rc6(struct drm_i915_private *dev_priv)
7047 {
7048         struct intel_engine_cs *engine;
7049         enum intel_engine_id id;
7050         u32 rc6vids, rc6_mask;
7051         u32 gtfifodbg;
7052         int ret;
7053
7054         I915_WRITE(GEN6_RC_STATE, 0);
7055
7056         /* Clear the DBG now so we don't confuse earlier errors */
7057         gtfifodbg = I915_READ(GTFIFODBG);
7058         if (gtfifodbg) {
7059                 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
7060                 I915_WRITE(GTFIFODBG, gtfifodbg);
7061         }
7062
7063         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7064
7065         /* disable the counters and set deterministic thresholds */
7066         I915_WRITE(GEN6_RC_CONTROL, 0);
7067
7068         I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
7069         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
7070         I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
7071         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7072         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7073
7074         for_each_engine(engine, dev_priv, id)
7075                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7076
7077         I915_WRITE(GEN6_RC_SLEEP, 0);
7078         I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
7079         if (IS_IVYBRIDGE(dev_priv))
7080                 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
7081         else
7082                 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
7083         I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
7084         I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
7085
7086         /* We don't use those on Haswell */
7087         rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
7088         if (HAS_RC6p(dev_priv))
7089                 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
7090         if (HAS_RC6pp(dev_priv))
7091                 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
7092         I915_WRITE(GEN6_RC_CONTROL,
7093                    rc6_mask |
7094                    GEN6_RC_CTL_EI_MODE(1) |
7095                    GEN6_RC_CTL_HW_ENABLE);
7096
7097         rc6vids = 0;
7098         ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
7099         if (IS_GEN6(dev_priv) && ret) {
7100                 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
7101         } else if (IS_GEN6(dev_priv) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
7102                 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
7103                           GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
7104                 rc6vids &= 0xffff00;
7105                 rc6vids |= GEN6_ENCODE_RC6_VID(450);
7106                 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
7107                 if (ret)
7108                         DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
7109         }
7110
7111         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7112 }
7113
7114 static void gen6_enable_rps(struct drm_i915_private *dev_priv)
7115 {
7116         /* Here begins a magic sequence of register writes to enable
7117          * auto-downclocking.
7118          *
7119          * Perhaps there might be some value in exposing these to
7120          * userspace...
7121          */
7122         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7123
7124         /* Power down if completely idle for over 50ms */
7125         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
7126         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7127
7128         reset_rps(dev_priv, gen6_set_rps);
7129
7130         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7131 }
7132
7133 static void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
7134 {
7135         struct intel_rps *rps = &dev_priv->gt_pm.rps;
7136         const int min_freq = 15;
7137         const int scaling_factor = 180;
7138         unsigned int gpu_freq;
7139         unsigned int max_ia_freq, min_ring_freq;
7140         unsigned int max_gpu_freq, min_gpu_freq;
7141         struct cpufreq_policy *policy;
7142
7143         WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
7144
7145         if (rps->max_freq <= rps->min_freq)
7146                 return;
7147
7148         policy = cpufreq_cpu_get(0);
7149         if (policy) {
7150                 max_ia_freq = policy->cpuinfo.max_freq;
7151                 cpufreq_cpu_put(policy);
7152         } else {
7153                 /*
7154                  * Default to measured freq if none found, PCU will ensure we
7155                  * don't go over
7156                  */
7157                 max_ia_freq = tsc_khz;
7158         }
7159
7160         /* Convert from kHz to MHz */
7161         max_ia_freq /= 1000;
7162
7163         min_ring_freq = I915_READ(DCLK) & 0xf;
7164         /* convert DDR frequency from units of 266.6MHz to bandwidth */
7165         min_ring_freq = mult_frac(min_ring_freq, 8, 3);
7166
7167         min_gpu_freq = rps->min_freq;
7168         max_gpu_freq = rps->max_freq;
7169         if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
7170                 /* Convert GT frequency to 50 HZ units */
7171                 min_gpu_freq /= GEN9_FREQ_SCALER;
7172                 max_gpu_freq /= GEN9_FREQ_SCALER;
7173         }
7174
7175         /*
7176          * For each potential GPU frequency, load a ring frequency we'd like
7177          * to use for memory access.  We do this by specifying the IA frequency
7178          * the PCU should use as a reference to determine the ring frequency.
7179          */
7180         for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) {
7181                 const int diff = max_gpu_freq - gpu_freq;
7182                 unsigned int ia_freq = 0, ring_freq = 0;
7183
7184                 if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
7185                         /*
7186                          * ring_freq = 2 * GT. ring_freq is in 100MHz units
7187                          * No floor required for ring frequency on SKL.
7188                          */
7189                         ring_freq = gpu_freq;
7190                 } else if (INTEL_GEN(dev_priv) >= 8) {
7191                         /* max(2 * GT, DDR). NB: GT is 50MHz units */
7192                         ring_freq = max(min_ring_freq, gpu_freq);
7193                 } else if (IS_HASWELL(dev_priv)) {
7194                         ring_freq = mult_frac(gpu_freq, 5, 4);
7195                         ring_freq = max(min_ring_freq, ring_freq);
7196                         /* leave ia_freq as the default, chosen by cpufreq */
7197                 } else {
7198                         /* On older processors, there is no separate ring
7199                          * clock domain, so in order to boost the bandwidth
7200                          * of the ring, we need to upclock the CPU (ia_freq).
7201                          *
7202                          * For GPU frequencies less than 750MHz,
7203                          * just use the lowest ring freq.
7204                          */
7205                         if (gpu_freq < min_freq)
7206                                 ia_freq = 800;
7207                         else
7208                                 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
7209                         ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
7210                 }
7211
7212                 sandybridge_pcode_write(dev_priv,
7213                                         GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
7214                                         ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
7215                                         ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
7216                                         gpu_freq);
7217         }
7218 }
7219
7220 static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
7221 {
7222         u32 val, rp0;
7223
7224         val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
7225
7226         switch (INTEL_INFO(dev_priv)->sseu.eu_total) {
7227         case 8:
7228                 /* (2 * 4) config */
7229                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
7230                 break;
7231         case 12:
7232                 /* (2 * 6) config */
7233                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
7234                 break;
7235         case 16:
7236                 /* (2 * 8) config */
7237         default:
7238                 /* Setting (2 * 8) Min RP0 for any other combination */
7239                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
7240                 break;
7241         }
7242
7243         rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
7244
7245         return rp0;
7246 }
7247
7248 static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
7249 {
7250         u32 val, rpe;
7251
7252         val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
7253         rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
7254
7255         return rpe;
7256 }
7257
7258 static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
7259 {
7260         u32 val, rp1;
7261
7262         val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
7263         rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
7264
7265         return rp1;
7266 }
7267
7268 static u32 cherryview_rps_min_freq(struct drm_i915_private *dev_priv)
7269 {
7270         u32 val, rpn;
7271
7272         val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE);
7273         rpn = ((val >> FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT) &
7274                        FB_GFX_FREQ_FUSE_MASK);
7275
7276         return rpn;
7277 }
7278
7279 static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
7280 {
7281         u32 val, rp1;
7282
7283         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
7284
7285         rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
7286
7287         return rp1;
7288 }
7289
7290 static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
7291 {
7292         u32 val, rp0;
7293
7294         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
7295
7296         rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
7297         /* Clamp to max */
7298         rp0 = min_t(u32, rp0, 0xea);
7299
7300         return rp0;
7301 }
7302
7303 static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
7304 {
7305         u32 val, rpe;
7306
7307         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
7308         rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
7309         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
7310         rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
7311
7312         return rpe;
7313 }
7314
7315 static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
7316 {
7317         u32 val;
7318
7319         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
7320         /*
7321          * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
7322          * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
7323          * a BYT-M B0 the above register contains 0xbf. Moreover when setting
7324          * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
7325          * to make sure it matches what Punit accepts.
7326          */
7327         return max_t(u32, val, 0xc0);
7328 }
7329
7330 /* Check that the pctx buffer wasn't move under us. */
7331 static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
7332 {
7333         unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
7334
7335         WARN_ON(pctx_addr != dev_priv->dsm.start +
7336                              dev_priv->vlv_pctx->stolen->start);
7337 }
7338
7339
7340 /* Check that the pcbr address is not empty. */
7341 static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
7342 {
7343         unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
7344
7345         WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
7346 }
7347
7348 static void cherryview_setup_pctx(struct drm_i915_private *dev_priv)
7349 {
7350         resource_size_t pctx_paddr, paddr;
7351         resource_size_t pctx_size = 32*1024;
7352         u32 pcbr;
7353
7354         pcbr = I915_READ(VLV_PCBR);
7355         if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
7356                 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
7357                 paddr = dev_priv->dsm.end + 1 - pctx_size;
7358                 GEM_BUG_ON(paddr > U32_MAX);
7359
7360                 pctx_paddr = (paddr & (~4095));
7361                 I915_WRITE(VLV_PCBR, pctx_paddr);
7362         }
7363
7364         DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
7365 }
7366
7367 static void valleyview_setup_pctx(struct drm_i915_private *dev_priv)
7368 {
7369         struct drm_i915_gem_object *pctx;
7370         resource_size_t pctx_paddr;
7371         resource_size_t pctx_size = 24*1024;
7372         u32 pcbr;
7373
7374         pcbr = I915_READ(VLV_PCBR);
7375         if (pcbr) {
7376                 /* BIOS set it up already, grab the pre-alloc'd space */
7377                 resource_size_t pcbr_offset;
7378
7379                 pcbr_offset = (pcbr & (~4095)) - dev_priv->dsm.start;
7380                 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv,
7381                                                                       pcbr_offset,
7382                                                                       I915_GTT_OFFSET_NONE,
7383                                                                       pctx_size);
7384                 goto out;
7385         }
7386
7387         DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
7388
7389         /*
7390          * From the Gunit register HAS:
7391          * The Gfx driver is expected to program this register and ensure
7392          * proper allocation within Gfx stolen memory.  For example, this
7393          * register should be programmed such than the PCBR range does not
7394          * overlap with other ranges, such as the frame buffer, protected
7395          * memory, or any other relevant ranges.
7396          */
7397         pctx = i915_gem_object_create_stolen(dev_priv, pctx_size);
7398         if (!pctx) {
7399                 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
7400                 goto out;
7401         }
7402
7403         GEM_BUG_ON(range_overflows_t(u64,
7404                                      dev_priv->dsm.start,
7405                                      pctx->stolen->start,
7406                                      U32_MAX));
7407         pctx_paddr = dev_priv->dsm.start + pctx->stolen->start;
7408         I915_WRITE(VLV_PCBR, pctx_paddr);
7409
7410 out:
7411         DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
7412         dev_priv->vlv_pctx = pctx;
7413 }
7414
7415 static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv)
7416 {
7417         struct drm_i915_gem_object *pctx;
7418
7419         pctx = fetch_and_zero(&dev_priv->vlv_pctx);
7420         if (pctx)
7421                 i915_gem_object_put(pctx);
7422 }
7423
7424 static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv)
7425 {
7426         dev_priv->gt_pm.rps.gpll_ref_freq =
7427                 vlv_get_cck_clock(dev_priv, "GPLL ref",
7428                                   CCK_GPLL_CLOCK_CONTROL,
7429                                   dev_priv->czclk_freq);
7430
7431         DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n",
7432                          dev_priv->gt_pm.rps.gpll_ref_freq);
7433 }
7434
7435 static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv)
7436 {
7437         struct intel_rps *rps = &dev_priv->gt_pm.rps;
7438         u32 val;
7439
7440         valleyview_setup_pctx(dev_priv);
7441
7442         vlv_init_gpll_ref_freq(dev_priv);
7443
7444         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7445         switch ((val >> 6) & 3) {
7446         case 0:
7447         case 1:
7448                 dev_priv->mem_freq = 800;
7449                 break;
7450         case 2:
7451                 dev_priv->mem_freq = 1066;
7452                 break;
7453         case 3:
7454                 dev_priv->mem_freq = 1333;
7455                 break;
7456         }
7457         DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
7458
7459         rps->max_freq = valleyview_rps_max_freq(dev_priv);
7460         rps->rp0_freq = rps->max_freq;
7461         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
7462                          intel_gpu_freq(dev_priv, rps->max_freq),
7463                          rps->max_freq);
7464
7465         rps->efficient_freq = valleyview_rps_rpe_freq(dev_priv);
7466         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
7467                          intel_gpu_freq(dev_priv, rps->efficient_freq),
7468                          rps->efficient_freq);
7469
7470         rps->rp1_freq = valleyview_rps_guar_freq(dev_priv);
7471         DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
7472                          intel_gpu_freq(dev_priv, rps->rp1_freq),
7473                          rps->rp1_freq);
7474
7475         rps->min_freq = valleyview_rps_min_freq(dev_priv);
7476         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
7477                          intel_gpu_freq(dev_priv, rps->min_freq),
7478                          rps->min_freq);
7479 }
7480
7481 static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
7482 {
7483         struct intel_rps *rps = &dev_priv->gt_pm.rps;
7484         u32 val;
7485
7486         cherryview_setup_pctx(dev_priv);
7487
7488         vlv_init_gpll_ref_freq(dev_priv);
7489
7490         mutex_lock(&dev_priv->sb_lock);
7491         val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
7492         mutex_unlock(&dev_priv->sb_lock);
7493
7494         switch ((val >> 2) & 0x7) {
7495         case 3:
7496                 dev_priv->mem_freq = 2000;
7497                 break;
7498         default:
7499                 dev_priv->mem_freq = 1600;
7500                 break;
7501         }
7502         DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
7503
7504         rps->max_freq = cherryview_rps_max_freq(dev_priv);
7505         rps->rp0_freq = rps->max_freq;
7506         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
7507                          intel_gpu_freq(dev_priv, rps->max_freq),
7508                          rps->max_freq);
7509
7510         rps->efficient_freq = cherryview_rps_rpe_freq(dev_priv);
7511         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
7512                          intel_gpu_freq(dev_priv, rps->efficient_freq),
7513                          rps->efficient_freq);
7514
7515         rps->rp1_freq = cherryview_rps_guar_freq(dev_priv);
7516         DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
7517                          intel_gpu_freq(dev_priv, rps->rp1_freq),
7518                          rps->rp1_freq);
7519
7520         rps->min_freq = cherryview_rps_min_freq(dev_priv);
7521         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
7522                          intel_gpu_freq(dev_priv, rps->min_freq),
7523                          rps->min_freq);
7524
7525         WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq |
7526                    rps->min_freq) & 1,
7527                   "Odd GPU freq values\n");
7528 }
7529
7530 static void valleyview_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
7531 {
7532         valleyview_cleanup_pctx(dev_priv);
7533 }
7534
7535 static void cherryview_enable_rc6(struct drm_i915_private *dev_priv)
7536 {
7537         struct intel_engine_cs *engine;
7538         enum intel_engine_id id;
7539         u32 gtfifodbg, rc6_mode, pcbr;
7540
7541         gtfifodbg = I915_READ(GTFIFODBG) & ~(GT_FIFO_SBDEDICATE_FREE_ENTRY_CHV |
7542                                              GT_FIFO_FREE_ENTRIES_CHV);
7543         if (gtfifodbg) {
7544                 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
7545                                  gtfifodbg);
7546                 I915_WRITE(GTFIFODBG, gtfifodbg);
7547         }
7548
7549         cherryview_check_pctx(dev_priv);
7550
7551         /* 1a & 1b: Get forcewake during program sequence. Although the driver
7552          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
7553         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7554
7555         /*  Disable RC states. */
7556         I915_WRITE(GEN6_RC_CONTROL, 0);
7557
7558         /* 2a: Program RC6 thresholds.*/
7559         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
7560         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
7561         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
7562
7563         for_each_engine(engine, dev_priv, id)
7564                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7565         I915_WRITE(GEN6_RC_SLEEP, 0);
7566
7567         /* TO threshold set to 500 us ( 0x186 * 1.28 us) */
7568         I915_WRITE(GEN6_RC6_THRESHOLD, 0x186);
7569
7570         /* Allows RC6 residency counter to work */
7571         I915_WRITE(VLV_COUNTER_CONTROL,
7572                    _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
7573                                       VLV_MEDIA_RC6_COUNT_EN |
7574                                       VLV_RENDER_RC6_COUNT_EN));
7575
7576         /* For now we assume BIOS is allocating and populating the PCBR  */
7577         pcbr = I915_READ(VLV_PCBR);
7578
7579         /* 3: Enable RC6 */
7580         rc6_mode = 0;
7581         if (pcbr >> VLV_PCBR_ADDR_SHIFT)
7582                 rc6_mode = GEN7_RC_CTL_TO_MODE;
7583         I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
7584
7585         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7586 }
7587
7588 static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
7589 {
7590         u32 val;
7591
7592         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7593
7594         /* 1: Program defaults and thresholds for RPS*/
7595         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
7596         I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
7597         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
7598         I915_WRITE(GEN6_RP_UP_EI, 66000);
7599         I915_WRITE(GEN6_RP_DOWN_EI, 350000);
7600
7601         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7602
7603         /* 2: Enable RPS */
7604         I915_WRITE(GEN6_RP_CONTROL,
7605                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
7606                    GEN6_RP_MEDIA_IS_GFX |
7607                    GEN6_RP_ENABLE |
7608                    GEN6_RP_UP_BUSY_AVG |
7609                    GEN6_RP_DOWN_IDLE_AVG);
7610
7611         /* Setting Fixed Bias */
7612         val = VLV_OVERRIDE_EN |
7613                   VLV_SOC_TDP_EN |
7614                   CHV_BIAS_CPU_50_SOC_50;
7615         vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
7616
7617         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7618
7619         /* RPS code assumes GPLL is used */
7620         WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
7621
7622         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
7623         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
7624
7625         reset_rps(dev_priv, valleyview_set_rps);
7626
7627         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7628 }
7629
7630 static void valleyview_enable_rc6(struct drm_i915_private *dev_priv)
7631 {
7632         struct intel_engine_cs *engine;
7633         enum intel_engine_id id;
7634         u32 gtfifodbg;
7635
7636         valleyview_check_pctx(dev_priv);
7637
7638         gtfifodbg = I915_READ(GTFIFODBG);
7639         if (gtfifodbg) {
7640                 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
7641                                  gtfifodbg);
7642                 I915_WRITE(GTFIFODBG, gtfifodbg);
7643         }
7644
7645         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7646
7647         /*  Disable RC states. */
7648         I915_WRITE(GEN6_RC_CONTROL, 0);
7649
7650         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
7651         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7652         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7653
7654         for_each_engine(engine, dev_priv, id)
7655                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7656
7657         I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
7658
7659         /* Allows RC6 residency counter to work */
7660         I915_WRITE(VLV_COUNTER_CONTROL,
7661                    _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
7662                                       VLV_MEDIA_RC0_COUNT_EN |
7663                                       VLV_RENDER_RC0_COUNT_EN |
7664                                       VLV_MEDIA_RC6_COUNT_EN |
7665                                       VLV_RENDER_RC6_COUNT_EN));
7666
7667         I915_WRITE(GEN6_RC_CONTROL,
7668                    GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL);
7669
7670         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7671 }
7672
7673 static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
7674 {
7675         u32 val;
7676
7677         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7678
7679         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
7680         I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
7681         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
7682         I915_WRITE(GEN6_RP_UP_EI, 66000);
7683         I915_WRITE(GEN6_RP_DOWN_EI, 350000);
7684
7685         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7686
7687         I915_WRITE(GEN6_RP_CONTROL,
7688                    GEN6_RP_MEDIA_TURBO |
7689                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
7690                    GEN6_RP_MEDIA_IS_GFX |
7691                    GEN6_RP_ENABLE |
7692                    GEN6_RP_UP_BUSY_AVG |
7693                    GEN6_RP_DOWN_IDLE_CONT);
7694
7695         /* Setting Fixed Bias */
7696         val = VLV_OVERRIDE_EN |
7697                   VLV_SOC_TDP_EN |
7698                   VLV_BIAS_CPU_125_SOC_875;
7699         vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
7700
7701         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7702
7703         /* RPS code assumes GPLL is used */
7704         WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
7705
7706         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
7707         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
7708
7709         reset_rps(dev_priv, valleyview_set_rps);
7710
7711         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7712 }
7713
7714 static unsigned long intel_pxfreq(u32 vidfreq)
7715 {
7716         unsigned long freq;
7717         int div = (vidfreq & 0x3f0000) >> 16;
7718         int post = (vidfreq & 0x3000) >> 12;
7719         int pre = (vidfreq & 0x7);
7720
7721         if (!pre)
7722                 return 0;
7723
7724         freq = ((div * 133333) / ((1<<post) * pre));
7725
7726         return freq;
7727 }
7728
7729 static const struct cparams {
7730         u16 i;
7731         u16 t;
7732         u16 m;
7733         u16 c;
7734 } cparams[] = {
7735         { 1, 1333, 301, 28664 },
7736         { 1, 1066, 294, 24460 },
7737         { 1, 800, 294, 25192 },
7738         { 0, 1333, 276, 27605 },
7739         { 0, 1066, 276, 27605 },
7740         { 0, 800, 231, 23784 },
7741 };
7742
7743 static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
7744 {
7745         u64 total_count, diff, ret;
7746         u32 count1, count2, count3, m = 0, c = 0;
7747         unsigned long now = jiffies_to_msecs(jiffies), diff1;
7748         int i;
7749
7750         lockdep_assert_held(&mchdev_lock);
7751
7752         diff1 = now - dev_priv->ips.last_time1;
7753
7754         /* Prevent division-by-zero if we are asking too fast.
7755          * Also, we don't get interesting results if we are polling
7756          * faster than once in 10ms, so just return the saved value
7757          * in such cases.
7758          */
7759         if (diff1 <= 10)
7760                 return dev_priv->ips.chipset_power;
7761
7762         count1 = I915_READ(DMIEC);
7763         count2 = I915_READ(DDREC);
7764         count3 = I915_READ(CSIEC);
7765
7766         total_count = count1 + count2 + count3;
7767
7768         /* FIXME: handle per-counter overflow */
7769         if (total_count < dev_priv->ips.last_count1) {
7770                 diff = ~0UL - dev_priv->ips.last_count1;
7771                 diff += total_count;
7772         } else {
7773                 diff = total_count - dev_priv->ips.last_count1;
7774         }
7775
7776         for (i = 0; i < ARRAY_SIZE(cparams); i++) {
7777                 if (cparams[i].i == dev_priv->ips.c_m &&
7778                     cparams[i].t == dev_priv->ips.r_t) {
7779                         m = cparams[i].m;
7780                         c = cparams[i].c;
7781                         break;
7782                 }
7783         }
7784
7785         diff = div_u64(diff, diff1);
7786         ret = ((m * diff) + c);
7787         ret = div_u64(ret, 10);
7788
7789         dev_priv->ips.last_count1 = total_count;
7790         dev_priv->ips.last_time1 = now;
7791
7792         dev_priv->ips.chipset_power = ret;
7793
7794         return ret;
7795 }
7796
7797 unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
7798 {
7799         unsigned long val;
7800
7801         if (!IS_GEN5(dev_priv))
7802                 return 0;
7803
7804         spin_lock_irq(&mchdev_lock);
7805
7806         val = __i915_chipset_val(dev_priv);
7807
7808         spin_unlock_irq(&mchdev_lock);
7809
7810         return val;
7811 }
7812
7813 unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
7814 {
7815         unsigned long m, x, b;
7816         u32 tsfs;
7817
7818         tsfs = I915_READ(TSFS);
7819
7820         m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
7821         x = I915_READ8(TR1);
7822
7823         b = tsfs & TSFS_INTR_MASK;
7824
7825         return ((m * x) / 127) - b;
7826 }
7827
7828 static int _pxvid_to_vd(u8 pxvid)
7829 {
7830         if (pxvid == 0)
7831                 return 0;
7832
7833         if (pxvid >= 8 && pxvid < 31)
7834                 pxvid = 31;
7835
7836         return (pxvid + 2) * 125;
7837 }
7838
7839 static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
7840 {
7841         const int vd = _pxvid_to_vd(pxvid);
7842         const int vm = vd - 1125;
7843
7844         if (INTEL_INFO(dev_priv)->is_mobile)
7845                 return vm > 0 ? vm : 0;
7846
7847         return vd;
7848 }
7849
7850 static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
7851 {
7852         u64 now, diff, diffms;
7853         u32 count;
7854
7855         lockdep_assert_held(&mchdev_lock);
7856
7857         now = ktime_get_raw_ns();
7858         diffms = now - dev_priv->ips.last_time2;
7859         do_div(diffms, NSEC_PER_MSEC);
7860
7861         /* Don't divide by 0 */
7862         if (!diffms)
7863                 return;
7864
7865         count = I915_READ(GFXEC);
7866
7867         if (count < dev_priv->ips.last_count2) {
7868                 diff = ~0UL - dev_priv->ips.last_count2;
7869                 diff += count;
7870         } else {
7871                 diff = count - dev_priv->ips.last_count2;
7872         }
7873
7874         dev_priv->ips.last_count2 = count;
7875         dev_priv->ips.last_time2 = now;
7876
7877         /* More magic constants... */
7878         diff = diff * 1181;
7879         diff = div_u64(diff, diffms * 10);
7880         dev_priv->ips.gfx_power = diff;
7881 }
7882
7883 void i915_update_gfx_val(struct drm_i915_private *dev_priv)
7884 {
7885         if (!IS_GEN5(dev_priv))
7886                 return;
7887
7888         spin_lock_irq(&mchdev_lock);
7889
7890         __i915_update_gfx_val(dev_priv);
7891
7892         spin_unlock_irq(&mchdev_lock);
7893 }
7894
7895 static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
7896 {
7897         unsigned long t, corr, state1, corr2, state2;
7898         u32 pxvid, ext_v;
7899
7900         lockdep_assert_held(&mchdev_lock);
7901
7902         pxvid = I915_READ(PXVFREQ(dev_priv->gt_pm.rps.cur_freq));
7903         pxvid = (pxvid >> 24) & 0x7f;
7904         ext_v = pvid_to_extvid(dev_priv, pxvid);
7905
7906         state1 = ext_v;
7907
7908         t = i915_mch_val(dev_priv);
7909
7910         /* Revel in the empirically derived constants */
7911
7912         /* Correction factor in 1/100000 units */
7913         if (t > 80)
7914                 corr = ((t * 2349) + 135940);
7915         else if (t >= 50)
7916                 corr = ((t * 964) + 29317);
7917         else /* < 50 */
7918                 corr = ((t * 301) + 1004);
7919
7920         corr = corr * ((150142 * state1) / 10000 - 78642);
7921         corr /= 100000;
7922         corr2 = (corr * dev_priv->ips.corr);
7923
7924         state2 = (corr2 * state1) / 10000;
7925         state2 /= 100; /* convert to mW */
7926
7927         __i915_update_gfx_val(dev_priv);
7928
7929         return dev_priv->ips.gfx_power + state2;
7930 }
7931
7932 unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
7933 {
7934         unsigned long val;
7935
7936         if (!IS_GEN5(dev_priv))
7937                 return 0;
7938
7939         spin_lock_irq(&mchdev_lock);
7940
7941         val = __i915_gfx_val(dev_priv);
7942
7943         spin_unlock_irq(&mchdev_lock);
7944
7945         return val;
7946 }
7947
7948 /**
7949  * i915_read_mch_val - return value for IPS use
7950  *
7951  * Calculate and return a value for the IPS driver to use when deciding whether
7952  * we have thermal and power headroom to increase CPU or GPU power budget.
7953  */
7954 unsigned long i915_read_mch_val(void)
7955 {
7956         struct drm_i915_private *dev_priv;
7957         unsigned long chipset_val, graphics_val, ret = 0;
7958
7959         spin_lock_irq(&mchdev_lock);
7960         if (!i915_mch_dev)
7961                 goto out_unlock;
7962         dev_priv = i915_mch_dev;
7963
7964         chipset_val = __i915_chipset_val(dev_priv);
7965         graphics_val = __i915_gfx_val(dev_priv);
7966
7967         ret = chipset_val + graphics_val;
7968
7969 out_unlock:
7970         spin_unlock_irq(&mchdev_lock);
7971
7972         return ret;
7973 }
7974 EXPORT_SYMBOL_GPL(i915_read_mch_val);
7975
7976 /**
7977  * i915_gpu_raise - raise GPU frequency limit
7978  *
7979  * Raise the limit; IPS indicates we have thermal headroom.
7980  */
7981 bool i915_gpu_raise(void)
7982 {
7983         struct drm_i915_private *dev_priv;
7984         bool ret = true;
7985
7986         spin_lock_irq(&mchdev_lock);
7987         if (!i915_mch_dev) {
7988                 ret = false;
7989                 goto out_unlock;
7990         }
7991         dev_priv = i915_mch_dev;
7992
7993         if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
7994                 dev_priv->ips.max_delay--;
7995
7996 out_unlock:
7997         spin_unlock_irq(&mchdev_lock);
7998
7999         return ret;
8000 }
8001 EXPORT_SYMBOL_GPL(i915_gpu_raise);
8002
8003 /**
8004  * i915_gpu_lower - lower GPU frequency limit
8005  *
8006  * IPS indicates we're close to a thermal limit, so throttle back the GPU
8007  * frequency maximum.
8008  */
8009 bool i915_gpu_lower(void)
8010 {
8011         struct drm_i915_private *dev_priv;
8012         bool ret = true;
8013
8014         spin_lock_irq(&mchdev_lock);
8015         if (!i915_mch_dev) {
8016                 ret = false;
8017                 goto out_unlock;
8018         }
8019         dev_priv = i915_mch_dev;
8020
8021         if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
8022                 dev_priv->ips.max_delay++;
8023
8024 out_unlock:
8025         spin_unlock_irq(&mchdev_lock);
8026
8027         return ret;
8028 }
8029 EXPORT_SYMBOL_GPL(i915_gpu_lower);
8030
8031 /**
8032  * i915_gpu_busy - indicate GPU business to IPS
8033  *
8034  * Tell the IPS driver whether or not the GPU is busy.
8035  */
8036 bool i915_gpu_busy(void)
8037 {
8038         bool ret = false;
8039
8040         spin_lock_irq(&mchdev_lock);
8041         if (i915_mch_dev)
8042                 ret = i915_mch_dev->gt.awake;
8043         spin_unlock_irq(&mchdev_lock);
8044
8045         return ret;
8046 }
8047 EXPORT_SYMBOL_GPL(i915_gpu_busy);
8048
8049 /**
8050  * i915_gpu_turbo_disable - disable graphics turbo
8051  *
8052  * Disable graphics turbo by resetting the max frequency and setting the
8053  * current frequency to the default.
8054  */
8055 bool i915_gpu_turbo_disable(void)
8056 {
8057         struct drm_i915_private *dev_priv;
8058         bool ret = true;
8059
8060         spin_lock_irq(&mchdev_lock);
8061         if (!i915_mch_dev) {
8062                 ret = false;
8063                 goto out_unlock;
8064         }
8065         dev_priv = i915_mch_dev;
8066
8067         dev_priv->ips.max_delay = dev_priv->ips.fstart;
8068
8069         if (!ironlake_set_drps(dev_priv, dev_priv->ips.fstart))
8070                 ret = false;
8071
8072 out_unlock:
8073         spin_unlock_irq(&mchdev_lock);
8074
8075         return ret;
8076 }
8077 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
8078
8079 /**
8080  * Tells the intel_ips driver that the i915 driver is now loaded, if
8081  * IPS got loaded first.
8082  *
8083  * This awkward dance is so that neither module has to depend on the
8084  * other in order for IPS to do the appropriate communication of
8085  * GPU turbo limits to i915.
8086  */
8087 static void
8088 ips_ping_for_i915_load(void)
8089 {
8090         void (*link)(void);
8091
8092         link = symbol_get(ips_link_to_i915_driver);
8093         if (link) {
8094                 link();
8095                 symbol_put(ips_link_to_i915_driver);
8096         }
8097 }
8098
8099 void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
8100 {
8101         /* We only register the i915 ips part with intel-ips once everything is
8102          * set up, to avoid intel-ips sneaking in and reading bogus values. */
8103         spin_lock_irq(&mchdev_lock);
8104         i915_mch_dev = dev_priv;
8105         spin_unlock_irq(&mchdev_lock);
8106
8107         ips_ping_for_i915_load();
8108 }
8109
8110 void intel_gpu_ips_teardown(void)
8111 {
8112         spin_lock_irq(&mchdev_lock);
8113         i915_mch_dev = NULL;
8114         spin_unlock_irq(&mchdev_lock);
8115 }
8116
8117 static void intel_init_emon(struct drm_i915_private *dev_priv)
8118 {
8119         u32 lcfuse;
8120         u8 pxw[16];
8121         int i;
8122
8123         /* Disable to program */
8124         I915_WRITE(ECR, 0);
8125         POSTING_READ(ECR);
8126
8127         /* Program energy weights for various events */
8128         I915_WRITE(SDEW, 0x15040d00);
8129         I915_WRITE(CSIEW0, 0x007f0000);
8130         I915_WRITE(CSIEW1, 0x1e220004);
8131         I915_WRITE(CSIEW2, 0x04000004);
8132
8133         for (i = 0; i < 5; i++)
8134                 I915_WRITE(PEW(i), 0);
8135         for (i = 0; i < 3; i++)
8136                 I915_WRITE(DEW(i), 0);
8137
8138         /* Program P-state weights to account for frequency power adjustment */
8139         for (i = 0; i < 16; i++) {
8140                 u32 pxvidfreq = I915_READ(PXVFREQ(i));
8141                 unsigned long freq = intel_pxfreq(pxvidfreq);
8142                 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
8143                         PXVFREQ_PX_SHIFT;
8144                 unsigned long val;
8145
8146                 val = vid * vid;
8147                 val *= (freq / 1000);
8148                 val *= 255;
8149                 val /= (127*127*900);
8150                 if (val > 0xff)
8151                         DRM_ERROR("bad pxval: %ld\n", val);
8152                 pxw[i] = val;
8153         }
8154         /* Render standby states get 0 weight */
8155         pxw[14] = 0;
8156         pxw[15] = 0;
8157
8158         for (i = 0; i < 4; i++) {
8159                 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
8160                         (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
8161                 I915_WRITE(PXW(i), val);
8162         }
8163
8164         /* Adjust magic regs to magic values (more experimental results) */
8165         I915_WRITE(OGW0, 0);
8166         I915_WRITE(OGW1, 0);
8167         I915_WRITE(EG0, 0x00007f00);
8168         I915_WRITE(EG1, 0x0000000e);
8169         I915_WRITE(EG2, 0x000e0000);
8170         I915_WRITE(EG3, 0x68000300);
8171         I915_WRITE(EG4, 0x42000000);
8172         I915_WRITE(EG5, 0x00140031);
8173         I915_WRITE(EG6, 0);
8174         I915_WRITE(EG7, 0);
8175
8176         for (i = 0; i < 8; i++)
8177                 I915_WRITE(PXWL(i), 0);
8178
8179         /* Enable PMON + select events */
8180         I915_WRITE(ECR, 0x80000019);
8181
8182         lcfuse = I915_READ(LCFUSE02);
8183
8184         dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
8185 }
8186
8187 void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
8188 {
8189         struct intel_rps *rps = &dev_priv->gt_pm.rps;
8190
8191         /*
8192          * RPM depends on RC6 to save restore the GT HW context, so make RC6 a
8193          * requirement.
8194          */
8195         if (!sanitize_rc6(dev_priv)) {
8196                 DRM_INFO("RC6 disabled, disabling runtime PM support\n");
8197                 pm_runtime_get(&dev_priv->drm.pdev->dev);
8198         }
8199
8200         mutex_lock(&dev_priv->pcu_lock);
8201
8202         /* Initialize RPS limits (for userspace) */
8203         if (IS_CHERRYVIEW(dev_priv))
8204                 cherryview_init_gt_powersave(dev_priv);
8205         else if (IS_VALLEYVIEW(dev_priv))
8206                 valleyview_init_gt_powersave(dev_priv);
8207         else if (INTEL_GEN(dev_priv) >= 6)
8208                 gen6_init_rps_frequencies(dev_priv);
8209
8210         /* Derive initial user preferences/limits from the hardware limits */
8211         rps->idle_freq = rps->min_freq;
8212         rps->cur_freq = rps->idle_freq;
8213
8214         rps->max_freq_softlimit = rps->max_freq;
8215         rps->min_freq_softlimit = rps->min_freq;
8216
8217         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
8218                 rps->min_freq_softlimit =
8219                         max_t(int,
8220                               rps->efficient_freq,
8221                               intel_freq_opcode(dev_priv, 450));
8222
8223         /* After setting max-softlimit, find the overclock max freq */
8224         if (IS_GEN6(dev_priv) ||
8225             IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) {
8226                 u32 params = 0;
8227
8228                 sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &params);
8229                 if (params & BIT(31)) { /* OC supported */
8230                         DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n",
8231                                          (rps->max_freq & 0xff) * 50,
8232                                          (params & 0xff) * 50);
8233                         rps->max_freq = params & 0xff;
8234                 }
8235         }
8236
8237         /* Finally allow us to boost to max by default */
8238         rps->boost_freq = rps->max_freq;
8239
8240         mutex_unlock(&dev_priv->pcu_lock);
8241 }
8242
8243 void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
8244 {
8245         if (IS_VALLEYVIEW(dev_priv))
8246                 valleyview_cleanup_gt_powersave(dev_priv);
8247
8248         if (!HAS_RC6(dev_priv))
8249                 pm_runtime_put(&dev_priv->drm.pdev->dev);
8250 }
8251
8252 /**
8253  * intel_suspend_gt_powersave - suspend PM work and helper threads
8254  * @dev_priv: i915 device
8255  *
8256  * We don't want to disable RC6 or other features here, we just want
8257  * to make sure any work we've queued has finished and won't bother
8258  * us while we're suspended.
8259  */
8260 void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv)
8261 {
8262         if (INTEL_GEN(dev_priv) < 6)
8263                 return;
8264
8265         /* gen6_rps_idle() will be called later to disable interrupts */
8266 }
8267
8268 void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv)
8269 {
8270         dev_priv->gt_pm.rps.enabled = true; /* force RPS disabling */
8271         dev_priv->gt_pm.rc6.enabled = true; /* force RC6 disabling */
8272         intel_disable_gt_powersave(dev_priv);
8273
8274         if (INTEL_GEN(dev_priv) >= 11)
8275                 gen11_reset_rps_interrupts(dev_priv);
8276         else if (INTEL_GEN(dev_priv) >= 6)
8277                 gen6_reset_rps_interrupts(dev_priv);
8278 }
8279
8280 static inline void intel_disable_llc_pstate(struct drm_i915_private *i915)
8281 {
8282         lockdep_assert_held(&i915->pcu_lock);
8283
8284         if (!i915->gt_pm.llc_pstate.enabled)
8285                 return;
8286
8287         /* Currently there is no HW configuration to be done to disable. */
8288
8289         i915->gt_pm.llc_pstate.enabled = false;
8290 }
8291
8292 static void intel_disable_rc6(struct drm_i915_private *dev_priv)
8293 {
8294         lockdep_assert_held(&dev_priv->pcu_lock);
8295
8296         if (!dev_priv->gt_pm.rc6.enabled)
8297                 return;
8298
8299         if (INTEL_GEN(dev_priv) >= 9)
8300                 gen9_disable_rc6(dev_priv);
8301         else if (IS_CHERRYVIEW(dev_priv))
8302                 cherryview_disable_rc6(dev_priv);
8303         else if (IS_VALLEYVIEW(dev_priv))
8304                 valleyview_disable_rc6(dev_priv);
8305         else if (INTEL_GEN(dev_priv) >= 6)
8306                 gen6_disable_rc6(dev_priv);
8307
8308         dev_priv->gt_pm.rc6.enabled = false;
8309 }
8310
8311 static void intel_disable_rps(struct drm_i915_private *dev_priv)
8312 {
8313         lockdep_assert_held(&dev_priv->pcu_lock);
8314
8315         if (!dev_priv->gt_pm.rps.enabled)
8316                 return;
8317
8318         if (INTEL_GEN(dev_priv) >= 9)
8319                 gen9_disable_rps(dev_priv);
8320         else if (IS_CHERRYVIEW(dev_priv))
8321                 cherryview_disable_rps(dev_priv);
8322         else if (IS_VALLEYVIEW(dev_priv))
8323                 valleyview_disable_rps(dev_priv);
8324         else if (INTEL_GEN(dev_priv) >= 6)
8325                 gen6_disable_rps(dev_priv);
8326         else if (IS_IRONLAKE_M(dev_priv))
8327                 ironlake_disable_drps(dev_priv);
8328
8329         dev_priv->gt_pm.rps.enabled = false;
8330 }
8331
8332 void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
8333 {
8334         mutex_lock(&dev_priv->pcu_lock);
8335
8336         intel_disable_rc6(dev_priv);
8337         intel_disable_rps(dev_priv);
8338         if (HAS_LLC(dev_priv))
8339                 intel_disable_llc_pstate(dev_priv);
8340
8341         mutex_unlock(&dev_priv->pcu_lock);
8342 }
8343
8344 static inline void intel_enable_llc_pstate(struct drm_i915_private *i915)
8345 {
8346         lockdep_assert_held(&i915->pcu_lock);
8347
8348         if (i915->gt_pm.llc_pstate.enabled)
8349                 return;
8350
8351         gen6_update_ring_freq(i915);
8352
8353         i915->gt_pm.llc_pstate.enabled = true;
8354 }
8355
8356 static void intel_enable_rc6(struct drm_i915_private *dev_priv)
8357 {
8358         lockdep_assert_held(&dev_priv->pcu_lock);
8359
8360         if (dev_priv->gt_pm.rc6.enabled)
8361                 return;
8362
8363         if (IS_CHERRYVIEW(dev_priv))
8364                 cherryview_enable_rc6(dev_priv);
8365         else if (IS_VALLEYVIEW(dev_priv))
8366                 valleyview_enable_rc6(dev_priv);
8367         else if (INTEL_GEN(dev_priv) >= 9)
8368                 gen9_enable_rc6(dev_priv);
8369         else if (IS_BROADWELL(dev_priv))
8370                 gen8_enable_rc6(dev_priv);
8371         else if (INTEL_GEN(dev_priv) >= 6)
8372                 gen6_enable_rc6(dev_priv);
8373
8374         dev_priv->gt_pm.rc6.enabled = true;
8375 }
8376
8377 static void intel_enable_rps(struct drm_i915_private *dev_priv)
8378 {
8379         struct intel_rps *rps = &dev_priv->gt_pm.rps;
8380
8381         lockdep_assert_held(&dev_priv->pcu_lock);
8382
8383         if (rps->enabled)
8384                 return;
8385
8386         if (IS_CHERRYVIEW(dev_priv)) {
8387                 cherryview_enable_rps(dev_priv);
8388         } else if (IS_VALLEYVIEW(dev_priv)) {
8389                 valleyview_enable_rps(dev_priv);
8390         } else if (INTEL_GEN(dev_priv) >= 9) {
8391                 gen9_enable_rps(dev_priv);
8392         } else if (IS_BROADWELL(dev_priv)) {
8393                 gen8_enable_rps(dev_priv);
8394         } else if (INTEL_GEN(dev_priv) >= 6) {
8395                 gen6_enable_rps(dev_priv);
8396         } else if (IS_IRONLAKE_M(dev_priv)) {
8397                 ironlake_enable_drps(dev_priv);
8398                 intel_init_emon(dev_priv);
8399         }
8400
8401         WARN_ON(rps->max_freq < rps->min_freq);
8402         WARN_ON(rps->idle_freq > rps->max_freq);
8403
8404         WARN_ON(rps->efficient_freq < rps->min_freq);
8405         WARN_ON(rps->efficient_freq > rps->max_freq);
8406
8407         rps->enabled = true;
8408 }
8409
8410 void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
8411 {
8412         /* Powersaving is controlled by the host when inside a VM */
8413         if (intel_vgpu_active(dev_priv))
8414                 return;
8415
8416         mutex_lock(&dev_priv->pcu_lock);
8417
8418         if (HAS_RC6(dev_priv))
8419                 intel_enable_rc6(dev_priv);
8420         intel_enable_rps(dev_priv);
8421         if (HAS_LLC(dev_priv))
8422                 intel_enable_llc_pstate(dev_priv);
8423
8424         mutex_unlock(&dev_priv->pcu_lock);
8425 }
8426
8427 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
8428 {
8429         /*
8430          * On Ibex Peak and Cougar Point, we need to disable clock
8431          * gating for the panel power sequencer or it will fail to
8432          * start up when no ports are active.
8433          */
8434         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
8435 }
8436
8437 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
8438 {
8439         enum pipe pipe;
8440
8441         for_each_pipe(dev_priv, pipe) {
8442                 I915_WRITE(DSPCNTR(pipe),
8443                            I915_READ(DSPCNTR(pipe)) |
8444                            DISPPLANE_TRICKLE_FEED_DISABLE);
8445
8446                 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
8447                 POSTING_READ(DSPSURF(pipe));
8448         }
8449 }
8450
8451 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
8452 {
8453         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
8454
8455         /*
8456          * Required for FBC
8457          * WaFbcDisableDpfcClockGating:ilk
8458          */
8459         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
8460                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
8461                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
8462
8463         I915_WRITE(PCH_3DCGDIS0,
8464                    MARIUNIT_CLOCK_GATE_DISABLE |
8465                    SVSMUNIT_CLOCK_GATE_DISABLE);
8466         I915_WRITE(PCH_3DCGDIS1,
8467                    VFMUNIT_CLOCK_GATE_DISABLE);
8468
8469         /*
8470          * According to the spec the following bits should be set in
8471          * order to enable memory self-refresh
8472          * The bit 22/21 of 0x42004
8473          * The bit 5 of 0x42020
8474          * The bit 15 of 0x45000
8475          */
8476         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8477                    (I915_READ(ILK_DISPLAY_CHICKEN2) |
8478                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
8479         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
8480         I915_WRITE(DISP_ARB_CTL,
8481                    (I915_READ(DISP_ARB_CTL) |
8482                     DISP_FBC_WM_DIS));
8483
8484         /*
8485          * Based on the document from hardware guys the following bits
8486          * should be set unconditionally in order to enable FBC.
8487          * The bit 22 of 0x42000
8488          * The bit 22 of 0x42004
8489          * The bit 7,8,9 of 0x42020.
8490          */
8491         if (IS_IRONLAKE_M(dev_priv)) {
8492                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
8493                 I915_WRITE(ILK_DISPLAY_CHICKEN1,
8494                            I915_READ(ILK_DISPLAY_CHICKEN1) |
8495                            ILK_FBCQ_DIS);
8496                 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8497                            I915_READ(ILK_DISPLAY_CHICKEN2) |
8498                            ILK_DPARB_GATE);
8499         }
8500
8501         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
8502
8503         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8504                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8505                    ILK_ELPIN_409_SELECT);
8506         I915_WRITE(_3D_CHICKEN2,
8507                    _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
8508                    _3D_CHICKEN2_WM_READ_PIPELINED);
8509
8510         /* WaDisableRenderCachePipelinedFlush:ilk */
8511         I915_WRITE(CACHE_MODE_0,
8512                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
8513
8514         /* WaDisable_RenderCache_OperationalFlush:ilk */
8515         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8516
8517         g4x_disable_trickle_feed(dev_priv);
8518
8519         ibx_init_clock_gating(dev_priv);
8520 }
8521
8522 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
8523 {
8524         int pipe;
8525         uint32_t val;
8526
8527         /*
8528          * On Ibex Peak and Cougar Point, we need to disable clock
8529          * gating for the panel power sequencer or it will fail to
8530          * start up when no ports are active.
8531          */
8532         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
8533                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
8534                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
8535         I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
8536                    DPLS_EDP_PPS_FIX_DIS);
8537         /* The below fixes the weird display corruption, a few pixels shifted
8538          * downward, on (only) LVDS of some HP laptops with IVY.
8539          */
8540         for_each_pipe(dev_priv, pipe) {
8541                 val = I915_READ(TRANS_CHICKEN2(pipe));
8542                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
8543                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
8544                 if (dev_priv->vbt.fdi_rx_polarity_inverted)
8545                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
8546                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
8547                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
8548                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
8549                 I915_WRITE(TRANS_CHICKEN2(pipe), val);
8550         }
8551         /* WADP0ClockGatingDisable */
8552         for_each_pipe(dev_priv, pipe) {
8553                 I915_WRITE(TRANS_CHICKEN1(pipe),
8554                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
8555         }
8556 }
8557
8558 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
8559 {
8560         uint32_t tmp;
8561
8562         tmp = I915_READ(MCH_SSKPD);
8563         if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
8564                 DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
8565                               tmp);
8566 }
8567
8568 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
8569 {
8570         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
8571
8572         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
8573
8574         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8575                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8576                    ILK_ELPIN_409_SELECT);
8577
8578         /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
8579         I915_WRITE(_3D_CHICKEN,
8580                    _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
8581
8582         /* WaDisable_RenderCache_OperationalFlush:snb */
8583         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8584
8585         /*
8586          * BSpec recoomends 8x4 when MSAA is used,
8587          * however in practice 16x4 seems fastest.
8588          *
8589          * Note that PS/WM thread counts depend on the WIZ hashing
8590          * disable bit, which we don't touch here, but it's good
8591          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
8592          */
8593         I915_WRITE(GEN6_GT_MODE,
8594                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
8595
8596         I915_WRITE(CACHE_MODE_0,
8597                    _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
8598
8599         I915_WRITE(GEN6_UCGCTL1,
8600                    I915_READ(GEN6_UCGCTL1) |
8601                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
8602                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
8603
8604         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
8605          * gating disable must be set.  Failure to set it results in
8606          * flickering pixels due to Z write ordering failures after
8607          * some amount of runtime in the Mesa "fire" demo, and Unigine
8608          * Sanctuary and Tropics, and apparently anything else with
8609          * alpha test or pixel discard.
8610          *
8611          * According to the spec, bit 11 (RCCUNIT) must also be set,
8612          * but we didn't debug actual testcases to find it out.
8613          *
8614          * WaDisableRCCUnitClockGating:snb
8615          * WaDisableRCPBUnitClockGating:snb
8616          */
8617         I915_WRITE(GEN6_UCGCTL2,
8618                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
8619                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
8620
8621         /* WaStripsFansDisableFastClipPerformanceFix:snb */
8622         I915_WRITE(_3D_CHICKEN3,
8623                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
8624
8625         /*
8626          * Bspec says:
8627          * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
8628          * 3DSTATE_SF number of SF output attributes is more than 16."
8629          */
8630         I915_WRITE(_3D_CHICKEN3,
8631                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
8632
8633         /*
8634          * According to the spec the following bits should be
8635          * set in order to enable memory self-refresh and fbc:
8636          * The bit21 and bit22 of 0x42000
8637          * The bit21 and bit22 of 0x42004
8638          * The bit5 and bit7 of 0x42020
8639          * The bit14 of 0x70180
8640          * The bit14 of 0x71180
8641          *
8642          * WaFbcAsynchFlipDisableFbcQueue:snb
8643          */
8644         I915_WRITE(ILK_DISPLAY_CHICKEN1,
8645                    I915_READ(ILK_DISPLAY_CHICKEN1) |
8646                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
8647         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8648                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8649                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
8650         I915_WRITE(ILK_DSPCLK_GATE_D,
8651                    I915_READ(ILK_DSPCLK_GATE_D) |
8652                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
8653                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
8654
8655         g4x_disable_trickle_feed(dev_priv);
8656
8657         cpt_init_clock_gating(dev_priv);
8658
8659         gen6_check_mch_setup(dev_priv);
8660 }
8661
8662 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
8663 {
8664         uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
8665
8666         /*
8667          * WaVSThreadDispatchOverride:ivb,vlv
8668          *
8669          * This actually overrides the dispatch
8670          * mode for all thread types.
8671          */
8672         reg &= ~GEN7_FF_SCHED_MASK;
8673         reg |= GEN7_FF_TS_SCHED_HW;
8674         reg |= GEN7_FF_VS_SCHED_HW;
8675         reg |= GEN7_FF_DS_SCHED_HW;
8676
8677         I915_WRITE(GEN7_FF_THREAD_MODE, reg);
8678 }
8679
8680 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
8681 {
8682         /*
8683          * TODO: this bit should only be enabled when really needed, then
8684          * disabled when not needed anymore in order to save power.
8685          */
8686         if (HAS_PCH_LPT_LP(dev_priv))
8687                 I915_WRITE(SOUTH_DSPCLK_GATE_D,
8688                            I915_READ(SOUTH_DSPCLK_GATE_D) |
8689                            PCH_LP_PARTITION_LEVEL_DISABLE);
8690
8691         /* WADPOClockGatingDisable:hsw */
8692         I915_WRITE(TRANS_CHICKEN1(PIPE_A),
8693                    I915_READ(TRANS_CHICKEN1(PIPE_A)) |
8694                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
8695 }
8696
8697 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
8698 {
8699         if (HAS_PCH_LPT_LP(dev_priv)) {
8700                 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
8701
8702                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
8703                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
8704         }
8705 }
8706
8707 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
8708                                    int general_prio_credits,
8709                                    int high_prio_credits)
8710 {
8711         u32 misccpctl;
8712         u32 val;
8713
8714         /* WaTempDisableDOPClkGating:bdw */
8715         misccpctl = I915_READ(GEN7_MISCCPCTL);
8716         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
8717
8718         val = I915_READ(GEN8_L3SQCREG1);
8719         val &= ~L3_PRIO_CREDITS_MASK;
8720         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
8721         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
8722         I915_WRITE(GEN8_L3SQCREG1, val);
8723
8724         /*
8725          * Wait at least 100 clocks before re-enabling clock gating.
8726          * See the definition of L3SQCREG1 in BSpec.
8727          */
8728         POSTING_READ(GEN8_L3SQCREG1);
8729         udelay(1);
8730         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
8731 }
8732
8733 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
8734 {
8735         /* This is not an Wa. Enable to reduce Sampler power */
8736         I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
8737                    I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
8738 }
8739
8740 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
8741 {
8742         if (!HAS_PCH_CNP(dev_priv))
8743                 return;
8744
8745         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
8746         I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
8747                    CNP_PWM_CGE_GATING_DISABLE);
8748 }
8749
8750 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
8751 {
8752         u32 val;
8753         cnp_init_clock_gating(dev_priv);
8754
8755         /* This is not an Wa. Enable for better image quality */
8756         I915_WRITE(_3D_CHICKEN3,
8757                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
8758
8759         /* WaEnableChickenDCPR:cnl */
8760         I915_WRITE(GEN8_CHICKEN_DCPR_1,
8761                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
8762
8763         /* WaFbcWakeMemOn:cnl */
8764         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
8765                    DISP_FBC_MEMORY_WAKE);
8766
8767         val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
8768         /* ReadHitWriteOnlyDisable:cnl */
8769         val |= RCCUNIT_CLKGATE_DIS;
8770         /* WaSarbUnitClockGatingDisable:cnl (pre-prod) */
8771         if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0))
8772                 val |= SARBUNIT_CLKGATE_DIS;
8773         I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
8774
8775         /* Wa_2201832410:cnl */
8776         val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
8777         val |= GWUNIT_CLKGATE_DIS;
8778         I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);
8779
8780         /* WaDisableVFclkgate:cnl */
8781         /* WaVFUnitClockGatingDisable:cnl */
8782         val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
8783         val |= VFUNIT_CLKGATE_DIS;
8784         I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
8785 }
8786
8787 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
8788 {
8789         cnp_init_clock_gating(dev_priv);
8790         gen9_init_clock_gating(dev_priv);
8791
8792         /* WaFbcNukeOnHostModify:cfl */
8793         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
8794                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
8795 }
8796
8797 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
8798 {
8799         gen9_init_clock_gating(dev_priv);
8800
8801         /* WaDisableSDEUnitClockGating:kbl */
8802         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
8803                 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
8804                            GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
8805
8806         /* WaDisableGamClockGating:kbl */
8807         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
8808                 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
8809                            GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
8810
8811         /* WaFbcNukeOnHostModify:kbl */
8812         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
8813                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
8814 }
8815
8816 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
8817 {
8818         gen9_init_clock_gating(dev_priv);
8819
8820         /* WAC6entrylatency:skl */
8821         I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
8822                    FBC_LLC_FULLY_OPEN);
8823
8824         /* WaFbcNukeOnHostModify:skl */
8825         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
8826                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
8827 }
8828
8829 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
8830 {
8831         /* The GTT cache must be disabled if the system is using 2M pages. */
8832         bool can_use_gtt_cache = !HAS_PAGE_SIZES(dev_priv,
8833                                                  I915_GTT_PAGE_SIZE_2M);
8834         enum pipe pipe;
8835
8836         /* WaSwitchSolVfFArbitrationPriority:bdw */
8837         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
8838
8839         /* WaPsrDPAMaskVBlankInSRD:bdw */
8840         I915_WRITE(CHICKEN_PAR1_1,
8841                    I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
8842
8843         /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
8844         for_each_pipe(dev_priv, pipe) {
8845                 I915_WRITE(CHICKEN_PIPESL_1(pipe),
8846                            I915_READ(CHICKEN_PIPESL_1(pipe)) |
8847                            BDW_DPRS_MASK_VBLANK_SRD);
8848         }
8849
8850         /* WaVSRefCountFullforceMissDisable:bdw */
8851         /* WaDSRefCountFullforceMissDisable:bdw */
8852         I915_WRITE(GEN7_FF_THREAD_MODE,
8853                    I915_READ(GEN7_FF_THREAD_MODE) &
8854                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
8855
8856         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
8857                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
8858
8859         /* WaDisableSDEUnitClockGating:bdw */
8860         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
8861                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
8862
8863         /* WaProgramL3SqcReg1Default:bdw */
8864         gen8_set_l3sqc_credits(dev_priv, 30, 2);
8865
8866         /* WaGttCachingOffByDefault:bdw */
8867         I915_WRITE(HSW_GTT_CACHE_EN, can_use_gtt_cache ? GTT_CACHE_EN_ALL : 0);
8868
8869         /* WaKVMNotificationOnConfigChange:bdw */
8870         I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
8871                    | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
8872
8873         lpt_init_clock_gating(dev_priv);
8874
8875         /* WaDisableDopClockGating:bdw
8876          *
8877          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
8878          * clock gating.
8879          */
8880         I915_WRITE(GEN6_UCGCTL1,
8881                    I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
8882 }
8883
8884 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
8885 {
8886         /* L3 caching of data atomics doesn't work -- disable it. */
8887         I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
8888         I915_WRITE(HSW_ROW_CHICKEN3,
8889                    _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
8890
8891         /* This is required by WaCatErrorRejectionIssue:hsw */
8892         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
8893                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
8894                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
8895
8896         /* WaVSRefCountFullforceMissDisable:hsw */
8897         I915_WRITE(GEN7_FF_THREAD_MODE,
8898                    I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
8899
8900         /* WaDisable_RenderCache_OperationalFlush:hsw */
8901         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8902
8903         /* enable HiZ Raw Stall Optimization */
8904         I915_WRITE(CACHE_MODE_0_GEN7,
8905                    _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
8906
8907         /* WaDisable4x2SubspanOptimization:hsw */
8908         I915_WRITE(CACHE_MODE_1,
8909                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
8910
8911         /*
8912          * BSpec recommends 8x4 when MSAA is used,
8913          * however in practice 16x4 seems fastest.
8914          *
8915          * Note that PS/WM thread counts depend on the WIZ hashing
8916          * disable bit, which we don't touch here, but it's good
8917          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
8918          */
8919         I915_WRITE(GEN7_GT_MODE,
8920                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
8921
8922         /* WaSampleCChickenBitEnable:hsw */
8923         I915_WRITE(HALF_SLICE_CHICKEN3,
8924                    _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
8925
8926         /* WaSwitchSolVfFArbitrationPriority:hsw */
8927         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
8928
8929         lpt_init_clock_gating(dev_priv);
8930 }
8931
8932 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
8933 {
8934         uint32_t snpcr;
8935
8936         I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
8937
8938         /* WaDisableEarlyCull:ivb */
8939         I915_WRITE(_3D_CHICKEN3,
8940                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
8941
8942         /* WaDisableBackToBackFlipFix:ivb */
8943         I915_WRITE(IVB_CHICKEN3,
8944                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
8945                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
8946
8947         /* WaDisablePSDDualDispatchEnable:ivb */
8948         if (IS_IVB_GT1(dev_priv))
8949                 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
8950                            _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
8951
8952         /* WaDisable_RenderCache_OperationalFlush:ivb */
8953         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8954
8955         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
8956         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
8957                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
8958
8959         /* WaApplyL3ControlAndL3ChickenMode:ivb */
8960         I915_WRITE(GEN7_L3CNTLREG1,
8961                         GEN7_WA_FOR_GEN7_L3_CONTROL);
8962         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
8963                    GEN7_WA_L3_CHICKEN_MODE);
8964         if (IS_IVB_GT1(dev_priv))
8965                 I915_WRITE(GEN7_ROW_CHICKEN2,
8966                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
8967         else {
8968                 /* must write both registers */
8969                 I915_WRITE(GEN7_ROW_CHICKEN2,
8970                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
8971                 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
8972                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
8973         }
8974
8975         /* WaForceL3Serialization:ivb */
8976         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
8977                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
8978
8979         /*
8980          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
8981          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
8982          */
8983         I915_WRITE(GEN6_UCGCTL2,
8984                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
8985
8986         /* This is required by WaCatErrorRejectionIssue:ivb */
8987         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
8988                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
8989                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
8990
8991         g4x_disable_trickle_feed(dev_priv);
8992
8993         gen7_setup_fixed_func_scheduler(dev_priv);
8994
8995         if (0) { /* causes HiZ corruption on ivb:gt1 */
8996                 /* enable HiZ Raw Stall Optimization */
8997                 I915_WRITE(CACHE_MODE_0_GEN7,
8998                            _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
8999         }
9000
9001         /* WaDisable4x2SubspanOptimization:ivb */
9002         I915_WRITE(CACHE_MODE_1,
9003                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
9004
9005         /*
9006          * BSpec recommends 8x4 when MSAA is used,
9007          * however in practice 16x4 seems fastest.
9008          *
9009          * Note that PS/WM thread counts depend on the WIZ hashing
9010          * disable bit, which we don't touch here, but it's good
9011          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
9012          */
9013         I915_WRITE(GEN7_GT_MODE,
9014                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
9015
9016         snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
9017         snpcr &= ~GEN6_MBC_SNPCR_MASK;
9018         snpcr |= GEN6_MBC_SNPCR_MED;
9019         I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
9020
9021         if (!HAS_PCH_NOP(dev_priv))
9022                 cpt_init_clock_gating(dev_priv);
9023
9024         gen6_check_mch_setup(dev_priv);
9025 }
9026
9027 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
9028 {
9029         /* WaDisableEarlyCull:vlv */
9030         I915_WRITE(_3D_CHICKEN3,
9031                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
9032
9033         /* WaDisableBackToBackFlipFix:vlv */
9034         I915_WRITE(IVB_CHICKEN3,
9035                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
9036                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
9037
9038         /* WaPsdDispatchEnable:vlv */
9039         /* WaDisablePSDDualDispatchEnable:vlv */
9040         I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
9041                    _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
9042                                       GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
9043
9044         /* WaDisable_RenderCache_OperationalFlush:vlv */
9045         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9046
9047         /* WaForceL3Serialization:vlv */
9048         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
9049                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
9050
9051         /* WaDisableDopClockGating:vlv */
9052         I915_WRITE(GEN7_ROW_CHICKEN2,
9053                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
9054
9055         /* This is required by WaCatErrorRejectionIssue:vlv */
9056         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
9057                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
9058                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
9059
9060         gen7_setup_fixed_func_scheduler(dev_priv);
9061
9062         /*
9063          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
9064          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
9065          */
9066         I915_WRITE(GEN6_UCGCTL2,
9067                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
9068
9069         /* WaDisableL3Bank2xClockGate:vlv
9070          * Disabling L3 clock gating- MMIO 940c[25] = 1
9071          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
9072         I915_WRITE(GEN7_UCGCTL4,
9073                    I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
9074
9075         /*
9076          * BSpec says this must be set, even though
9077          * WaDisable4x2SubspanOptimization isn't listed for VLV.
9078          */
9079         I915_WRITE(CACHE_MODE_1,
9080                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
9081
9082         /*
9083          * BSpec recommends 8x4 when MSAA is used,
9084          * however in practice 16x4 seems fastest.
9085          *
9086          * Note that PS/WM thread counts depend on the WIZ hashing
9087          * disable bit, which we don't touch here, but it's good
9088          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
9089          */
9090         I915_WRITE(GEN7_GT_MODE,
9091                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
9092
9093         /*
9094          * WaIncreaseL3CreditsForVLVB0:vlv
9095          * This is the hardware default actually.
9096          */
9097         I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
9098
9099         /*
9100          * WaDisableVLVClockGating_VBIIssue:vlv
9101          * Disable clock gating on th GCFG unit to prevent a delay
9102          * in the reporting of vblank events.
9103          */
9104         I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
9105 }
9106
9107 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
9108 {
9109         /* WaVSRefCountFullforceMissDisable:chv */
9110         /* WaDSRefCountFullforceMissDisable:chv */
9111         I915_WRITE(GEN7_FF_THREAD_MODE,
9112                    I915_READ(GEN7_FF_THREAD_MODE) &
9113                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
9114
9115         /* WaDisableSemaphoreAndSyncFlipWait:chv */
9116         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
9117                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
9118
9119         /* WaDisableCSUnitClockGating:chv */
9120         I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
9121                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
9122
9123         /* WaDisableSDEUnitClockGating:chv */
9124         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
9125                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
9126
9127         /*
9128          * WaProgramL3SqcReg1Default:chv
9129          * See gfxspecs/Related Documents/Performance Guide/
9130          * LSQC Setting Recommendations.
9131          */
9132         gen8_set_l3sqc_credits(dev_priv, 38, 2);
9133
9134         /*
9135          * GTT cache may not work with big pages, so if those
9136          * are ever enabled GTT cache may need to be disabled.
9137          */
9138         I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
9139 }
9140
9141 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
9142 {
9143         uint32_t dspclk_gate;
9144
9145         I915_WRITE(RENCLK_GATE_D1, 0);
9146         I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
9147                    GS_UNIT_CLOCK_GATE_DISABLE |
9148                    CL_UNIT_CLOCK_GATE_DISABLE);
9149         I915_WRITE(RAMCLK_GATE_D, 0);
9150         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
9151                 OVRUNIT_CLOCK_GATE_DISABLE |
9152                 OVCUNIT_CLOCK_GATE_DISABLE;
9153         if (IS_GM45(dev_priv))
9154                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
9155         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
9156
9157         /* WaDisableRenderCachePipelinedFlush */
9158         I915_WRITE(CACHE_MODE_0,
9159                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
9160
9161         /* WaDisable_RenderCache_OperationalFlush:g4x */
9162         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9163
9164         g4x_disable_trickle_feed(dev_priv);
9165 }
9166
9167 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
9168 {
9169         I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
9170         I915_WRITE(RENCLK_GATE_D2, 0);
9171         I915_WRITE(DSPCLK_GATE_D, 0);
9172         I915_WRITE(RAMCLK_GATE_D, 0);
9173         I915_WRITE16(DEUC, 0);
9174         I915_WRITE(MI_ARB_STATE,
9175                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9176
9177         /* WaDisable_RenderCache_OperationalFlush:gen4 */
9178         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9179 }
9180
9181 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
9182 {
9183         I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
9184                    I965_RCC_CLOCK_GATE_DISABLE |
9185                    I965_RCPB_CLOCK_GATE_DISABLE |
9186                    I965_ISC_CLOCK_GATE_DISABLE |
9187                    I965_FBC_CLOCK_GATE_DISABLE);
9188         I915_WRITE(RENCLK_GATE_D2, 0);
9189         I915_WRITE(MI_ARB_STATE,
9190                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9191
9192         /* WaDisable_RenderCache_OperationalFlush:gen4 */
9193         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9194 }
9195
9196 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
9197 {
9198         u32 dstate = I915_READ(D_STATE);
9199
9200         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
9201                 DSTATE_DOT_CLOCK_GATING;
9202         I915_WRITE(D_STATE, dstate);
9203
9204         if (IS_PINEVIEW(dev_priv))
9205                 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
9206
9207         /* IIR "flip pending" means done if this bit is set */
9208         I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
9209
9210         /* interrupts should cause a wake up from C3 */
9211         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
9212
9213         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
9214         I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
9215
9216         I915_WRITE(MI_ARB_STATE,
9217                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9218 }
9219
9220 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
9221 {
9222         I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
9223
9224         /* interrupts should cause a wake up from C3 */
9225         I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
9226                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
9227
9228         I915_WRITE(MEM_MODE,
9229                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
9230 }
9231
9232 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
9233 {
9234         I915_WRITE(MEM_MODE,
9235                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
9236                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
9237 }
9238
9239 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
9240 {
9241         dev_priv->display.init_clock_gating(dev_priv);
9242 }
9243
9244 void intel_suspend_hw(struct drm_i915_private *dev_priv)
9245 {
9246         if (HAS_PCH_LPT(dev_priv))
9247                 lpt_suspend_hw(dev_priv);
9248 }
9249
9250 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
9251 {
9252         DRM_DEBUG_KMS("No clock gating settings or workarounds applied.\n");
9253 }
9254
9255 /**
9256  * intel_init_clock_gating_hooks - setup the clock gating hooks
9257  * @dev_priv: device private
9258  *
9259  * Setup the hooks that configure which clocks of a given platform can be
9260  * gated and also apply various GT and display specific workarounds for these
9261  * platforms. Note that some GT specific workarounds are applied separately
9262  * when GPU contexts or batchbuffers start their execution.
9263  */
9264 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
9265 {
9266         if (IS_ICELAKE(dev_priv))
9267                 dev_priv->display.init_clock_gating = icl_init_clock_gating;
9268         else if (IS_CANNONLAKE(dev_priv))
9269                 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
9270         else if (IS_COFFEELAKE(dev_priv))
9271                 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
9272         else if (IS_SKYLAKE(dev_priv))
9273                 dev_priv->display.init_clock_gating = skl_init_clock_gating;
9274         else if (IS_KABYLAKE(dev_priv))
9275                 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
9276         else if (IS_BROXTON(dev_priv))
9277                 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
9278         else if (IS_GEMINILAKE(dev_priv))
9279                 dev_priv->display.init_clock_gating = glk_init_clock_gating;
9280         else if (IS_BROADWELL(dev_priv))
9281                 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
9282         else if (IS_CHERRYVIEW(dev_priv))
9283                 dev_priv->display.init_clock_gating = chv_init_clock_gating;
9284         else if (IS_HASWELL(dev_priv))
9285                 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
9286         else if (IS_IVYBRIDGE(dev_priv))
9287                 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
9288         else if (IS_VALLEYVIEW(dev_priv))
9289                 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
9290         else if (IS_GEN6(dev_priv))
9291                 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
9292         else if (IS_GEN5(dev_priv))
9293                 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
9294         else if (IS_G4X(dev_priv))
9295                 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
9296         else if (IS_I965GM(dev_priv))
9297                 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
9298         else if (IS_I965G(dev_priv))
9299                 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
9300         else if (IS_GEN3(dev_priv))
9301                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
9302         else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
9303                 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
9304         else if (IS_GEN2(dev_priv))
9305                 dev_priv->display.init_clock_gating = i830_init_clock_gating;
9306         else {
9307                 MISSING_CASE(INTEL_DEVID(dev_priv));
9308                 dev_priv->display.init_clock_gating = nop_init_clock_gating;
9309         }
9310 }
9311
9312 /* Set up chip specific power management-related functions */
9313 void intel_init_pm(struct drm_i915_private *dev_priv)
9314 {
9315         intel_fbc_init(dev_priv);
9316
9317         /* For cxsr */
9318         if (IS_PINEVIEW(dev_priv))
9319                 i915_pineview_get_mem_freq(dev_priv);
9320         else if (IS_GEN5(dev_priv))
9321                 i915_ironlake_get_mem_freq(dev_priv);
9322
9323         /* For FIFO watermark updates */
9324         if (INTEL_GEN(dev_priv) >= 9) {
9325                 skl_setup_wm_latency(dev_priv);
9326                 dev_priv->display.initial_watermarks = skl_initial_wm;
9327                 dev_priv->display.atomic_update_watermarks = skl_atomic_update_crtc_wm;
9328                 dev_priv->display.compute_global_watermarks = skl_compute_wm;
9329         } else if (HAS_PCH_SPLIT(dev_priv)) {
9330                 ilk_setup_wm_latency(dev_priv);
9331
9332                 if ((IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[1] &&
9333                      dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
9334                     (!IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[0] &&
9335                      dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
9336                         dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
9337                         dev_priv->display.compute_intermediate_wm =
9338                                 ilk_compute_intermediate_wm;
9339                         dev_priv->display.initial_watermarks =
9340                                 ilk_initial_watermarks;
9341                         dev_priv->display.optimize_watermarks =
9342                                 ilk_optimize_watermarks;
9343                 } else {
9344                         DRM_DEBUG_KMS("Failed to read display plane latency. "
9345                                       "Disable CxSR\n");
9346                 }
9347         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
9348                 vlv_setup_wm_latency(dev_priv);
9349                 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
9350                 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
9351                 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
9352                 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
9353                 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
9354         } else if (IS_G4X(dev_priv)) {
9355                 g4x_setup_wm_latency(dev_priv);
9356                 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
9357                 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
9358                 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
9359                 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
9360         } else if (IS_PINEVIEW(dev_priv)) {
9361                 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
9362                                             dev_priv->is_ddr3,
9363                                             dev_priv->fsb_freq,
9364                                             dev_priv->mem_freq)) {
9365                         DRM_INFO("failed to find known CxSR latency "
9366                                  "(found ddr%s fsb freq %d, mem freq %d), "
9367                                  "disabling CxSR\n",
9368                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
9369                                  dev_priv->fsb_freq, dev_priv->mem_freq);
9370                         /* Disable CxSR and never update its watermark again */
9371                         intel_set_memory_cxsr(dev_priv, false);
9372                         dev_priv->display.update_wm = NULL;
9373                 } else
9374                         dev_priv->display.update_wm = pineview_update_wm;
9375         } else if (IS_GEN4(dev_priv)) {
9376                 dev_priv->display.update_wm = i965_update_wm;
9377         } else if (IS_GEN3(dev_priv)) {
9378                 dev_priv->display.update_wm = i9xx_update_wm;
9379                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
9380         } else if (IS_GEN2(dev_priv)) {
9381                 if (INTEL_INFO(dev_priv)->num_pipes == 1) {
9382                         dev_priv->display.update_wm = i845_update_wm;
9383                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
9384                 } else {
9385                         dev_priv->display.update_wm = i9xx_update_wm;
9386                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
9387                 }
9388         } else {
9389                 DRM_ERROR("unexpected fall-through in intel_init_pm\n");
9390         }
9391 }
9392
9393 static inline int gen6_check_mailbox_status(struct drm_i915_private *dev_priv)
9394 {
9395         uint32_t flags =
9396                 I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK;
9397
9398         switch (flags) {
9399         case GEN6_PCODE_SUCCESS:
9400                 return 0;
9401         case GEN6_PCODE_UNIMPLEMENTED_CMD:
9402                 return -ENODEV;
9403         case GEN6_PCODE_ILLEGAL_CMD:
9404                 return -ENXIO;
9405         case GEN6_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
9406         case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
9407                 return -EOVERFLOW;
9408         case GEN6_PCODE_TIMEOUT:
9409                 return -ETIMEDOUT;
9410         default:
9411                 MISSING_CASE(flags);
9412                 return 0;
9413         }
9414 }
9415
9416 static inline int gen7_check_mailbox_status(struct drm_i915_private *dev_priv)
9417 {
9418         uint32_t flags =
9419                 I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK;
9420
9421         switch (flags) {
9422         case GEN6_PCODE_SUCCESS:
9423                 return 0;
9424         case GEN6_PCODE_ILLEGAL_CMD:
9425                 return -ENXIO;
9426         case GEN7_PCODE_TIMEOUT:
9427                 return -ETIMEDOUT;
9428         case GEN7_PCODE_ILLEGAL_DATA:
9429                 return -EINVAL;
9430         case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
9431                 return -EOVERFLOW;
9432         default:
9433                 MISSING_CASE(flags);
9434                 return 0;
9435         }
9436 }
9437
9438 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val)
9439 {
9440         int status;
9441
9442         WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
9443
9444         /* GEN6_PCODE_* are outside of the forcewake domain, we can
9445          * use te fw I915_READ variants to reduce the amount of work
9446          * required when reading/writing.
9447          */
9448
9449         if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
9450                 DRM_DEBUG_DRIVER("warning: pcode (read from mbox %x) mailbox access failed for %ps\n",
9451                                  mbox, __builtin_return_address(0));
9452                 return -EAGAIN;
9453         }
9454
9455         I915_WRITE_FW(GEN6_PCODE_DATA, *val);
9456         I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
9457         I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
9458
9459         if (__intel_wait_for_register_fw(dev_priv,
9460                                          GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
9461                                          500, 0, NULL)) {
9462                 DRM_ERROR("timeout waiting for pcode read (from mbox %x) to finish for %ps\n",
9463                           mbox, __builtin_return_address(0));
9464                 return -ETIMEDOUT;
9465         }
9466
9467         *val = I915_READ_FW(GEN6_PCODE_DATA);
9468         I915_WRITE_FW(GEN6_PCODE_DATA, 0);
9469
9470         if (INTEL_GEN(dev_priv) > 6)
9471                 status = gen7_check_mailbox_status(dev_priv);
9472         else
9473                 status = gen6_check_mailbox_status(dev_priv);
9474
9475         if (status) {
9476                 DRM_DEBUG_DRIVER("warning: pcode (read from mbox %x) mailbox access failed for %ps: %d\n",
9477                                  mbox, __builtin_return_address(0), status);
9478                 return status;
9479         }
9480
9481         return 0;
9482 }
9483
9484 int sandybridge_pcode_write_timeout(struct drm_i915_private *dev_priv,
9485                                     u32 mbox, u32 val,
9486                                     int fast_timeout_us, int slow_timeout_ms)
9487 {
9488         int status;
9489
9490         WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
9491
9492         /* GEN6_PCODE_* are outside of the forcewake domain, we can
9493          * use te fw I915_READ variants to reduce the amount of work
9494          * required when reading/writing.
9495          */
9496
9497         if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
9498                 DRM_DEBUG_DRIVER("warning: pcode (write of 0x%08x to mbox %x) mailbox access failed for %ps\n",
9499                                  val, mbox, __builtin_return_address(0));
9500                 return -EAGAIN;
9501         }
9502
9503         I915_WRITE_FW(GEN6_PCODE_DATA, val);
9504         I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
9505         I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
9506
9507         if (__intel_wait_for_register_fw(dev_priv,
9508                                          GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
9509                                          fast_timeout_us, slow_timeout_ms,
9510                                          NULL)) {
9511                 DRM_ERROR("timeout waiting for pcode write of 0x%08x to mbox %x to finish for %ps\n",
9512                           val, mbox, __builtin_return_address(0));
9513                 return -ETIMEDOUT;
9514         }
9515
9516         I915_WRITE_FW(GEN6_PCODE_DATA, 0);
9517
9518         if (INTEL_GEN(dev_priv) > 6)
9519                 status = gen7_check_mailbox_status(dev_priv);
9520         else
9521                 status = gen6_check_mailbox_status(dev_priv);
9522
9523         if (status) {
9524                 DRM_DEBUG_DRIVER("warning: pcode (write of 0x%08x to mbox %x) mailbox access failed for %ps: %d\n",
9525                                  val, mbox, __builtin_return_address(0), status);
9526                 return status;
9527         }
9528
9529         return 0;
9530 }
9531
9532 static bool skl_pcode_try_request(struct drm_i915_private *dev_priv, u32 mbox,
9533                                   u32 request, u32 reply_mask, u32 reply,
9534                                   u32 *status)
9535 {
9536         u32 val = request;
9537
9538         *status = sandybridge_pcode_read(dev_priv, mbox, &val);
9539
9540         return *status || ((val & reply_mask) == reply);
9541 }
9542
9543 /**
9544  * skl_pcode_request - send PCODE request until acknowledgment
9545  * @dev_priv: device private
9546  * @mbox: PCODE mailbox ID the request is targeted for
9547  * @request: request ID
9548  * @reply_mask: mask used to check for request acknowledgment
9549  * @reply: value used to check for request acknowledgment
9550  * @timeout_base_ms: timeout for polling with preemption enabled
9551  *
9552  * Keep resending the @request to @mbox until PCODE acknowledges it, PCODE
9553  * reports an error or an overall timeout of @timeout_base_ms+50 ms expires.
9554  * The request is acknowledged once the PCODE reply dword equals @reply after
9555  * applying @reply_mask. Polling is first attempted with preemption enabled
9556  * for @timeout_base_ms and if this times out for another 50 ms with
9557  * preemption disabled.
9558  *
9559  * Returns 0 on success, %-ETIMEDOUT in case of a timeout, <0 in case of some
9560  * other error as reported by PCODE.
9561  */
9562 int skl_pcode_request(struct drm_i915_private *dev_priv, u32 mbox, u32 request,
9563                       u32 reply_mask, u32 reply, int timeout_base_ms)
9564 {
9565         u32 status;
9566         int ret;
9567
9568         WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
9569
9570 #define COND skl_pcode_try_request(dev_priv, mbox, request, reply_mask, reply, \
9571                                    &status)
9572
9573         /*
9574          * Prime the PCODE by doing a request first. Normally it guarantees
9575          * that a subsequent request, at most @timeout_base_ms later, succeeds.
9576          * _wait_for() doesn't guarantee when its passed condition is evaluated
9577          * first, so send the first request explicitly.
9578          */
9579         if (COND) {
9580                 ret = 0;
9581                 goto out;
9582         }
9583         ret = _wait_for(COND, timeout_base_ms * 1000, 10, 10);
9584         if (!ret)
9585                 goto out;
9586
9587         /*
9588          * The above can time out if the number of requests was low (2 in the
9589          * worst case) _and_ PCODE was busy for some reason even after a
9590          * (queued) request and @timeout_base_ms delay. As a workaround retry
9591          * the poll with preemption disabled to maximize the number of
9592          * requests. Increase the timeout from @timeout_base_ms to 50ms to
9593          * account for interrupts that could reduce the number of these
9594          * requests, and for any quirks of the PCODE firmware that delays
9595          * the request completion.
9596          */
9597         DRM_DEBUG_KMS("PCODE timeout, retrying with preemption disabled\n");
9598         WARN_ON_ONCE(timeout_base_ms > 3);
9599         preempt_disable();
9600         ret = wait_for_atomic(COND, 50);
9601         preempt_enable();
9602
9603 out:
9604         return ret ? ret : status;
9605 #undef COND
9606 }
9607
9608 static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
9609 {
9610         struct intel_rps *rps = &dev_priv->gt_pm.rps;
9611
9612         /*
9613          * N = val - 0xb7
9614          * Slow = Fast = GPLL ref * N
9615          */
9616         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
9617 }
9618
9619 static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
9620 {
9621         struct intel_rps *rps = &dev_priv->gt_pm.rps;
9622
9623         return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
9624 }
9625
9626 static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
9627 {
9628         struct intel_rps *rps = &dev_priv->gt_pm.rps;
9629
9630         /*
9631          * N = val / 2
9632          * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
9633          */
9634         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
9635 }
9636
9637 static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
9638 {
9639         struct intel_rps *rps = &dev_priv->gt_pm.rps;
9640
9641         /* CHV needs even values */
9642         return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
9643 }
9644
9645 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
9646 {
9647         if (INTEL_GEN(dev_priv) >= 9)
9648                 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
9649                                          GEN9_FREQ_SCALER);
9650         else if (IS_CHERRYVIEW(dev_priv))
9651                 return chv_gpu_freq(dev_priv, val);
9652         else if (IS_VALLEYVIEW(dev_priv))
9653                 return byt_gpu_freq(dev_priv, val);
9654         else
9655                 return val * GT_FREQUENCY_MULTIPLIER;
9656 }
9657
9658 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
9659 {
9660         if (INTEL_GEN(dev_priv) >= 9)
9661                 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
9662                                          GT_FREQUENCY_MULTIPLIER);
9663         else if (IS_CHERRYVIEW(dev_priv))
9664                 return chv_freq_opcode(dev_priv, val);
9665         else if (IS_VALLEYVIEW(dev_priv))
9666                 return byt_freq_opcode(dev_priv, val);
9667         else
9668                 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
9669 }
9670
9671 void intel_pm_setup(struct drm_i915_private *dev_priv)
9672 {
9673         mutex_init(&dev_priv->pcu_lock);
9674         mutex_init(&dev_priv->gt_pm.rps.power.mutex);
9675
9676         atomic_set(&dev_priv->gt_pm.rps.num_waiters, 0);
9677
9678         dev_priv->runtime_pm.suspended = false;
9679         atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
9680 }
9681
9682 static u64 vlv_residency_raw(struct drm_i915_private *dev_priv,
9683                              const i915_reg_t reg)
9684 {
9685         u32 lower, upper, tmp;
9686         int loop = 2;
9687
9688         /*
9689          * The register accessed do not need forcewake. We borrow
9690          * uncore lock to prevent concurrent access to range reg.
9691          */
9692         lockdep_assert_held(&dev_priv->uncore.lock);
9693
9694         /*
9695          * vlv and chv residency counters are 40 bits in width.
9696          * With a control bit, we can choose between upper or lower
9697          * 32bit window into this counter.
9698          *
9699          * Although we always use the counter in high-range mode elsewhere,
9700          * userspace may attempt to read the value before rc6 is initialised,
9701          * before we have set the default VLV_COUNTER_CONTROL value. So always
9702          * set the high bit to be safe.
9703          */
9704         I915_WRITE_FW(VLV_COUNTER_CONTROL,
9705                       _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
9706         upper = I915_READ_FW(reg);
9707         do {
9708                 tmp = upper;
9709
9710                 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9711                               _MASKED_BIT_DISABLE(VLV_COUNT_RANGE_HIGH));
9712                 lower = I915_READ_FW(reg);
9713
9714                 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9715                               _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
9716                 upper = I915_READ_FW(reg);
9717         } while (upper != tmp && --loop);
9718
9719         /*
9720          * Everywhere else we always use VLV_COUNTER_CONTROL with the
9721          * VLV_COUNT_RANGE_HIGH bit set - so it is safe to leave it set
9722          * now.
9723          */
9724
9725         return lower | (u64)upper << 8;
9726 }
9727
9728 u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv,
9729                            const i915_reg_t reg)
9730 {
9731         u64 time_hw, prev_hw, overflow_hw;
9732         unsigned int fw_domains;
9733         unsigned long flags;
9734         unsigned int i;
9735         u32 mul, div;
9736
9737         if (!HAS_RC6(dev_priv))
9738                 return 0;
9739
9740         /*
9741          * Store previous hw counter values for counter wrap-around handling.
9742          *
9743          * There are only four interesting registers and they live next to each
9744          * other so we can use the relative address, compared to the smallest
9745          * one as the index into driver storage.
9746          */
9747         i = (i915_mmio_reg_offset(reg) -
9748              i915_mmio_reg_offset(GEN6_GT_GFX_RC6_LOCKED)) / sizeof(u32);
9749         if (WARN_ON_ONCE(i >= ARRAY_SIZE(dev_priv->gt_pm.rc6.cur_residency)))
9750                 return 0;
9751
9752         fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ);
9753
9754         spin_lock_irqsave(&dev_priv->uncore.lock, flags);
9755         intel_uncore_forcewake_get__locked(dev_priv, fw_domains);
9756
9757         /* On VLV and CHV, residency time is in CZ units rather than 1.28us */
9758         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
9759                 mul = 1000000;
9760                 div = dev_priv->czclk_freq;
9761                 overflow_hw = BIT_ULL(40);
9762                 time_hw = vlv_residency_raw(dev_priv, reg);
9763         } else {
9764                 /* 833.33ns units on Gen9LP, 1.28us elsewhere. */
9765                 if (IS_GEN9_LP(dev_priv)) {
9766                         mul = 10000;
9767                         div = 12;
9768                 } else {
9769                         mul = 1280;
9770                         div = 1;
9771                 }
9772
9773                 overflow_hw = BIT_ULL(32);
9774                 time_hw = I915_READ_FW(reg);
9775         }
9776
9777         /*
9778          * Counter wrap handling.
9779          *
9780          * But relying on a sufficient frequency of queries otherwise counters
9781          * can still wrap.
9782          */
9783         prev_hw = dev_priv->gt_pm.rc6.prev_hw_residency[i];
9784         dev_priv->gt_pm.rc6.prev_hw_residency[i] = time_hw;
9785
9786         /* RC6 delta from last sample. */
9787         if (time_hw >= prev_hw)
9788                 time_hw -= prev_hw;
9789         else
9790                 time_hw += overflow_hw - prev_hw;
9791
9792         /* Add delta to RC6 extended raw driver copy. */
9793         time_hw += dev_priv->gt_pm.rc6.cur_residency[i];
9794         dev_priv->gt_pm.rc6.cur_residency[i] = time_hw;
9795
9796         intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
9797         spin_unlock_irqrestore(&dev_priv->uncore.lock, flags);
9798
9799         return mul_u64_u32_div(time_hw, mul, div);
9800 }
9801
9802 u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat)
9803 {
9804         u32 cagf;
9805
9806         if (INTEL_GEN(dev_priv) >= 9)
9807                 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
9808         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
9809                 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
9810         else
9811                 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
9812
9813         return  cagf;
9814 }