drivers/gpu/drm/i915/display/intel_cdclk.c

   1 /*
   2  * Copyright © 2006-2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include <linux/time.h>
  25
  26 #include "hsw_ips.h"
  27 #include "i915_reg.h"
  28 #include "intel_atomic.h"
  29 #include "intel_atomic_plane.h"
  30 #include "intel_audio.h"
  31 #include "intel_bw.h"
  32 #include "intel_cdclk.h"
  33 #include "intel_crtc.h"
  34 #include "intel_de.h"
  35 #include "intel_dp.h"
  36 #include "intel_display_types.h"
  37 #include "intel_mchbar_regs.h"
  38 #include "intel_pci_config.h"
  39 #include "intel_pcode.h"
  40 #include "intel_psr.h"
  41 #include "intel_vdsc.h"
  42 #include "skl_watermark.h"
  43 #include "skl_watermark_regs.h"
  44 #include "vlv_sideband.h"
  45
  46 /**
  47  * DOC: CDCLK / RAWCLK
  48  *
  49  * The display engine uses several different clocks to do its work. There
  50  * are two main clocks involved that aren't directly related to the actual
  51  * pixel clock or any symbol/bit clock of the actual output port. These
  52  * are the core display clock (CDCLK) and RAWCLK.
  53  *
  54  * CDCLK clocks most of the display pipe logic, and thus its frequency
  55  * must be high enough to support the rate at which pixels are flowing
  56  * through the pipes. Downscaling must also be accounted as that increases
  57  * the effective pixel rate.
  58  *
  59  * On several platforms the CDCLK frequency can be changed dynamically
  60  * to minimize power consumption for a given display configuration.
  61  * Typically changes to the CDCLK frequency require all the display pipes
  62  * to be shut down while the frequency is being changed.
  63  *
  64  * On SKL+ the DMC will toggle the CDCLK off/on during DC5/6 entry/exit.
  65  * DMC will not change the active CDCLK frequency however, so that part
  66  * will still be performed by the driver directly.
  67  *
  68  * There are multiple components involved in the generation of the CDCLK
  69  * frequency:
  70  *
  71  * - We have the CDCLK PLL, which generates an output clock based on a
  72  *   reference clock and a ratio parameter.
  73  * - The CD2X Divider, which divides the output of the PLL based on a
  74  *   divisor selected from a set of pre-defined choices.
  75  * - The CD2X Squasher, which further divides the output based on a
  76  *   waveform represented as a sequence of bits where each zero
  77  *   "squashes out" a clock cycle.
  78  * - And, finally, a fixed divider that divides the output frequency by 2.
  79  *
  80  * As such, the resulting CDCLK frequency can be calculated with the
  81  * following formula:
  82  *
  83  *     cdclk = vco / cd2x_div / (sq_len / sq_div) / 2
  84  *
  85  * , where vco is the frequency generated by the PLL; cd2x_div
  86  * represents the CD2X Divider; sq_len and sq_div are the bit length
  87  * and the number of high bits for the CD2X Squasher waveform, respectively;
  88  * and 2 represents the fixed divider.
  89  *
  90  * Note that some older platforms do not contain the CD2X Divider
  91  * and/or CD2X Squasher, in which case we can ignore their respective
  92  * factors in the formula above.
  93  *
  94  * Several methods exist to change the CDCLK frequency, which ones are
  95  * supported depends on the platform:
  96  *
  97  * - Full PLL disable + re-enable with new VCO frequency. Pipes must be inactive.
  98  * - CD2X divider update. Single pipe can be active as the divider update
  99  *   can be synchronized with the pipe's start of vblank.
 100  * - Crawl the PLL smoothly to the new VCO frequency. Pipes can be active.
 101  * - Squash waveform update. Pipes can be active.
 102  * - Crawl and squash can also be done back to back. Pipes can be active.
 103  *
 104  * RAWCLK is a fixed frequency clock, often used by various auxiliary
 105  * blocks such as AUX CH or backlight PWM. Hence the only thing we
 106  * really need to know about RAWCLK is its frequency so that various
 107  * dividers can be programmed correctly.
 108  */
 109
 110 struct intel_cdclk_funcs {
 111         void (*get_cdclk)(struct drm_i915_private *i915,
 112                           struct intel_cdclk_config *cdclk_config);
 113         void (*set_cdclk)(struct drm_i915_private *i915,
 114                           const struct intel_cdclk_config *cdclk_config,
 115                           enum pipe pipe);
 116         int (*modeset_calc_cdclk)(struct intel_cdclk_state *state);
 117         u8 (*calc_voltage_level)(int cdclk);
 118 };
 119
 120 void intel_cdclk_get_cdclk(struct drm_i915_private *dev_priv,
 121                            struct intel_cdclk_config *cdclk_config)
 122 {
 123         dev_priv->display.funcs.cdclk->get_cdclk(dev_priv, cdclk_config);
 124 }
 125
 126 static void intel_cdclk_set_cdclk(struct drm_i915_private *dev_priv,
 127                                   const struct intel_cdclk_config *cdclk_config,
 128                                   enum pipe pipe)
 129 {
 130         dev_priv->display.funcs.cdclk->set_cdclk(dev_priv, cdclk_config, pipe);
 131 }
 132
 133 static int intel_cdclk_modeset_calc_cdclk(struct drm_i915_private *dev_priv,
 134                                           struct intel_cdclk_state *cdclk_config)
 135 {
 136         return dev_priv->display.funcs.cdclk->modeset_calc_cdclk(cdclk_config);
 137 }
 138
 139 static u8 intel_cdclk_calc_voltage_level(struct drm_i915_private *dev_priv,
 140                                          int cdclk)
 141 {
 142         return dev_priv->display.funcs.cdclk->calc_voltage_level(cdclk);
 143 }
 144
 145 static void fixed_133mhz_get_cdclk(struct drm_i915_private *dev_priv,
 146                                    struct intel_cdclk_config *cdclk_config)
 147 {
 148         cdclk_config->cdclk = 133333;
 149 }
 150
 151 static void fixed_200mhz_get_cdclk(struct drm_i915_private *dev_priv,
 152                                    struct intel_cdclk_config *cdclk_config)
 153 {
 154         cdclk_config->cdclk = 200000;
 155 }
 156
 157 static void fixed_266mhz_get_cdclk(struct drm_i915_private *dev_priv,
 158                                    struct intel_cdclk_config *cdclk_config)
 159 {
 160         cdclk_config->cdclk = 266667;
 161 }
 162
 163 static void fixed_333mhz_get_cdclk(struct drm_i915_private *dev_priv,
 164                                    struct intel_cdclk_config *cdclk_config)
 165 {
 166         cdclk_config->cdclk = 333333;
 167 }
 168
 169 static void fixed_400mhz_get_cdclk(struct drm_i915_private *dev_priv,
 170                                    struct intel_cdclk_config *cdclk_config)
 171 {
 172         cdclk_config->cdclk = 400000;
 173 }
 174
 175 static void fixed_450mhz_get_cdclk(struct drm_i915_private *dev_priv,
 176                                    struct intel_cdclk_config *cdclk_config)
 177 {
 178         cdclk_config->cdclk = 450000;
 179 }
 180
 181 static void i85x_get_cdclk(struct drm_i915_private *dev_priv,
 182                            struct intel_cdclk_config *cdclk_config)
 183 {
 184         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 185         u16 hpllcc = 0;
 186
 187         /*
 188          * 852GM/852GMV only supports 133 MHz and the HPLLCC
 189          * encoding is different :(
 190          * FIXME is this the right way to detect 852GM/852GMV?
 191          */
 192         if (pdev->revision == 0x1) {
 193                 cdclk_config->cdclk = 133333;
 194                 return;
 195         }
 196
 197         pci_bus_read_config_word(pdev->bus,
 198                                  PCI_DEVFN(0, 3), HPLLCC, &hpllcc);
 199
 200         /* Assume that the hardware is in the high speed state.  This
 201          * should be the default.
 202          */
 203         switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
 204         case GC_CLOCK_133_200:
 205         case GC_CLOCK_133_200_2:
 206         case GC_CLOCK_100_200:
 207                 cdclk_config->cdclk = 200000;
 208                 break;
 209         case GC_CLOCK_166_250:
 210                 cdclk_config->cdclk = 250000;
 211                 break;
 212         case GC_CLOCK_100_133:
 213                 cdclk_config->cdclk = 133333;
 214                 break;
 215         case GC_CLOCK_133_266:
 216         case GC_CLOCK_133_266_2:
 217         case GC_CLOCK_166_266:
 218                 cdclk_config->cdclk = 266667;
 219                 break;
 220         }
 221 }
 222
 223 static void i915gm_get_cdclk(struct drm_i915_private *dev_priv,
 224                              struct intel_cdclk_config *cdclk_config)
 225 {
 226         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 227         u16 gcfgc = 0;
 228
 229         pci_read_config_word(pdev, GCFGC, &gcfgc);
 230
 231         if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 232                 cdclk_config->cdclk = 133333;
 233                 return;
 234         }
 235
 236         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 237         case GC_DISPLAY_CLOCK_333_320_MHZ:
 238                 cdclk_config->cdclk = 333333;
 239                 break;
 240         default:
 241         case GC_DISPLAY_CLOCK_190_200_MHZ:
 242                 cdclk_config->cdclk = 190000;
 243                 break;
 244         }
 245 }
 246
 247 static void i945gm_get_cdclk(struct drm_i915_private *dev_priv,
 248                              struct intel_cdclk_config *cdclk_config)
 249 {
 250         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 251         u16 gcfgc = 0;
 252
 253         pci_read_config_word(pdev, GCFGC, &gcfgc);
 254
 255         if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 256                 cdclk_config->cdclk = 133333;
 257                 return;
 258         }
 259
 260         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 261         case GC_DISPLAY_CLOCK_333_320_MHZ:
 262                 cdclk_config->cdclk = 320000;
 263                 break;
 264         default:
 265         case GC_DISPLAY_CLOCK_190_200_MHZ:
 266                 cdclk_config->cdclk = 200000;
 267                 break;
 268         }
 269 }
 270
 271 static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv)
 272 {
 273         static const unsigned int blb_vco[8] = {
 274                 [0] = 3200000,
 275                 [1] = 4000000,
 276                 [2] = 5333333,
 277                 [3] = 4800000,
 278                 [4] = 6400000,
 279         };
 280         static const unsigned int pnv_vco[8] = {
 281                 [0] = 3200000,
 282                 [1] = 4000000,
 283                 [2] = 5333333,
 284                 [3] = 4800000,
 285                 [4] = 2666667,
 286         };
 287         static const unsigned int cl_vco[8] = {
 288                 [0] = 3200000,
 289                 [1] = 4000000,
 290                 [2] = 5333333,
 291                 [3] = 6400000,
 292                 [4] = 3333333,
 293                 [5] = 3566667,
 294                 [6] = 4266667,
 295         };
 296         static const unsigned int elk_vco[8] = {
 297                 [0] = 3200000,
 298                 [1] = 4000000,
 299                 [2] = 5333333,
 300                 [3] = 4800000,
 301         };
 302         static const unsigned int ctg_vco[8] = {
 303                 [0] = 3200000,
 304                 [1] = 4000000,
 305                 [2] = 5333333,
 306                 [3] = 6400000,
 307                 [4] = 2666667,
 308                 [5] = 4266667,
 309         };
 310         const unsigned int *vco_table;
 311         unsigned int vco;
 312         u8 tmp = 0;
 313
 314         /* FIXME other chipsets? */
 315         if (IS_GM45(dev_priv))
 316                 vco_table = ctg_vco;
 317         else if (IS_G45(dev_priv))
 318                 vco_table = elk_vco;
 319         else if (IS_I965GM(dev_priv))
 320                 vco_table = cl_vco;
 321         else if (IS_PINEVIEW(dev_priv))
 322                 vco_table = pnv_vco;
 323         else if (IS_G33(dev_priv))
 324                 vco_table = blb_vco;
 325         else
 326                 return 0;
 327
 328         tmp = intel_de_read(dev_priv,
 329                             IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv) ? HPLLVCO_MOBILE : HPLLVCO);
 330
 331         vco = vco_table[tmp & 0x7];
 332         if (vco == 0)
 333                 drm_err(&dev_priv->drm, "Bad HPLL VCO (HPLLVCO=0x%02x)\n",
 334                         tmp);
 335         else
 336                 drm_dbg_kms(&dev_priv->drm, "HPLL VCO %u kHz\n", vco);
 337
 338         return vco;
 339 }
 340
 341 static void g33_get_cdclk(struct drm_i915_private *dev_priv,
 342                           struct intel_cdclk_config *cdclk_config)
 343 {
 344         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 345         static const u8 div_3200[] = { 12, 10,  8,  7, 5, 16 };
 346         static const u8 div_4000[] = { 14, 12, 10,  8, 6, 20 };
 347         static const u8 div_4800[] = { 20, 14, 12, 10, 8, 24 };
 348         static const u8 div_5333[] = { 20, 16, 12, 12, 8, 28 };
 349         const u8 *div_table;
 350         unsigned int cdclk_sel;
 351         u16 tmp = 0;
 352
 353         cdclk_config->vco = intel_hpll_vco(dev_priv);
 354
 355         pci_read_config_word(pdev, GCFGC, &tmp);
 356
 357         cdclk_sel = (tmp >> 4) & 0x7;
 358
 359         if (cdclk_sel >= ARRAY_SIZE(div_3200))
 360                 goto fail;
 361
 362         switch (cdclk_config->vco) {
 363         case 3200000:
 364                 div_table = div_3200;
 365                 break;
 366         case 4000000:
 367                 div_table = div_4000;
 368                 break;
 369         case 4800000:
 370                 div_table = div_4800;
 371                 break;
 372         case 5333333:
 373                 div_table = div_5333;
 374                 break;
 375         default:
 376                 goto fail;
 377         }
 378
 379         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 380                                                 div_table[cdclk_sel]);
 381         return;
 382
 383 fail:
 384         drm_err(&dev_priv->drm,
 385                 "Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n",
 386                 cdclk_config->vco, tmp);
 387         cdclk_config->cdclk = 190476;
 388 }
 389
 390 static void pnv_get_cdclk(struct drm_i915_private *dev_priv,
 391                           struct intel_cdclk_config *cdclk_config)
 392 {
 393         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 394         u16 gcfgc = 0;
 395
 396         pci_read_config_word(pdev, GCFGC, &gcfgc);
 397
 398         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 399         case GC_DISPLAY_CLOCK_267_MHZ_PNV:
 400                 cdclk_config->cdclk = 266667;
 401                 break;
 402         case GC_DISPLAY_CLOCK_333_MHZ_PNV:
 403                 cdclk_config->cdclk = 333333;
 404                 break;
 405         case GC_DISPLAY_CLOCK_444_MHZ_PNV:
 406                 cdclk_config->cdclk = 444444;
 407                 break;
 408         case GC_DISPLAY_CLOCK_200_MHZ_PNV:
 409                 cdclk_config->cdclk = 200000;
 410                 break;
 411         default:
 412                 drm_err(&dev_priv->drm,
 413                         "Unknown pnv display core clock 0x%04x\n", gcfgc);
 414                 fallthrough;
 415         case GC_DISPLAY_CLOCK_133_MHZ_PNV:
 416                 cdclk_config->cdclk = 133333;
 417                 break;
 418         case GC_DISPLAY_CLOCK_167_MHZ_PNV:
 419                 cdclk_config->cdclk = 166667;
 420                 break;
 421         }
 422 }
 423
 424 static void i965gm_get_cdclk(struct drm_i915_private *dev_priv,
 425                              struct intel_cdclk_config *cdclk_config)
 426 {
 427         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 428         static const u8 div_3200[] = { 16, 10,  8 };
 429         static const u8 div_4000[] = { 20, 12, 10 };
 430         static const u8 div_5333[] = { 24, 16, 14 };
 431         const u8 *div_table;
 432         unsigned int cdclk_sel;
 433         u16 tmp = 0;
 434
 435         cdclk_config->vco = intel_hpll_vco(dev_priv);
 436
 437         pci_read_config_word(pdev, GCFGC, &tmp);
 438
 439         cdclk_sel = ((tmp >> 8) & 0x1f) - 1;
 440
 441         if (cdclk_sel >= ARRAY_SIZE(div_3200))
 442                 goto fail;
 443
 444         switch (cdclk_config->vco) {
 445         case 3200000:
 446                 div_table = div_3200;
 447                 break;
 448         case 4000000:
 449                 div_table = div_4000;
 450                 break;
 451         case 5333333:
 452                 div_table = div_5333;
 453                 break;
 454         default:
 455                 goto fail;
 456         }
 457
 458         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 459                                                 div_table[cdclk_sel]);
 460         return;
 461
 462 fail:
 463         drm_err(&dev_priv->drm,
 464                 "Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n",
 465                 cdclk_config->vco, tmp);
 466         cdclk_config->cdclk = 200000;
 467 }
 468
 469 static void gm45_get_cdclk(struct drm_i915_private *dev_priv,
 470                            struct intel_cdclk_config *cdclk_config)
 471 {
 472         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 473         unsigned int cdclk_sel;
 474         u16 tmp = 0;
 475
 476         cdclk_config->vco = intel_hpll_vco(dev_priv);
 477
 478         pci_read_config_word(pdev, GCFGC, &tmp);
 479
 480         cdclk_sel = (tmp >> 12) & 0x1;
 481
 482         switch (cdclk_config->vco) {
 483         case 2666667:
 484         case 4000000:
 485         case 5333333:
 486                 cdclk_config->cdclk = cdclk_sel ? 333333 : 222222;
 487                 break;
 488         case 3200000:
 489                 cdclk_config->cdclk = cdclk_sel ? 320000 : 228571;
 490                 break;
 491         default:
 492                 drm_err(&dev_priv->drm,
 493                         "Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n",
 494                         cdclk_config->vco, tmp);
 495                 cdclk_config->cdclk = 222222;
 496                 break;
 497         }
 498 }
 499
 500 static void hsw_get_cdclk(struct drm_i915_private *dev_priv,
 501                           struct intel_cdclk_config *cdclk_config)
 502 {
 503         u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 504         u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 505
 506         if (lcpll & LCPLL_CD_SOURCE_FCLK)
 507                 cdclk_config->cdclk = 800000;
 508         else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 509                 cdclk_config->cdclk = 450000;
 510         else if (freq == LCPLL_CLK_FREQ_450)
 511                 cdclk_config->cdclk = 450000;
 512         else if (IS_HASWELL_ULT(dev_priv))
 513                 cdclk_config->cdclk = 337500;
 514         else
 515                 cdclk_config->cdclk = 540000;
 516 }
 517
 518 static int vlv_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
 519 {
 520         int freq_320 = (dev_priv->hpll_freq <<  1) % 320000 != 0 ?
 521                 333333 : 320000;
 522
 523         /*
 524          * We seem to get an unstable or solid color picture at 200MHz.
 525          * Not sure what's wrong. For now use 200MHz only when all pipes
 526          * are off.
 527          */
 528         if (IS_VALLEYVIEW(dev_priv) && min_cdclk > freq_320)
 529                 return 400000;
 530         else if (min_cdclk > 266667)
 531                 return freq_320;
 532         else if (min_cdclk > 0)
 533                 return 266667;
 534         else
 535                 return 200000;
 536 }
 537
 538 static u8 vlv_calc_voltage_level(struct drm_i915_private *dev_priv, int cdclk)
 539 {
 540         if (IS_VALLEYVIEW(dev_priv)) {
 541                 if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
 542                         return 2;
 543                 else if (cdclk >= 266667)
 544                         return 1;
 545                 else
 546                         return 0;
 547         } else {
 548                 /*
 549                  * Specs are full of misinformation, but testing on actual
 550                  * hardware has shown that we just need to write the desired
 551                  * CCK divider into the Punit register.
 552                  */
 553                 return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
 554         }
 555 }
 556
 557 static void vlv_get_cdclk(struct drm_i915_private *dev_priv,
 558                           struct intel_cdclk_config *cdclk_config)
 559 {
 560         u32 val;
 561
 562         vlv_iosf_sb_get(dev_priv,
 563                         BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 564
 565         cdclk_config->vco = vlv_get_hpll_vco(dev_priv);
 566         cdclk_config->cdclk = vlv_get_cck_clock(dev_priv, "cdclk",
 567                                                 CCK_DISPLAY_CLOCK_CONTROL,
 568                                                 cdclk_config->vco);
 569
 570         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 571
 572         vlv_iosf_sb_put(dev_priv,
 573                         BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 574
 575         if (IS_VALLEYVIEW(dev_priv))
 576                 cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK) >>
 577                         DSPFREQGUAR_SHIFT;
 578         else
 579                 cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK_CHV) >>
 580                         DSPFREQGUAR_SHIFT_CHV;
 581 }
 582
 583 static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
 584 {
 585         unsigned int credits, default_credits;
 586
 587         if (IS_CHERRYVIEW(dev_priv))
 588                 default_credits = PFI_CREDIT(12);
 589         else
 590                 default_credits = PFI_CREDIT(8);
 591
 592         if (dev_priv->display.cdclk.hw.cdclk >= dev_priv->czclk_freq) {
 593                 /* CHV suggested value is 31 or 63 */
 594                 if (IS_CHERRYVIEW(dev_priv))
 595                         credits = PFI_CREDIT_63;
 596                 else
 597                         credits = PFI_CREDIT(15);
 598         } else {
 599                 credits = default_credits;
 600         }
 601
 602         /*
 603          * WA - write default credits before re-programming
 604          * FIXME: should we also set the resend bit here?
 605          */
 606         intel_de_write(dev_priv, GCI_CONTROL,
 607                        VGA_FAST_MODE_DISABLE | default_credits);
 608
 609         intel_de_write(dev_priv, GCI_CONTROL,
 610                        VGA_FAST_MODE_DISABLE | credits | PFI_CREDIT_RESEND);
 611
 612         /*
 613          * FIXME is this guaranteed to clear
 614          * immediately or should we poll for it?
 615          */
 616         drm_WARN_ON(&dev_priv->drm,
 617                     intel_de_read(dev_priv, GCI_CONTROL) & PFI_CREDIT_RESEND);
 618 }
 619
 620 static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
 621                           const struct intel_cdclk_config *cdclk_config,
 622                           enum pipe pipe)
 623 {
 624         int cdclk = cdclk_config->cdclk;
 625         u32 val, cmd = cdclk_config->voltage_level;
 626         intel_wakeref_t wakeref;
 627
 628         switch (cdclk) {
 629         case 400000:
 630         case 333333:
 631         case 320000:
 632         case 266667:
 633         case 200000:
 634                 break;
 635         default:
 636                 MISSING_CASE(cdclk);
 637                 return;
 638         }
 639
 640         /* There are cases where we can end up here with power domains
 641          * off and a CDCLK frequency other than the minimum, like when
 642          * issuing a modeset without actually changing any display after
 643          * a system suspend.  So grab the display core domain, which covers
 644          * the HW blocks needed for the following programming.
 645          */
 646         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 647
 648         vlv_iosf_sb_get(dev_priv,
 649                         BIT(VLV_IOSF_SB_CCK) |
 650                         BIT(VLV_IOSF_SB_BUNIT) |
 651                         BIT(VLV_IOSF_SB_PUNIT));
 652
 653         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 654         val &= ~DSPFREQGUAR_MASK;
 655         val |= (cmd << DSPFREQGUAR_SHIFT);
 656         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 657         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 658                       DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
 659                      50)) {
 660                 drm_err(&dev_priv->drm,
 661                         "timed out waiting for CDclk change\n");
 662         }
 663
 664         if (cdclk == 400000) {
 665                 u32 divider;
 666
 667                 divider = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1,
 668                                             cdclk) - 1;
 669
 670                 /* adjust cdclk divider */
 671                 val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
 672                 val &= ~CCK_FREQUENCY_VALUES;
 673                 val |= divider;
 674                 vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val);
 675
 676                 if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) &
 677                               CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT),
 678                              50))
 679                         drm_err(&dev_priv->drm,
 680                                 "timed out waiting for CDclk change\n");
 681         }
 682
 683         /* adjust self-refresh exit latency value */
 684         val = vlv_bunit_read(dev_priv, BUNIT_REG_BISOC);
 685         val &= ~0x7f;
 686
 687         /*
 688          * For high bandwidth configs, we set a higher latency in the bunit
 689          * so that the core display fetch happens in time to avoid underruns.
 690          */
 691         if (cdclk == 400000)
 692                 val |= 4500 / 250; /* 4.5 usec */
 693         else
 694                 val |= 3000 / 250; /* 3.0 usec */
 695         vlv_bunit_write(dev_priv, BUNIT_REG_BISOC, val);
 696
 697         vlv_iosf_sb_put(dev_priv,
 698                         BIT(VLV_IOSF_SB_CCK) |
 699                         BIT(VLV_IOSF_SB_BUNIT) |
 700                         BIT(VLV_IOSF_SB_PUNIT));
 701
 702         intel_update_cdclk(dev_priv);
 703
 704         vlv_program_pfi_credits(dev_priv);
 705
 706         intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 707 }
 708
 709 static void chv_set_cdclk(struct drm_i915_private *dev_priv,
 710                           const struct intel_cdclk_config *cdclk_config,
 711                           enum pipe pipe)
 712 {
 713         int cdclk = cdclk_config->cdclk;
 714         u32 val, cmd = cdclk_config->voltage_level;
 715         intel_wakeref_t wakeref;
 716
 717         switch (cdclk) {
 718         case 333333:
 719         case 320000:
 720         case 266667:
 721         case 200000:
 722                 break;
 723         default:
 724                 MISSING_CASE(cdclk);
 725                 return;
 726         }
 727
 728         /* There are cases where we can end up here with power domains
 729          * off and a CDCLK frequency other than the minimum, like when
 730          * issuing a modeset without actually changing any display after
 731          * a system suspend.  So grab the display core domain, which covers
 732          * the HW blocks needed for the following programming.
 733          */
 734         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 735
 736         vlv_punit_get(dev_priv);
 737         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 738         val &= ~DSPFREQGUAR_MASK_CHV;
 739         val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
 740         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 741         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 742                       DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
 743                      50)) {
 744                 drm_err(&dev_priv->drm,
 745                         "timed out waiting for CDclk change\n");
 746         }
 747
 748         vlv_punit_put(dev_priv);
 749
 750         intel_update_cdclk(dev_priv);
 751
 752         vlv_program_pfi_credits(dev_priv);
 753
 754         intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 755 }
 756
 757 static int bdw_calc_cdclk(int min_cdclk)
 758 {
 759         if (min_cdclk > 540000)
 760                 return 675000;
 761         else if (min_cdclk > 450000)
 762                 return 540000;
 763         else if (min_cdclk > 337500)
 764                 return 450000;
 765         else
 766                 return 337500;
 767 }
 768
 769 static u8 bdw_calc_voltage_level(int cdclk)
 770 {
 771         switch (cdclk) {
 772         default:
 773         case 337500:
 774                 return 2;
 775         case 450000:
 776                 return 0;
 777         case 540000:
 778                 return 1;
 779         case 675000:
 780                 return 3;
 781         }
 782 }
 783
 784 static void bdw_get_cdclk(struct drm_i915_private *dev_priv,
 785                           struct intel_cdclk_config *cdclk_config)
 786 {
 787         u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 788         u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 789
 790         if (lcpll & LCPLL_CD_SOURCE_FCLK)
 791                 cdclk_config->cdclk = 800000;
 792         else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 793                 cdclk_config->cdclk = 450000;
 794         else if (freq == LCPLL_CLK_FREQ_450)
 795                 cdclk_config->cdclk = 450000;
 796         else if (freq == LCPLL_CLK_FREQ_54O_BDW)
 797                 cdclk_config->cdclk = 540000;
 798         else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
 799                 cdclk_config->cdclk = 337500;
 800         else
 801                 cdclk_config->cdclk = 675000;
 802
 803         /*
 804          * Can't read this out :( Let's assume it's
 805          * at least what the CDCLK frequency requires.
 806          */
 807         cdclk_config->voltage_level =
 808                 bdw_calc_voltage_level(cdclk_config->cdclk);
 809 }
 810
 811 static u32 bdw_cdclk_freq_sel(int cdclk)
 812 {
 813         switch (cdclk) {
 814         default:
 815                 MISSING_CASE(cdclk);
 816                 fallthrough;
 817         case 337500:
 818                 return LCPLL_CLK_FREQ_337_5_BDW;
 819         case 450000:
 820                 return LCPLL_CLK_FREQ_450;
 821         case 540000:
 822                 return LCPLL_CLK_FREQ_54O_BDW;
 823         case 675000:
 824                 return LCPLL_CLK_FREQ_675_BDW;
 825         }
 826 }
 827
 828 static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
 829                           const struct intel_cdclk_config *cdclk_config,
 830                           enum pipe pipe)
 831 {
 832         int cdclk = cdclk_config->cdclk;
 833         int ret;
 834
 835         if (drm_WARN(&dev_priv->drm,
 836                      (intel_de_read(dev_priv, LCPLL_CTL) &
 837                       (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK |
 838                        LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE |
 839                        LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW |
 840                        LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK,
 841                      "trying to change cdclk frequency with cdclk not enabled\n"))
 842                 return;
 843
 844         ret = snb_pcode_write(&dev_priv->uncore, BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
 845         if (ret) {
 846                 drm_err(&dev_priv->drm,
 847                         "failed to inform pcode about cdclk change\n");
 848                 return;
 849         }
 850
 851         intel_de_rmw(dev_priv, LCPLL_CTL,
 852                      0, LCPLL_CD_SOURCE_FCLK);
 853
 854         /*
 855          * According to the spec, it should be enough to poll for this 1 us.
 856          * However, extensive testing shows that this can take longer.
 857          */
 858         if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
 859                         LCPLL_CD_SOURCE_FCLK_DONE, 100))
 860                 drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
 861
 862         intel_de_rmw(dev_priv, LCPLL_CTL,
 863                      LCPLL_CLK_FREQ_MASK, bdw_cdclk_freq_sel(cdclk));
 864
 865         intel_de_rmw(dev_priv, LCPLL_CTL,
 866                      LCPLL_CD_SOURCE_FCLK, 0);
 867
 868         if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
 869                          LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
 870                 drm_err(&dev_priv->drm, "Switching back to LCPLL failed\n");
 871
 872         snb_pcode_write(&dev_priv->uncore, HSW_PCODE_DE_WRITE_FREQ_REQ,
 873                         cdclk_config->voltage_level);
 874
 875         intel_de_write(dev_priv, CDCLK_FREQ,
 876                        DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
 877
 878         intel_update_cdclk(dev_priv);
 879 }
 880
 881 static int skl_calc_cdclk(int min_cdclk, int vco)
 882 {
 883         if (vco == 8640000) {
 884                 if (min_cdclk > 540000)
 885                         return 617143;
 886                 else if (min_cdclk > 432000)
 887                         return 540000;
 888                 else if (min_cdclk > 308571)
 889                         return 432000;
 890                 else
 891                         return 308571;
 892         } else {
 893                 if (min_cdclk > 540000)
 894                         return 675000;
 895                 else if (min_cdclk > 450000)
 896                         return 540000;
 897                 else if (min_cdclk > 337500)
 898                         return 450000;
 899                 else
 900                         return 337500;
 901         }
 902 }
 903
 904 static u8 skl_calc_voltage_level(int cdclk)
 905 {
 906         if (cdclk > 540000)
 907                 return 3;
 908         else if (cdclk > 450000)
 909                 return 2;
 910         else if (cdclk > 337500)
 911                 return 1;
 912         else
 913                 return 0;
 914 }
 915
 916 static void skl_dpll0_update(struct drm_i915_private *dev_priv,
 917                              struct intel_cdclk_config *cdclk_config)
 918 {
 919         u32 val;
 920
 921         cdclk_config->ref = 24000;
 922         cdclk_config->vco = 0;
 923
 924         val = intel_de_read(dev_priv, LCPLL1_CTL);
 925         if ((val & LCPLL_PLL_ENABLE) == 0)
 926                 return;
 927
 928         if (drm_WARN_ON(&dev_priv->drm, (val & LCPLL_PLL_LOCK) == 0))
 929                 return;
 930
 931         val = intel_de_read(dev_priv, DPLL_CTRL1);
 932
 933         if (drm_WARN_ON(&dev_priv->drm,
 934                         (val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
 935                                 DPLL_CTRL1_SSC(SKL_DPLL0) |
 936                                 DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) !=
 937                         DPLL_CTRL1_OVERRIDE(SKL_DPLL0)))
 938                 return;
 939
 940         switch (val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) {
 941         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0):
 942         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, SKL_DPLL0):
 943         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, SKL_DPLL0):
 944         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, SKL_DPLL0):
 945                 cdclk_config->vco = 8100000;
 946                 break;
 947         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0):
 948         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, SKL_DPLL0):
 949                 cdclk_config->vco = 8640000;
 950                 break;
 951         default:
 952                 MISSING_CASE(val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
 953                 break;
 954         }
 955 }
 956
 957 static void skl_get_cdclk(struct drm_i915_private *dev_priv,
 958                           struct intel_cdclk_config *cdclk_config)
 959 {
 960         u32 cdctl;
 961
 962         skl_dpll0_update(dev_priv, cdclk_config);
 963
 964         cdclk_config->cdclk = cdclk_config->bypass = cdclk_config->ref;
 965
 966         if (cdclk_config->vco == 0)
 967                 goto out;
 968
 969         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
 970
 971         if (cdclk_config->vco == 8640000) {
 972                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 973                 case CDCLK_FREQ_450_432:
 974                         cdclk_config->cdclk = 432000;
 975                         break;
 976                 case CDCLK_FREQ_337_308:
 977                         cdclk_config->cdclk = 308571;
 978                         break;
 979                 case CDCLK_FREQ_540:
 980                         cdclk_config->cdclk = 540000;
 981                         break;
 982                 case CDCLK_FREQ_675_617:
 983                         cdclk_config->cdclk = 617143;
 984                         break;
 985                 default:
 986                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
 987                         break;
 988                 }
 989         } else {
 990                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 991                 case CDCLK_FREQ_450_432:
 992                         cdclk_config->cdclk = 450000;
 993                         break;
 994                 case CDCLK_FREQ_337_308:
 995                         cdclk_config->cdclk = 337500;
 996                         break;
 997                 case CDCLK_FREQ_540:
 998                         cdclk_config->cdclk = 540000;
 999                         break;
1000                 case CDCLK_FREQ_675_617:
1001                         cdclk_config->cdclk = 675000;
1002                         break;
1003                 default:
1004                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
1005                         break;
1006                 }
1007         }
1008
1009  out:
1010         /*
1011          * Can't read this out :( Let's assume it's
1012          * at least what the CDCLK frequency requires.
1013          */
1014         cdclk_config->voltage_level =
1015                 skl_calc_voltage_level(cdclk_config->cdclk);
1016 }
1017
1018 /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
1019 static int skl_cdclk_decimal(int cdclk)
1020 {
1021         return DIV_ROUND_CLOSEST(cdclk - 1000, 500);
1022 }
1023
1024 static void skl_set_preferred_cdclk_vco(struct drm_i915_private *i915, int vco)
1025 {
1026         bool changed = i915->display.cdclk.skl_preferred_vco_freq != vco;
1027
1028         i915->display.cdclk.skl_preferred_vco_freq = vco;
1029
1030         if (changed)
1031                 intel_update_max_cdclk(i915);
1032 }
1033
1034 static u32 skl_dpll0_link_rate(struct drm_i915_private *dev_priv, int vco)
1035 {
1036         drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
1037
1038         /*
1039          * We always enable DPLL0 with the lowest link rate possible, but still
1040          * taking into account the VCO required to operate the eDP panel at the
1041          * desired frequency. The usual DP link rates operate with a VCO of
1042          * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640.
1043          * The modeset code is responsible for the selection of the exact link
1044          * rate later on, with the constraint of choosing a frequency that
1045          * works with vco.
1046          */
1047         if (vco == 8640000)
1048                 return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0);
1049         else
1050                 return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0);
1051 }
1052
1053 static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco)
1054 {
1055         intel_de_rmw(dev_priv, DPLL_CTRL1,
1056                      DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
1057                      DPLL_CTRL1_SSC(SKL_DPLL0) |
1058                      DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0),
1059                      DPLL_CTRL1_OVERRIDE(SKL_DPLL0) |
1060                      skl_dpll0_link_rate(dev_priv, vco));
1061         intel_de_posting_read(dev_priv, DPLL_CTRL1);
1062
1063         intel_de_rmw(dev_priv, LCPLL1_CTL,
1064                      0, LCPLL_PLL_ENABLE);
1065
1066         if (intel_de_wait_for_set(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 5))
1067                 drm_err(&dev_priv->drm, "DPLL0 not locked\n");
1068
1069         dev_priv->display.cdclk.hw.vco = vco;
1070
1071         /* We'll want to keep using the current vco from now on. */
1072         skl_set_preferred_cdclk_vco(dev_priv, vco);
1073 }
1074
1075 static void skl_dpll0_disable(struct drm_i915_private *dev_priv)
1076 {
1077         intel_de_rmw(dev_priv, LCPLL1_CTL,
1078                      LCPLL_PLL_ENABLE, 0);
1079
1080         if (intel_de_wait_for_clear(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 1))
1081                 drm_err(&dev_priv->drm, "Couldn't disable DPLL0\n");
1082
1083         dev_priv->display.cdclk.hw.vco = 0;
1084 }
1085
1086 static u32 skl_cdclk_freq_sel(struct drm_i915_private *dev_priv,
1087                               int cdclk, int vco)
1088 {
1089         switch (cdclk) {
1090         default:
1091                 drm_WARN_ON(&dev_priv->drm,
1092                             cdclk != dev_priv->display.cdclk.hw.bypass);
1093                 drm_WARN_ON(&dev_priv->drm, vco != 0);
1094                 fallthrough;
1095         case 308571:
1096         case 337500:
1097                 return CDCLK_FREQ_337_308;
1098         case 450000:
1099         case 432000:
1100                 return CDCLK_FREQ_450_432;
1101         case 540000:
1102                 return CDCLK_FREQ_540;
1103         case 617143:
1104         case 675000:
1105                 return CDCLK_FREQ_675_617;
1106         }
1107 }
1108
1109 static void skl_set_cdclk(struct drm_i915_private *dev_priv,
1110                           const struct intel_cdclk_config *cdclk_config,
1111                           enum pipe pipe)
1112 {
1113         int cdclk = cdclk_config->cdclk;
1114         int vco = cdclk_config->vco;
1115         u32 freq_select, cdclk_ctl;
1116         int ret;
1117
1118         /*
1119          * Based on WA#1183 CDCLK rates 308 and 617MHz CDCLK rates are
1120          * unsupported on SKL. In theory this should never happen since only
1121          * the eDP1.4 2.16 and 4.32Gbps rates require it, but eDP1.4 is not
1122          * supported on SKL either, see the above WA. WARN whenever trying to
1123          * use the corresponding VCO freq as that always leads to using the
1124          * minimum 308MHz CDCLK.
1125          */
1126         drm_WARN_ON_ONCE(&dev_priv->drm,
1127                          IS_SKYLAKE(dev_priv) && vco == 8640000);
1128
1129         ret = skl_pcode_request(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1130                                 SKL_CDCLK_PREPARE_FOR_CHANGE,
1131                                 SKL_CDCLK_READY_FOR_CHANGE,
1132                                 SKL_CDCLK_READY_FOR_CHANGE, 3);
1133         if (ret) {
1134                 drm_err(&dev_priv->drm,
1135                         "Failed to inform PCU about cdclk change (%d)\n", ret);
1136                 return;
1137         }
1138
1139         freq_select = skl_cdclk_freq_sel(dev_priv, cdclk, vco);
1140
1141         if (dev_priv->display.cdclk.hw.vco != 0 &&
1142             dev_priv->display.cdclk.hw.vco != vco)
1143                 skl_dpll0_disable(dev_priv);
1144
1145         cdclk_ctl = intel_de_read(dev_priv, CDCLK_CTL);
1146
1147         if (dev_priv->display.cdclk.hw.vco != vco) {
1148                 /* Wa Display #1183: skl,kbl,cfl */
1149                 cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1150                 cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1151                 intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1152         }
1153
1154         /* Wa Display #1183: skl,kbl,cfl */
1155         cdclk_ctl |= CDCLK_DIVMUX_CD_OVERRIDE;
1156         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1157         intel_de_posting_read(dev_priv, CDCLK_CTL);
1158
1159         if (dev_priv->display.cdclk.hw.vco != vco)
1160                 skl_dpll0_enable(dev_priv, vco);
1161
1162         /* Wa Display #1183: skl,kbl,cfl */
1163         cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1164         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1165
1166         cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1167         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1168
1169         /* Wa Display #1183: skl,kbl,cfl */
1170         cdclk_ctl &= ~CDCLK_DIVMUX_CD_OVERRIDE;
1171         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1172         intel_de_posting_read(dev_priv, CDCLK_CTL);
1173
1174         /* inform PCU of the change */
1175         snb_pcode_write(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1176                         cdclk_config->voltage_level);
1177
1178         intel_update_cdclk(dev_priv);
1179 }
1180
1181 static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
1182 {
1183         u32 cdctl, expected;
1184
1185         /*
1186          * check if the pre-os initialized the display
1187          * There is SWF18 scratchpad register defined which is set by the
1188          * pre-os which can be used by the OS drivers to check the status
1189          */
1190         if ((intel_de_read(dev_priv, SWF_ILK(0x18)) & 0x00FFFFFF) == 0)
1191                 goto sanitize;
1192
1193         intel_update_cdclk(dev_priv);
1194         intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
1195
1196         /* Is PLL enabled and locked ? */
1197         if (dev_priv->display.cdclk.hw.vco == 0 ||
1198             dev_priv->display.cdclk.hw.cdclk == dev_priv->display.cdclk.hw.bypass)
1199                 goto sanitize;
1200
1201         /* DPLL okay; verify the cdclock
1202          *
1203          * Noticed in some instances that the freq selection is correct but
1204          * decimal part is programmed wrong from BIOS where pre-os does not
1205          * enable display. Verify the same as well.
1206          */
1207         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1208         expected = (cdctl & CDCLK_FREQ_SEL_MASK) |
1209                 skl_cdclk_decimal(dev_priv->display.cdclk.hw.cdclk);
1210         if (cdctl == expected)
1211                 /* All well; nothing to sanitize */
1212                 return;
1213
1214 sanitize:
1215         drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1216
1217         /* force cdclk programming */
1218         dev_priv->display.cdclk.hw.cdclk = 0;
1219         /* force full PLL disable + enable */
1220         dev_priv->display.cdclk.hw.vco = ~0;
1221 }
1222
1223 static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv)
1224 {
1225         struct intel_cdclk_config cdclk_config;
1226
1227         skl_sanitize_cdclk(dev_priv);
1228
1229         if (dev_priv->display.cdclk.hw.cdclk != 0 &&
1230             dev_priv->display.cdclk.hw.vco != 0) {
1231                 /*
1232                  * Use the current vco as our initial
1233                  * guess as to what the preferred vco is.
1234                  */
1235                 if (dev_priv->display.cdclk.skl_preferred_vco_freq == 0)
1236                         skl_set_preferred_cdclk_vco(dev_priv,
1237                                                     dev_priv->display.cdclk.hw.vco);
1238                 return;
1239         }
1240
1241         cdclk_config = dev_priv->display.cdclk.hw;
1242
1243         cdclk_config.vco = dev_priv->display.cdclk.skl_preferred_vco_freq;
1244         if (cdclk_config.vco == 0)
1245                 cdclk_config.vco = 8100000;
1246         cdclk_config.cdclk = skl_calc_cdclk(0, cdclk_config.vco);
1247         cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1248
1249         skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1250 }
1251
1252 static void skl_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
1253 {
1254         struct intel_cdclk_config cdclk_config = dev_priv->display.cdclk.hw;
1255
1256         cdclk_config.cdclk = cdclk_config.bypass;
1257         cdclk_config.vco = 0;
1258         cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1259
1260         skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1261 }
1262
1263 struct intel_cdclk_vals {
1264         u32 cdclk;
1265         u16 refclk;
1266         u16 waveform;
1267         u8 ratio;
1268 };
1269
1270 static const struct intel_cdclk_vals bxt_cdclk_table[] = {
1271         { .refclk = 19200, .cdclk = 144000, .ratio = 60 },
1272         { .refclk = 19200, .cdclk = 288000, .ratio = 60 },
1273         { .refclk = 19200, .cdclk = 384000, .ratio = 60 },
1274         { .refclk = 19200, .cdclk = 576000, .ratio = 60 },
1275         { .refclk = 19200, .cdclk = 624000, .ratio = 65 },
1276         {}
1277 };
1278
1279 static const struct intel_cdclk_vals glk_cdclk_table[] = {
1280         { .refclk = 19200, .cdclk =  79200, .ratio = 33 },
1281         { .refclk = 19200, .cdclk = 158400, .ratio = 33 },
1282         { .refclk = 19200, .cdclk = 316800, .ratio = 33 },
1283         {}
1284 };
1285
1286 static const struct intel_cdclk_vals icl_cdclk_table[] = {
1287         { .refclk = 19200, .cdclk = 172800, .ratio = 18 },
1288         { .refclk = 19200, .cdclk = 192000, .ratio = 20 },
1289         { .refclk = 19200, .cdclk = 307200, .ratio = 32 },
1290         { .refclk = 19200, .cdclk = 326400, .ratio = 68 },
1291         { .refclk = 19200, .cdclk = 556800, .ratio = 58 },
1292         { .refclk = 19200, .cdclk = 652800, .ratio = 68 },
1293
1294         { .refclk = 24000, .cdclk = 180000, .ratio = 15 },
1295         { .refclk = 24000, .cdclk = 192000, .ratio = 16 },
1296         { .refclk = 24000, .cdclk = 312000, .ratio = 26 },
1297         { .refclk = 24000, .cdclk = 324000, .ratio = 54 },
1298         { .refclk = 24000, .cdclk = 552000, .ratio = 46 },
1299         { .refclk = 24000, .cdclk = 648000, .ratio = 54 },
1300
1301         { .refclk = 38400, .cdclk = 172800, .ratio =  9 },
1302         { .refclk = 38400, .cdclk = 192000, .ratio = 10 },
1303         { .refclk = 38400, .cdclk = 307200, .ratio = 16 },
1304         { .refclk = 38400, .cdclk = 326400, .ratio = 34 },
1305         { .refclk = 38400, .cdclk = 556800, .ratio = 29 },
1306         { .refclk = 38400, .cdclk = 652800, .ratio = 34 },
1307         {}
1308 };
1309
1310 static const struct intel_cdclk_vals rkl_cdclk_table[] = {
1311         { .refclk = 19200, .cdclk = 172800, .ratio =  36 },
1312         { .refclk = 19200, .cdclk = 192000, .ratio =  40 },
1313         { .refclk = 19200, .cdclk = 307200, .ratio =  64 },
1314         { .refclk = 19200, .cdclk = 326400, .ratio = 136 },
1315         { .refclk = 19200, .cdclk = 556800, .ratio = 116 },
1316         { .refclk = 19200, .cdclk = 652800, .ratio = 136 },
1317
1318         { .refclk = 24000, .cdclk = 180000, .ratio =  30 },
1319         { .refclk = 24000, .cdclk = 192000, .ratio =  32 },
1320         { .refclk = 24000, .cdclk = 312000, .ratio =  52 },
1321         { .refclk = 24000, .cdclk = 324000, .ratio = 108 },
1322         { .refclk = 24000, .cdclk = 552000, .ratio =  92 },
1323         { .refclk = 24000, .cdclk = 648000, .ratio = 108 },
1324
1325         { .refclk = 38400, .cdclk = 172800, .ratio = 18 },
1326         { .refclk = 38400, .cdclk = 192000, .ratio = 20 },
1327         { .refclk = 38400, .cdclk = 307200, .ratio = 32 },
1328         { .refclk = 38400, .cdclk = 326400, .ratio = 68 },
1329         { .refclk = 38400, .cdclk = 556800, .ratio = 58 },
1330         { .refclk = 38400, .cdclk = 652800, .ratio = 68 },
1331         {}
1332 };
1333
1334 static const struct intel_cdclk_vals adlp_a_step_cdclk_table[] = {
1335         { .refclk = 19200, .cdclk = 307200, .ratio = 32 },
1336         { .refclk = 19200, .cdclk = 556800, .ratio = 58 },
1337         { .refclk = 19200, .cdclk = 652800, .ratio = 68 },
1338
1339         { .refclk = 24000, .cdclk = 312000, .ratio = 26 },
1340         { .refclk = 24000, .cdclk = 552000, .ratio = 46 },
1341         { .refclk = 24400, .cdclk = 648000, .ratio = 54 },
1342
1343         { .refclk = 38400, .cdclk = 307200, .ratio = 16 },
1344         { .refclk = 38400, .cdclk = 556800, .ratio = 29 },
1345         { .refclk = 38400, .cdclk = 652800, .ratio = 34 },
1346         {}
1347 };
1348
1349 static const struct intel_cdclk_vals adlp_cdclk_table[] = {
1350         { .refclk = 19200, .cdclk = 172800, .ratio = 27 },
1351         { .refclk = 19200, .cdclk = 192000, .ratio = 20 },
1352         { .refclk = 19200, .cdclk = 307200, .ratio = 32 },
1353         { .refclk = 19200, .cdclk = 556800, .ratio = 58 },
1354         { .refclk = 19200, .cdclk = 652800, .ratio = 68 },
1355
1356         { .refclk = 24000, .cdclk = 176000, .ratio = 22 },
1357         { .refclk = 24000, .cdclk = 192000, .ratio = 16 },
1358         { .refclk = 24000, .cdclk = 312000, .ratio = 26 },
1359         { .refclk = 24000, .cdclk = 552000, .ratio = 46 },
1360         { .refclk = 24000, .cdclk = 648000, .ratio = 54 },
1361
1362         { .refclk = 38400, .cdclk = 179200, .ratio = 14 },
1363         { .refclk = 38400, .cdclk = 192000, .ratio = 10 },
1364         { .refclk = 38400, .cdclk = 307200, .ratio = 16 },
1365         { .refclk = 38400, .cdclk = 556800, .ratio = 29 },
1366         { .refclk = 38400, .cdclk = 652800, .ratio = 34 },
1367         {}
1368 };
1369
1370 static const struct intel_cdclk_vals rplu_cdclk_table[] = {
1371         { .refclk = 19200, .cdclk = 172800, .ratio = 27 },
1372         { .refclk = 19200, .cdclk = 192000, .ratio = 20 },
1373         { .refclk = 19200, .cdclk = 307200, .ratio = 32 },
1374         { .refclk = 19200, .cdclk = 480000, .ratio = 50 },
1375         { .refclk = 19200, .cdclk = 556800, .ratio = 58 },
1376         { .refclk = 19200, .cdclk = 652800, .ratio = 68 },
1377
1378         { .refclk = 24000, .cdclk = 176000, .ratio = 22 },
1379         { .refclk = 24000, .cdclk = 192000, .ratio = 16 },
1380         { .refclk = 24000, .cdclk = 312000, .ratio = 26 },
1381         { .refclk = 24000, .cdclk = 480000, .ratio = 40 },
1382         { .refclk = 24000, .cdclk = 552000, .ratio = 46 },
1383         { .refclk = 24000, .cdclk = 648000, .ratio = 54 },
1384
1385         { .refclk = 38400, .cdclk = 179200, .ratio = 14 },
1386         { .refclk = 38400, .cdclk = 192000, .ratio = 10 },
1387         { .refclk = 38400, .cdclk = 307200, .ratio = 16 },
1388         { .refclk = 38400, .cdclk = 480000, .ratio = 25 },
1389         { .refclk = 38400, .cdclk = 556800, .ratio = 29 },
1390         { .refclk = 38400, .cdclk = 652800, .ratio = 34 },
1391         {}
1392 };
1393
1394 static const struct intel_cdclk_vals dg2_cdclk_table[] = {
1395         { .refclk = 38400, .cdclk = 163200, .ratio = 34, .waveform = 0x8888 },
1396         { .refclk = 38400, .cdclk = 204000, .ratio = 34, .waveform = 0x9248 },
1397         { .refclk = 38400, .cdclk = 244800, .ratio = 34, .waveform = 0xa4a4 },
1398         { .refclk = 38400, .cdclk = 285600, .ratio = 34, .waveform = 0xa54a },
1399         { .refclk = 38400, .cdclk = 326400, .ratio = 34, .waveform = 0xaaaa },
1400         { .refclk = 38400, .cdclk = 367200, .ratio = 34, .waveform = 0xad5a },
1401         { .refclk = 38400, .cdclk = 408000, .ratio = 34, .waveform = 0xb6b6 },
1402         { .refclk = 38400, .cdclk = 448800, .ratio = 34, .waveform = 0xdbb6 },
1403         { .refclk = 38400, .cdclk = 489600, .ratio = 34, .waveform = 0xeeee },
1404         { .refclk = 38400, .cdclk = 530400, .ratio = 34, .waveform = 0xf7de },
1405         { .refclk = 38400, .cdclk = 571200, .ratio = 34, .waveform = 0xfefe },
1406         { .refclk = 38400, .cdclk = 612000, .ratio = 34, .waveform = 0xfffe },
1407         { .refclk = 38400, .cdclk = 652800, .ratio = 34, .waveform = 0xffff },
1408         {}
1409 };
1410
1411 static const struct intel_cdclk_vals mtl_cdclk_table[] = {
1412         { .refclk = 38400, .cdclk = 172800, .ratio = 16, .waveform = 0xad5a },
1413         { .refclk = 38400, .cdclk = 192000, .ratio = 16, .waveform = 0xb6b6 },
1414         { .refclk = 38400, .cdclk = 307200, .ratio = 16, .waveform = 0x0000 },
1415         { .refclk = 38400, .cdclk = 480000, .ratio = 25, .waveform = 0x0000 },
1416         { .refclk = 38400, .cdclk = 556800, .ratio = 29, .waveform = 0x0000 },
1417         { .refclk = 38400, .cdclk = 652800, .ratio = 34, .waveform = 0x0000 },
1418         {}
1419 };
1420
1421 static const struct intel_cdclk_vals xe2lpd_cdclk_table[] = {
1422         { .refclk = 38400, .cdclk = 153600, .ratio = 16, .waveform = 0xaaaa },
1423         { .refclk = 38400, .cdclk = 172800, .ratio = 16, .waveform = 0xad5a },
1424         { .refclk = 38400, .cdclk = 192000, .ratio = 16, .waveform = 0xb6b6 },
1425         { .refclk = 38400, .cdclk = 211200, .ratio = 16, .waveform = 0xdbb6 },
1426         { .refclk = 38400, .cdclk = 230400, .ratio = 16, .waveform = 0xeeee },
1427         { .refclk = 38400, .cdclk = 249600, .ratio = 16, .waveform = 0xf7de },
1428         { .refclk = 38400, .cdclk = 268800, .ratio = 16, .waveform = 0xfefe },
1429         { .refclk = 38400, .cdclk = 288000, .ratio = 16, .waveform = 0xfffe },
1430         { .refclk = 38400, .cdclk = 307200, .ratio = 16, .waveform = 0xffff },
1431         { .refclk = 38400, .cdclk = 330000, .ratio = 25, .waveform = 0xdbb6 },
1432         { .refclk = 38400, .cdclk = 360000, .ratio = 25, .waveform = 0xeeee },
1433         { .refclk = 38400, .cdclk = 390000, .ratio = 25, .waveform = 0xf7de },
1434         { .refclk = 38400, .cdclk = 420000, .ratio = 25, .waveform = 0xfefe },
1435         { .refclk = 38400, .cdclk = 450000, .ratio = 25, .waveform = 0xfffe },
1436         { .refclk = 38400, .cdclk = 480000, .ratio = 25, .waveform = 0xffff },
1437         { .refclk = 38400, .cdclk = 487200, .ratio = 29, .waveform = 0xfefe },
1438         { .refclk = 38400, .cdclk = 522000, .ratio = 29, .waveform = 0xfffe },
1439         { .refclk = 38400, .cdclk = 556800, .ratio = 29, .waveform = 0xffff },
1440         { .refclk = 38400, .cdclk = 571200, .ratio = 34, .waveform = 0xfefe },
1441         { .refclk = 38400, .cdclk = 612000, .ratio = 34, .waveform = 0xfffe },
1442         { .refclk = 38400, .cdclk = 652800, .ratio = 34, .waveform = 0xffff },
1443         {}
1444 };
1445
1446 static const int cdclk_squash_len = 16;
1447
1448 static int cdclk_squash_divider(u16 waveform)
1449 {
1450         return hweight16(waveform ?: 0xffff);
1451 }
1452
1453 static int cdclk_divider(int cdclk, int vco, u16 waveform)
1454 {
1455         /* 2 * cd2x divider */
1456         return DIV_ROUND_CLOSEST(vco * cdclk_squash_divider(waveform),
1457                                  cdclk * cdclk_squash_len);
1458 }
1459
1460 static int bxt_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
1461 {
1462         const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1463         int i;
1464
1465         for (i = 0; table[i].refclk; i++)
1466                 if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1467                     table[i].cdclk >= min_cdclk)
1468                         return table[i].cdclk;
1469
1470         drm_WARN(&dev_priv->drm, 1,
1471                  "Cannot satisfy minimum cdclk %d with refclk %u\n",
1472                  min_cdclk, dev_priv->display.cdclk.hw.ref);
1473         return 0;
1474 }
1475
1476 static int bxt_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
1477 {
1478         const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1479         int i;
1480
1481         if (cdclk == dev_priv->display.cdclk.hw.bypass)
1482                 return 0;
1483
1484         for (i = 0; table[i].refclk; i++)
1485                 if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1486                     table[i].cdclk == cdclk)
1487                         return dev_priv->display.cdclk.hw.ref * table[i].ratio;
1488
1489         drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1490                  cdclk, dev_priv->display.cdclk.hw.ref);
1491         return 0;
1492 }
1493
1494 static u8 bxt_calc_voltage_level(int cdclk)
1495 {
1496         return DIV_ROUND_UP(cdclk, 25000);
1497 }
1498
1499 static u8 calc_voltage_level(int cdclk, int num_voltage_levels,
1500                              const int voltage_level_max_cdclk[])
1501 {
1502         int voltage_level;
1503
1504         for (voltage_level = 0; voltage_level < num_voltage_levels; voltage_level++) {
1505                 if (cdclk <= voltage_level_max_cdclk[voltage_level])
1506                         return voltage_level;
1507         }
1508
1509         MISSING_CASE(cdclk);
1510         return num_voltage_levels - 1;
1511 }
1512
1513 static u8 icl_calc_voltage_level(int cdclk)
1514 {
1515         static const int icl_voltage_level_max_cdclk[] = {
1516                 [0] = 312000,
1517                 [1] = 556800,
1518                 [2] = 652800,
1519         };
1520
1521         return calc_voltage_level(cdclk,
1522                                   ARRAY_SIZE(icl_voltage_level_max_cdclk),
1523                                   icl_voltage_level_max_cdclk);
1524 }
1525
1526 static u8 ehl_calc_voltage_level(int cdclk)
1527 {
1528         static const int ehl_voltage_level_max_cdclk[] = {
1529                 [0] = 180000,
1530                 [1] = 312000,
1531                 [2] = 326400,
1532                 /*
1533                  * Bspec lists the limit as 556.8 MHz, but some JSL
1534                  * development boards (at least) boot with 652.8 MHz
1535                  */
1536                 [3] = 652800,
1537         };
1538
1539         return calc_voltage_level(cdclk,
1540                                   ARRAY_SIZE(ehl_voltage_level_max_cdclk),
1541                                   ehl_voltage_level_max_cdclk);
1542 }
1543
1544 static u8 tgl_calc_voltage_level(int cdclk)
1545 {
1546         static const int tgl_voltage_level_max_cdclk[] = {
1547                 [0] = 312000,
1548                 [1] = 326400,
1549                 [2] = 556800,
1550                 [3] = 652800,
1551         };
1552
1553         return calc_voltage_level(cdclk,
1554                                   ARRAY_SIZE(tgl_voltage_level_max_cdclk),
1555                                   tgl_voltage_level_max_cdclk);
1556 }
1557
1558 static u8 rplu_calc_voltage_level(int cdclk)
1559 {
1560         static const int rplu_voltage_level_max_cdclk[] = {
1561                 [0] = 312000,
1562                 [1] = 480000,
1563                 [2] = 556800,
1564                 [3] = 652800,
1565         };
1566
1567         return calc_voltage_level(cdclk,
1568                                   ARRAY_SIZE(rplu_voltage_level_max_cdclk),
1569                                   rplu_voltage_level_max_cdclk);
1570 }
1571
1572 static void icl_readout_refclk(struct drm_i915_private *dev_priv,
1573                                struct intel_cdclk_config *cdclk_config)
1574 {
1575         u32 dssm = intel_de_read(dev_priv, SKL_DSSM) & ICL_DSSM_CDCLK_PLL_REFCLK_MASK;
1576
1577         switch (dssm) {
1578         default:
1579                 MISSING_CASE(dssm);
1580                 fallthrough;
1581         case ICL_DSSM_CDCLK_PLL_REFCLK_24MHz:
1582                 cdclk_config->ref = 24000;
1583                 break;
1584         case ICL_DSSM_CDCLK_PLL_REFCLK_19_2MHz:
1585                 cdclk_config->ref = 19200;
1586                 break;
1587         case ICL_DSSM_CDCLK_PLL_REFCLK_38_4MHz:
1588                 cdclk_config->ref = 38400;
1589                 break;
1590         }
1591 }
1592
1593 static void bxt_de_pll_readout(struct drm_i915_private *dev_priv,
1594                                struct intel_cdclk_config *cdclk_config)
1595 {
1596         u32 val, ratio;
1597
1598         if (IS_DG2(dev_priv))
1599                 cdclk_config->ref = 38400;
1600         else if (DISPLAY_VER(dev_priv) >= 11)
1601                 icl_readout_refclk(dev_priv, cdclk_config);
1602         else
1603                 cdclk_config->ref = 19200;
1604
1605         val = intel_de_read(dev_priv, BXT_DE_PLL_ENABLE);
1606         if ((val & BXT_DE_PLL_PLL_ENABLE) == 0 ||
1607             (val & BXT_DE_PLL_LOCK) == 0) {
1608                 /*
1609                  * CDCLK PLL is disabled, the VCO/ratio doesn't matter, but
1610                  * setting it to zero is a way to signal that.
1611                  */
1612                 cdclk_config->vco = 0;
1613                 return;
1614         }
1615
1616         /*
1617          * DISPLAY_VER >= 11 have the ratio directly in the PLL enable register,
1618          * gen9lp had it in a separate PLL control register.
1619          */
1620         if (DISPLAY_VER(dev_priv) >= 11)
1621                 ratio = val & ICL_CDCLK_PLL_RATIO_MASK;
1622         else
1623                 ratio = intel_de_read(dev_priv, BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK;
1624
1625         cdclk_config->vco = ratio * cdclk_config->ref;
1626 }
1627
1628 static void bxt_get_cdclk(struct drm_i915_private *dev_priv,
1629                           struct intel_cdclk_config *cdclk_config)
1630 {
1631         u32 squash_ctl = 0;
1632         u32 divider;
1633         int div;
1634
1635         bxt_de_pll_readout(dev_priv, cdclk_config);
1636
1637         if (DISPLAY_VER(dev_priv) >= 12)
1638                 cdclk_config->bypass = cdclk_config->ref / 2;
1639         else if (DISPLAY_VER(dev_priv) >= 11)
1640                 cdclk_config->bypass = 50000;
1641         else
1642                 cdclk_config->bypass = cdclk_config->ref;
1643
1644         if (cdclk_config->vco == 0) {
1645                 cdclk_config->cdclk = cdclk_config->bypass;
1646                 goto out;
1647         }
1648
1649         divider = intel_de_read(dev_priv, CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
1650
1651         switch (divider) {
1652         case BXT_CDCLK_CD2X_DIV_SEL_1:
1653                 div = 2;
1654                 break;
1655         case BXT_CDCLK_CD2X_DIV_SEL_1_5:
1656                 div = 3;
1657                 break;
1658         case BXT_CDCLK_CD2X_DIV_SEL_2:
1659                 div = 4;
1660                 break;
1661         case BXT_CDCLK_CD2X_DIV_SEL_4:
1662                 div = 8;
1663                 break;
1664         default:
1665                 MISSING_CASE(divider);
1666                 return;
1667         }
1668
1669         if (HAS_CDCLK_SQUASH(dev_priv))
1670                 squash_ctl = intel_de_read(dev_priv, CDCLK_SQUASH_CTL);
1671
1672         if (squash_ctl & CDCLK_SQUASH_ENABLE) {
1673                 u16 waveform;
1674                 int size;
1675
1676                 size = REG_FIELD_GET(CDCLK_SQUASH_WINDOW_SIZE_MASK, squash_ctl) + 1;
1677                 waveform = REG_FIELD_GET(CDCLK_SQUASH_WAVEFORM_MASK, squash_ctl) >> (16 - size);
1678
1679                 cdclk_config->cdclk = DIV_ROUND_CLOSEST(hweight16(waveform) *
1680                                                         cdclk_config->vco, size * div);
1681         } else {
1682                 cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco, div);
1683         }
1684
1685  out:
1686         if (DISPLAY_VER(dev_priv) >= 20)
1687                 cdclk_config->joined_mbus = intel_de_read(dev_priv, MBUS_CTL) & MBUS_JOIN;
1688         /*
1689          * Can't read this out :( Let's assume it's
1690          * at least what the CDCLK frequency requires.
1691          */
1692         cdclk_config->voltage_level =
1693                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config->cdclk);
1694 }
1695
1696 static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
1697 {
1698         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, 0);
1699
1700         /* Timeout 200us */
1701         if (intel_de_wait_for_clear(dev_priv,
1702                                     BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1703                 drm_err(&dev_priv->drm, "timeout waiting for DE PLL unlock\n");
1704
1705         dev_priv->display.cdclk.hw.vco = 0;
1706 }
1707
1708 static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
1709 {
1710         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1711
1712         intel_de_rmw(dev_priv, BXT_DE_PLL_CTL,
1713                      BXT_DE_PLL_RATIO_MASK, BXT_DE_PLL_RATIO(ratio));
1714
1715         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
1716
1717         /* Timeout 200us */
1718         if (intel_de_wait_for_set(dev_priv,
1719                                   BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1720                 drm_err(&dev_priv->drm, "timeout waiting for DE PLL lock\n");
1721
1722         dev_priv->display.cdclk.hw.vco = vco;
1723 }
1724
1725 static void icl_cdclk_pll_disable(struct drm_i915_private *dev_priv)
1726 {
1727         intel_de_rmw(dev_priv, BXT_DE_PLL_ENABLE,
1728                      BXT_DE_PLL_PLL_ENABLE, 0);
1729
1730         /* Timeout 200us */
1731         if (intel_de_wait_for_clear(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1732                 drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL unlock\n");
1733
1734         dev_priv->display.cdclk.hw.vco = 0;
1735 }
1736
1737 static void icl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco)
1738 {
1739         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1740         u32 val;
1741
1742         val = ICL_CDCLK_PLL_RATIO(ratio);
1743         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1744
1745         val |= BXT_DE_PLL_PLL_ENABLE;
1746         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1747
1748         /* Timeout 200us */
1749         if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1750                 drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL lock\n");
1751
1752         dev_priv->display.cdclk.hw.vco = vco;
1753 }
1754
1755 static void adlp_cdclk_pll_crawl(struct drm_i915_private *dev_priv, int vco)
1756 {
1757         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1758         u32 val;
1759
1760         /* Write PLL ratio without disabling */
1761         val = ICL_CDCLK_PLL_RATIO(ratio) | BXT_DE_PLL_PLL_ENABLE;
1762         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1763
1764         /* Submit freq change request */
1765         val |= BXT_DE_PLL_FREQ_REQ;
1766         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1767
1768         /* Timeout 200us */
1769         if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE,
1770                                   BXT_DE_PLL_LOCK | BXT_DE_PLL_FREQ_REQ_ACK, 1))
1771                 drm_err(&dev_priv->drm, "timeout waiting for FREQ change request ack\n");
1772
1773         val &= ~BXT_DE_PLL_FREQ_REQ;
1774         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1775
1776         dev_priv->display.cdclk.hw.vco = vco;
1777 }
1778
1779 static u32 bxt_cdclk_cd2x_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
1780 {
1781         if (DISPLAY_VER(dev_priv) >= 12) {
1782                 if (pipe == INVALID_PIPE)
1783                         return TGL_CDCLK_CD2X_PIPE_NONE;
1784                 else
1785                         return TGL_CDCLK_CD2X_PIPE(pipe);
1786         } else if (DISPLAY_VER(dev_priv) >= 11) {
1787                 if (pipe == INVALID_PIPE)
1788                         return ICL_CDCLK_CD2X_PIPE_NONE;
1789                 else
1790                         return ICL_CDCLK_CD2X_PIPE(pipe);
1791         } else {
1792                 if (pipe == INVALID_PIPE)
1793                         return BXT_CDCLK_CD2X_PIPE_NONE;
1794                 else
1795                         return BXT_CDCLK_CD2X_PIPE(pipe);
1796         }
1797 }
1798
1799 static u32 bxt_cdclk_cd2x_div_sel(struct drm_i915_private *dev_priv,
1800                                   int cdclk, int vco, u16 waveform)
1801 {
1802         /* cdclk = vco / 2 / div{1,1.5,2,4} */
1803         switch (cdclk_divider(cdclk, vco, waveform)) {
1804         default:
1805                 drm_WARN_ON(&dev_priv->drm,
1806                             cdclk != dev_priv->display.cdclk.hw.bypass);
1807                 drm_WARN_ON(&dev_priv->drm, vco != 0);
1808                 fallthrough;
1809         case 2:
1810                 return BXT_CDCLK_CD2X_DIV_SEL_1;
1811         case 3:
1812                 return BXT_CDCLK_CD2X_DIV_SEL_1_5;
1813         case 4:
1814                 return BXT_CDCLK_CD2X_DIV_SEL_2;
1815         case 8:
1816                 return BXT_CDCLK_CD2X_DIV_SEL_4;
1817         }
1818 }
1819
1820 static u16 cdclk_squash_waveform(struct drm_i915_private *dev_priv,
1821                                  int cdclk)
1822 {
1823         const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1824         int i;
1825
1826         if (cdclk == dev_priv->display.cdclk.hw.bypass)
1827                 return 0;
1828
1829         for (i = 0; table[i].refclk; i++)
1830                 if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1831                     table[i].cdclk == cdclk)
1832                         return table[i].waveform;
1833
1834         drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1835                  cdclk, dev_priv->display.cdclk.hw.ref);
1836
1837         return 0xffff;
1838 }
1839
1840 static void icl_cdclk_pll_update(struct drm_i915_private *i915, int vco)
1841 {
1842         if (i915->display.cdclk.hw.vco != 0 &&
1843             i915->display.cdclk.hw.vco != vco)
1844                 icl_cdclk_pll_disable(i915);
1845
1846         if (i915->display.cdclk.hw.vco != vco)
1847                 icl_cdclk_pll_enable(i915, vco);
1848 }
1849
1850 static void bxt_cdclk_pll_update(struct drm_i915_private *i915, int vco)
1851 {
1852         if (i915->display.cdclk.hw.vco != 0 &&
1853             i915->display.cdclk.hw.vco != vco)
1854                 bxt_de_pll_disable(i915);
1855
1856         if (i915->display.cdclk.hw.vco != vco)
1857                 bxt_de_pll_enable(i915, vco);
1858 }
1859
1860 static void dg2_cdclk_squash_program(struct drm_i915_private *i915,
1861                                      u16 waveform)
1862 {
1863         u32 squash_ctl = 0;
1864
1865         if (waveform)
1866                 squash_ctl = CDCLK_SQUASH_ENABLE |
1867                              CDCLK_SQUASH_WINDOW_SIZE(0xf) | waveform;
1868
1869         intel_de_write(i915, CDCLK_SQUASH_CTL, squash_ctl);
1870 }
1871
1872 static bool cdclk_pll_is_unknown(unsigned int vco)
1873 {
1874         /*
1875          * Ensure driver does not take the crawl path for the
1876          * case when the vco is set to ~0 in the
1877          * sanitize path.
1878          */
1879         return vco == ~0;
1880 }
1881
1882 static bool mdclk_source_is_cdclk_pll(struct drm_i915_private *i915)
1883 {
1884         return DISPLAY_VER(i915) >= 20;
1885 }
1886
1887 static u32 xe2lpd_mdclk_source_sel(struct drm_i915_private *i915)
1888 {
1889         if (mdclk_source_is_cdclk_pll(i915))
1890                 return MDCLK_SOURCE_SEL_CDCLK_PLL;
1891
1892         return MDCLK_SOURCE_SEL_CD2XCLK;
1893 }
1894
1895 int intel_mdclk_cdclk_ratio(struct drm_i915_private *i915,
1896                             const struct intel_cdclk_config *cdclk_config)
1897 {
1898         if (mdclk_source_is_cdclk_pll(i915))
1899                 return DIV_ROUND_UP(cdclk_config->vco, cdclk_config->cdclk);
1900
1901         /* Otherwise, source for MDCLK is CD2XCLK. */
1902         return 2;
1903 }
1904
1905 static void xe2lpd_mdclk_cdclk_ratio_program(struct drm_i915_private *i915,
1906                                              const struct intel_cdclk_config *cdclk_config)
1907 {
1908         intel_dbuf_mdclk_cdclk_ratio_update(i915,
1909                                             intel_mdclk_cdclk_ratio(i915, cdclk_config),
1910                                             cdclk_config->joined_mbus);
1911 }
1912
1913 static bool cdclk_compute_crawl_and_squash_midpoint(struct drm_i915_private *i915,
1914                                                     const struct intel_cdclk_config *old_cdclk_config,
1915                                                     const struct intel_cdclk_config *new_cdclk_config,
1916                                                     struct intel_cdclk_config *mid_cdclk_config)
1917 {
1918         u16 old_waveform, new_waveform, mid_waveform;
1919         int old_div, new_div, mid_div;
1920
1921         /* Return if PLL is in an unknown state, force a complete disable and re-enable. */
1922         if (cdclk_pll_is_unknown(old_cdclk_config->vco))
1923                 return false;
1924
1925         /* Return if both Squash and Crawl are not present */
1926         if (!HAS_CDCLK_CRAWL(i915) || !HAS_CDCLK_SQUASH(i915))
1927                 return false;
1928
1929         old_waveform = cdclk_squash_waveform(i915, old_cdclk_config->cdclk);
1930         new_waveform = cdclk_squash_waveform(i915, new_cdclk_config->cdclk);
1931
1932         /* Return if Squash only or Crawl only is the desired action */
1933         if (old_cdclk_config->vco == 0 || new_cdclk_config->vco == 0 ||
1934             old_cdclk_config->vco == new_cdclk_config->vco ||
1935             old_waveform == new_waveform)
1936                 return false;
1937
1938         old_div = cdclk_divider(old_cdclk_config->cdclk,
1939                                 old_cdclk_config->vco, old_waveform);
1940         new_div = cdclk_divider(new_cdclk_config->cdclk,
1941                                 new_cdclk_config->vco, new_waveform);
1942
1943         /*
1944          * Should not happen currently. We might need more midpoint
1945          * transitions if we need to also change the cd2x divider.
1946          */
1947         if (drm_WARN_ON(&i915->drm, old_div != new_div))
1948                 return false;
1949
1950         *mid_cdclk_config = *new_cdclk_config;
1951
1952         /*
1953          * Populate the mid_cdclk_config accordingly.
1954          * - If moving to a higher cdclk, the desired action is squashing.
1955          * The mid cdclk config should have the new (squash) waveform.
1956          * - If moving to a lower cdclk, the desired action is crawling.
1957          * The mid cdclk config should have the new vco.
1958          */
1959
1960         if (cdclk_squash_divider(new_waveform) > cdclk_squash_divider(old_waveform)) {
1961                 mid_cdclk_config->vco = old_cdclk_config->vco;
1962                 mid_div = old_div;
1963                 mid_waveform = new_waveform;
1964         } else {
1965                 mid_cdclk_config->vco = new_cdclk_config->vco;
1966                 mid_div = new_div;
1967                 mid_waveform = old_waveform;
1968         }
1969
1970         mid_cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_squash_divider(mid_waveform) *
1971                                                     mid_cdclk_config->vco,
1972                                                     cdclk_squash_len * mid_div);
1973
1974         /* make sure the mid clock came out sane */
1975
1976         drm_WARN_ON(&i915->drm, mid_cdclk_config->cdclk <
1977                     min(old_cdclk_config->cdclk, new_cdclk_config->cdclk));
1978         drm_WARN_ON(&i915->drm, mid_cdclk_config->cdclk >
1979                     i915->display.cdclk.max_cdclk_freq);
1980         drm_WARN_ON(&i915->drm, cdclk_squash_waveform(i915, mid_cdclk_config->cdclk) !=
1981                     mid_waveform);
1982
1983         return true;
1984 }
1985
1986 static bool pll_enable_wa_needed(struct drm_i915_private *dev_priv)
1987 {
1988         return (DISPLAY_VER_FULL(dev_priv) == IP_VER(20, 0) ||
1989                 DISPLAY_VER_FULL(dev_priv) == IP_VER(14, 0) ||
1990                 IS_DG2(dev_priv)) &&
1991                 dev_priv->display.cdclk.hw.vco > 0;
1992 }
1993
1994 static u32 bxt_cdclk_ctl(struct drm_i915_private *i915,
1995                          const struct intel_cdclk_config *cdclk_config,
1996                          enum pipe pipe)
1997 {
1998         int cdclk = cdclk_config->cdclk;
1999         int vco = cdclk_config->vco;
2000         u16 waveform;
2001         u32 val;
2002
2003         waveform = cdclk_squash_waveform(i915, cdclk);
2004
2005         val = bxt_cdclk_cd2x_div_sel(i915, cdclk, vco, waveform) |
2006                 bxt_cdclk_cd2x_pipe(i915, pipe);
2007
2008         /*
2009          * Disable SSA Precharge when CD clock frequency < 500 MHz,
2010          * enable otherwise.
2011          */
2012         if ((IS_GEMINILAKE(i915) || IS_BROXTON(i915)) &&
2013             cdclk >= 500000)
2014                 val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
2015
2016         if (DISPLAY_VER(i915) >= 20)
2017                 val |= xe2lpd_mdclk_source_sel(i915);
2018         else
2019                 val |= skl_cdclk_decimal(cdclk);
2020
2021         return val;
2022 }
2023
2024 static void _bxt_set_cdclk(struct drm_i915_private *dev_priv,
2025                            const struct intel_cdclk_config *cdclk_config,
2026                            enum pipe pipe)
2027 {
2028         int cdclk = cdclk_config->cdclk;
2029         int vco = cdclk_config->vco;
2030
2031         if (HAS_CDCLK_CRAWL(dev_priv) && dev_priv->display.cdclk.hw.vco > 0 && vco > 0 &&
2032             !cdclk_pll_is_unknown(dev_priv->display.cdclk.hw.vco)) {
2033                 if (dev_priv->display.cdclk.hw.vco != vco)
2034                         adlp_cdclk_pll_crawl(dev_priv, vco);
2035         } else if (DISPLAY_VER(dev_priv) >= 11) {
2036                 /* wa_15010685871: dg2, mtl */
2037                 if (pll_enable_wa_needed(dev_priv))
2038                         dg2_cdclk_squash_program(dev_priv, 0);
2039
2040                 icl_cdclk_pll_update(dev_priv, vco);
2041         } else
2042                 bxt_cdclk_pll_update(dev_priv, vco);
2043
2044         if (HAS_CDCLK_SQUASH(dev_priv)) {
2045                 u16 waveform = cdclk_squash_waveform(dev_priv, cdclk);
2046
2047                 dg2_cdclk_squash_program(dev_priv, waveform);
2048         }
2049
2050         intel_de_write(dev_priv, CDCLK_CTL, bxt_cdclk_ctl(dev_priv, cdclk_config, pipe));
2051
2052         if (pipe != INVALID_PIPE)
2053                 intel_crtc_wait_for_next_vblank(intel_crtc_for_pipe(dev_priv, pipe));
2054 }
2055
2056 static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
2057                           const struct intel_cdclk_config *cdclk_config,
2058                           enum pipe pipe)
2059 {
2060         struct intel_cdclk_config mid_cdclk_config;
2061         int cdclk = cdclk_config->cdclk;
2062         int ret = 0;
2063
2064         /*
2065          * Inform power controller of upcoming frequency change.
2066          * Display versions 14 and beyond do not follow the PUnit
2067          * mailbox communication, skip
2068          * this step.
2069          */
2070         if (DISPLAY_VER(dev_priv) >= 14 || IS_DG2(dev_priv))
2071                 /* NOOP */;
2072         else if (DISPLAY_VER(dev_priv) >= 11)
2073                 ret = skl_pcode_request(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
2074                                         SKL_CDCLK_PREPARE_FOR_CHANGE,
2075                                         SKL_CDCLK_READY_FOR_CHANGE,
2076                                         SKL_CDCLK_READY_FOR_CHANGE, 3);
2077         else
2078                 /*
2079                  * BSpec requires us to wait up to 150usec, but that leads to
2080                  * timeouts; the 2ms used here is based on experiment.
2081                  */
2082                 ret = snb_pcode_write_timeout(&dev_priv->uncore,
2083                                               HSW_PCODE_DE_WRITE_FREQ_REQ,
2084                                               0x80000000, 150, 2);
2085
2086         if (ret) {
2087                 drm_err(&dev_priv->drm,
2088                         "Failed to inform PCU about cdclk change (err %d, freq %d)\n",
2089                         ret, cdclk);
2090                 return;
2091         }
2092
2093         if (DISPLAY_VER(dev_priv) >= 20 && cdclk < dev_priv->display.cdclk.hw.cdclk)
2094                 xe2lpd_mdclk_cdclk_ratio_program(dev_priv, cdclk_config);
2095
2096         if (cdclk_compute_crawl_and_squash_midpoint(dev_priv, &dev_priv->display.cdclk.hw,
2097                                                     cdclk_config, &mid_cdclk_config)) {
2098                 _bxt_set_cdclk(dev_priv, &mid_cdclk_config, pipe);
2099                 _bxt_set_cdclk(dev_priv, cdclk_config, pipe);
2100         } else {
2101                 _bxt_set_cdclk(dev_priv, cdclk_config, pipe);
2102         }
2103
2104         if (DISPLAY_VER(dev_priv) >= 20 && cdclk > dev_priv->display.cdclk.hw.cdclk)
2105                 xe2lpd_mdclk_cdclk_ratio_program(dev_priv, cdclk_config);
2106
2107         if (DISPLAY_VER(dev_priv) >= 14)
2108                 /*
2109                  * NOOP - No Pcode communication needed for
2110                  * Display versions 14 and beyond
2111                  */;
2112         else if (DISPLAY_VER(dev_priv) >= 11 && !IS_DG2(dev_priv))
2113                 ret = snb_pcode_write(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
2114                                       cdclk_config->voltage_level);
2115         if (DISPLAY_VER(dev_priv) < 11) {
2116                 /*
2117                  * The timeout isn't specified, the 2ms used here is based on
2118                  * experiment.
2119                  * FIXME: Waiting for the request completion could be delayed
2120                  * until the next PCODE request based on BSpec.
2121                  */
2122                 ret = snb_pcode_write_timeout(&dev_priv->uncore,
2123                                               HSW_PCODE_DE_WRITE_FREQ_REQ,
2124                                               cdclk_config->voltage_level,
2125                                               150, 2);
2126         }
2127         if (ret) {
2128                 drm_err(&dev_priv->drm,
2129                         "PCode CDCLK freq set failed, (err %d, freq %d)\n",
2130                         ret, cdclk);
2131                 return;
2132         }
2133
2134         intel_update_cdclk(dev_priv);
2135
2136         if (DISPLAY_VER(dev_priv) >= 11)
2137                 /*
2138                  * Can't read out the voltage level :(
2139                  * Let's just assume everything is as expected.
2140                  */
2141                 dev_priv->display.cdclk.hw.voltage_level = cdclk_config->voltage_level;
2142 }
2143
2144 static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
2145 {
2146         u32 cdctl, expected;
2147         int cdclk, vco;
2148
2149         intel_update_cdclk(dev_priv);
2150         intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
2151
2152         if (dev_priv->display.cdclk.hw.vco == 0 ||
2153             dev_priv->display.cdclk.hw.cdclk == dev_priv->display.cdclk.hw.bypass)
2154                 goto sanitize;
2155
2156         /* Make sure this is a legal cdclk value for the platform */
2157         cdclk = bxt_calc_cdclk(dev_priv, dev_priv->display.cdclk.hw.cdclk);
2158         if (cdclk != dev_priv->display.cdclk.hw.cdclk)
2159                 goto sanitize;
2160
2161         /* Make sure the VCO is correct for the cdclk */
2162         vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2163         if (vco != dev_priv->display.cdclk.hw.vco)
2164                 goto sanitize;
2165
2166         /*
2167          * Some BIOS versions leave an incorrect decimal frequency value and
2168          * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4,
2169          * so sanitize this register.
2170          */
2171         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
2172         expected = bxt_cdclk_ctl(dev_priv, &dev_priv->display.cdclk.hw, INVALID_PIPE);
2173
2174         /*
2175          * Let's ignore the pipe field, since BIOS could have configured the
2176          * dividers both synching to an active pipe, or asynchronously
2177          * (PIPE_NONE).
2178          */
2179         cdctl &= ~bxt_cdclk_cd2x_pipe(dev_priv, INVALID_PIPE);
2180         expected &= ~bxt_cdclk_cd2x_pipe(dev_priv, INVALID_PIPE);
2181
2182         if (cdctl == expected)
2183                 /* All well; nothing to sanitize */
2184                 return;
2185
2186 sanitize:
2187         drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
2188
2189         /* force cdclk programming */
2190         dev_priv->display.cdclk.hw.cdclk = 0;
2191
2192         /* force full PLL disable + enable */
2193         dev_priv->display.cdclk.hw.vco = ~0;
2194 }
2195
2196 static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv)
2197 {
2198         struct intel_cdclk_config cdclk_config;
2199
2200         bxt_sanitize_cdclk(dev_priv);
2201
2202         if (dev_priv->display.cdclk.hw.cdclk != 0 &&
2203             dev_priv->display.cdclk.hw.vco != 0)
2204                 return;
2205
2206         cdclk_config = dev_priv->display.cdclk.hw;
2207
2208         /*
2209          * FIXME:
2210          * - The initial CDCLK needs to be read from VBT.
2211          *   Need to make this change after VBT has changes for BXT.
2212          */
2213         cdclk_config.cdclk = bxt_calc_cdclk(dev_priv, 0);
2214         cdclk_config.vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk_config.cdclk);
2215         cdclk_config.voltage_level =
2216                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk);
2217
2218         bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
2219 }
2220
2221 static void bxt_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
2222 {
2223         struct intel_cdclk_config cdclk_config = dev_priv->display.cdclk.hw;
2224
2225         cdclk_config.cdclk = cdclk_config.bypass;
2226         cdclk_config.vco = 0;
2227         cdclk_config.voltage_level =
2228                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk);
2229
2230         bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
2231 }
2232
2233 /**
2234  * intel_cdclk_init_hw - Initialize CDCLK hardware
2235  * @i915: i915 device
2236  *
2237  * Initialize CDCLK. This consists mainly of initializing dev_priv->display.cdclk.hw and
2238  * sanitizing the state of the hardware if needed. This is generally done only
2239  * during the display core initialization sequence, after which the DMC will
2240  * take care of turning CDCLK off/on as needed.
2241  */
2242 void intel_cdclk_init_hw(struct drm_i915_private *i915)
2243 {
2244         if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
2245                 bxt_cdclk_init_hw(i915);
2246         else if (DISPLAY_VER(i915) == 9)
2247                 skl_cdclk_init_hw(i915);
2248 }
2249
2250 /**
2251  * intel_cdclk_uninit_hw - Uninitialize CDCLK hardware
2252  * @i915: i915 device
2253  *
2254  * Uninitialize CDCLK. This is done only during the display core
2255  * uninitialization sequence.
2256  */
2257 void intel_cdclk_uninit_hw(struct drm_i915_private *i915)
2258 {
2259         if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
2260                 bxt_cdclk_uninit_hw(i915);
2261         else if (DISPLAY_VER(i915) == 9)
2262                 skl_cdclk_uninit_hw(i915);
2263 }
2264
2265 static bool intel_cdclk_can_crawl_and_squash(struct drm_i915_private *i915,
2266                                              const struct intel_cdclk_config *a,
2267                                              const struct intel_cdclk_config *b)
2268 {
2269         u16 old_waveform;
2270         u16 new_waveform;
2271
2272         drm_WARN_ON(&i915->drm, cdclk_pll_is_unknown(a->vco));
2273
2274         if (a->vco == 0 || b->vco == 0)
2275                 return false;
2276
2277         if (!HAS_CDCLK_CRAWL(i915) || !HAS_CDCLK_SQUASH(i915))
2278                 return false;
2279
2280         old_waveform = cdclk_squash_waveform(i915, a->cdclk);
2281         new_waveform = cdclk_squash_waveform(i915, b->cdclk);
2282
2283         return a->vco != b->vco &&
2284                old_waveform != new_waveform;
2285 }
2286
2287 static bool intel_cdclk_can_crawl(struct drm_i915_private *dev_priv,
2288                                   const struct intel_cdclk_config *a,
2289                                   const struct intel_cdclk_config *b)
2290 {
2291         int a_div, b_div;
2292
2293         if (!HAS_CDCLK_CRAWL(dev_priv))
2294                 return false;
2295
2296         /*
2297          * The vco and cd2x divider will change independently
2298          * from each, so we disallow cd2x change when crawling.
2299          */
2300         a_div = DIV_ROUND_CLOSEST(a->vco, a->cdclk);
2301         b_div = DIV_ROUND_CLOSEST(b->vco, b->cdclk);
2302
2303         return a->vco != 0 && b->vco != 0 &&
2304                 a->vco != b->vco &&
2305                 a_div == b_div &&
2306                 a->ref == b->ref;
2307 }
2308
2309 static bool intel_cdclk_can_squash(struct drm_i915_private *dev_priv,
2310                                    const struct intel_cdclk_config *a,
2311                                    const struct intel_cdclk_config *b)
2312 {
2313         /*
2314          * FIXME should store a bit more state in intel_cdclk_config
2315          * to differentiate squasher vs. cd2x divider properly. For
2316          * the moment all platforms with squasher use a fixed cd2x
2317          * divider.
2318          */
2319         if (!HAS_CDCLK_SQUASH(dev_priv))
2320                 return false;
2321
2322         return a->cdclk != b->cdclk &&
2323                 a->vco != 0 &&
2324                 a->vco == b->vco &&
2325                 a->ref == b->ref;
2326 }
2327
2328 /**
2329  * intel_cdclk_clock_changed - Check whether the clock changed
2330  * @a: first CDCLK configuration
2331  * @b: second CDCLK configuration
2332  *
2333  * Returns:
2334  * True if CDCLK changed in a way that requires re-programming and
2335  * False otherwise.
2336  */
2337 bool intel_cdclk_clock_changed(const struct intel_cdclk_config *a,
2338                                const struct intel_cdclk_config *b)
2339 {
2340         return a->cdclk != b->cdclk ||
2341                 a->vco != b->vco ||
2342                 a->ref != b->ref;
2343 }
2344
2345 /**
2346  * intel_cdclk_can_cd2x_update - Determine if changing between the two CDCLK
2347  *                               configurations requires only a cd2x divider update
2348  * @dev_priv: i915 device
2349  * @a: first CDCLK configuration
2350  * @b: second CDCLK configuration
2351  *
2352  * Returns:
2353  * True if changing between the two CDCLK configurations
2354  * can be done with just a cd2x divider update, false if not.
2355  */
2356 static bool intel_cdclk_can_cd2x_update(struct drm_i915_private *dev_priv,
2357                                         const struct intel_cdclk_config *a,
2358                                         const struct intel_cdclk_config *b)
2359 {
2360         /* Older hw doesn't have the capability */
2361         if (DISPLAY_VER(dev_priv) < 10 && !IS_BROXTON(dev_priv))
2362                 return false;
2363
2364         /*
2365          * FIXME should store a bit more state in intel_cdclk_config
2366          * to differentiate squasher vs. cd2x divider properly. For
2367          * the moment all platforms with squasher use a fixed cd2x
2368          * divider.
2369          */
2370         if (HAS_CDCLK_SQUASH(dev_priv))
2371                 return false;
2372
2373         return a->cdclk != b->cdclk &&
2374                 a->vco != 0 &&
2375                 a->vco == b->vco &&
2376                 a->ref == b->ref;
2377 }
2378
2379 /**
2380  * intel_cdclk_changed - Determine if two CDCLK configurations are different
2381  * @a: first CDCLK configuration
2382  * @b: second CDCLK configuration
2383  *
2384  * Returns:
2385  * True if the CDCLK configurations don't match, false if they do.
2386  */
2387 static bool intel_cdclk_changed(const struct intel_cdclk_config *a,
2388                                 const struct intel_cdclk_config *b)
2389 {
2390         return intel_cdclk_clock_changed(a, b) ||
2391                 a->voltage_level != b->voltage_level;
2392 }
2393
2394 void intel_cdclk_dump_config(struct drm_i915_private *i915,
2395                              const struct intel_cdclk_config *cdclk_config,
2396                              const char *context)
2397 {
2398         drm_dbg_kms(&i915->drm, "%s %d kHz, VCO %d kHz, ref %d kHz, bypass %d kHz, voltage level %d\n",
2399                     context, cdclk_config->cdclk, cdclk_config->vco,
2400                     cdclk_config->ref, cdclk_config->bypass,
2401                     cdclk_config->voltage_level);
2402 }
2403
2404 static void intel_pcode_notify(struct drm_i915_private *i915,
2405                                u8 voltage_level,
2406                                u8 active_pipe_count,
2407                                u16 cdclk,
2408                                bool cdclk_update_valid,
2409                                bool pipe_count_update_valid)
2410 {
2411         int ret;
2412         u32 update_mask = 0;
2413
2414         if (!IS_DG2(i915))
2415                 return;
2416
2417         update_mask = DISPLAY_TO_PCODE_UPDATE_MASK(cdclk, active_pipe_count, voltage_level);
2418
2419         if (cdclk_update_valid)
2420                 update_mask |= DISPLAY_TO_PCODE_CDCLK_VALID;
2421
2422         if (pipe_count_update_valid)
2423                 update_mask |= DISPLAY_TO_PCODE_PIPE_COUNT_VALID;
2424
2425         ret = skl_pcode_request(&i915->uncore, SKL_PCODE_CDCLK_CONTROL,
2426                                 SKL_CDCLK_PREPARE_FOR_CHANGE |
2427                                 update_mask,
2428                                 SKL_CDCLK_READY_FOR_CHANGE,
2429                                 SKL_CDCLK_READY_FOR_CHANGE, 3);
2430         if (ret)
2431                 drm_err(&i915->drm,
2432                         "Failed to inform PCU about display config (err %d)\n",
2433                         ret);
2434 }
2435
2436 static void intel_set_cdclk(struct drm_i915_private *dev_priv,
2437                             const struct intel_cdclk_config *cdclk_config,
2438                             enum pipe pipe, const char *context)
2439 {
2440         struct intel_encoder *encoder;
2441
2442         if (!intel_cdclk_changed(&dev_priv->display.cdclk.hw, cdclk_config))
2443                 return;
2444
2445         if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->display.funcs.cdclk->set_cdclk))
2446                 return;
2447
2448         intel_cdclk_dump_config(dev_priv, cdclk_config, context);
2449
2450         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2451                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2452
2453                 intel_psr_pause(intel_dp);
2454         }
2455
2456         intel_audio_cdclk_change_pre(dev_priv);
2457
2458         /*
2459          * Lock aux/gmbus while we change cdclk in case those
2460          * functions use cdclk. Not all platforms/ports do,
2461          * but we'll lock them all for simplicity.
2462          */
2463         mutex_lock(&dev_priv->display.gmbus.mutex);
2464         for_each_intel_dp(&dev_priv->drm, encoder) {
2465                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2466
2467                 mutex_lock_nest_lock(&intel_dp->aux.hw_mutex,
2468                                      &dev_priv->display.gmbus.mutex);
2469         }
2470
2471         intel_cdclk_set_cdclk(dev_priv, cdclk_config, pipe);
2472
2473         for_each_intel_dp(&dev_priv->drm, encoder) {
2474                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2475
2476                 mutex_unlock(&intel_dp->aux.hw_mutex);
2477         }
2478         mutex_unlock(&dev_priv->display.gmbus.mutex);
2479
2480         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2481                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2482
2483                 intel_psr_resume(intel_dp);
2484         }
2485
2486         intel_audio_cdclk_change_post(dev_priv);
2487
2488         if (drm_WARN(&dev_priv->drm,
2489                      intel_cdclk_changed(&dev_priv->display.cdclk.hw, cdclk_config),
2490                      "cdclk state doesn't match!\n")) {
2491                 intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "[hw state]");
2492                 intel_cdclk_dump_config(dev_priv, cdclk_config, "[sw state]");
2493         }
2494 }
2495
2496 static void intel_cdclk_pcode_pre_notify(struct intel_atomic_state *state)
2497 {
2498         struct drm_i915_private *i915 = to_i915(state->base.dev);
2499         const struct intel_cdclk_state *old_cdclk_state =
2500                 intel_atomic_get_old_cdclk_state(state);
2501         const struct intel_cdclk_state *new_cdclk_state =
2502                 intel_atomic_get_new_cdclk_state(state);
2503         unsigned int cdclk = 0; u8 voltage_level, num_active_pipes = 0;
2504         bool change_cdclk, update_pipe_count;
2505
2506         if (!intel_cdclk_changed(&old_cdclk_state->actual,
2507                                  &new_cdclk_state->actual) &&
2508                                  new_cdclk_state->active_pipes ==
2509                                  old_cdclk_state->active_pipes)
2510                 return;
2511
2512         /* According to "Sequence Before Frequency Change", voltage level set to 0x3 */
2513         voltage_level = DISPLAY_TO_PCODE_VOLTAGE_MAX;
2514
2515         change_cdclk = new_cdclk_state->actual.cdclk != old_cdclk_state->actual.cdclk;
2516         update_pipe_count = hweight8(new_cdclk_state->active_pipes) >
2517                             hweight8(old_cdclk_state->active_pipes);
2518
2519         /*
2520          * According to "Sequence Before Frequency Change",
2521          * if CDCLK is increasing, set bits 25:16 to upcoming CDCLK,
2522          * if CDCLK is decreasing or not changing, set bits 25:16 to current CDCLK,
2523          * which basically means we choose the maximum of old and new CDCLK, if we know both
2524          */
2525         if (change_cdclk)
2526                 cdclk = max(new_cdclk_state->actual.cdclk, old_cdclk_state->actual.cdclk);
2527
2528         /*
2529          * According to "Sequence For Pipe Count Change",
2530          * if pipe count is increasing, set bits 25:16 to upcoming pipe count
2531          * (power well is enabled)
2532          * no action if it is decreasing, before the change
2533          */
2534         if (update_pipe_count)
2535                 num_active_pipes = hweight8(new_cdclk_state->active_pipes);
2536
2537         intel_pcode_notify(i915, voltage_level, num_active_pipes, cdclk,
2538                            change_cdclk, update_pipe_count);
2539 }
2540
2541 static void intel_cdclk_pcode_post_notify(struct intel_atomic_state *state)
2542 {
2543         struct drm_i915_private *i915 = to_i915(state->base.dev);
2544         const struct intel_cdclk_state *new_cdclk_state =
2545                 intel_atomic_get_new_cdclk_state(state);
2546         const struct intel_cdclk_state *old_cdclk_state =
2547                 intel_atomic_get_old_cdclk_state(state);
2548         unsigned int cdclk = 0; u8 voltage_level, num_active_pipes = 0;
2549         bool update_cdclk, update_pipe_count;
2550
2551         /* According to "Sequence After Frequency Change", set voltage to used level */
2552         voltage_level = new_cdclk_state->actual.voltage_level;
2553
2554         update_cdclk = new_cdclk_state->actual.cdclk != old_cdclk_state->actual.cdclk;
2555         update_pipe_count = hweight8(new_cdclk_state->active_pipes) <
2556                             hweight8(old_cdclk_state->active_pipes);
2557
2558         /*
2559          * According to "Sequence After Frequency Change",
2560          * set bits 25:16 to current CDCLK
2561          */
2562         if (update_cdclk)
2563                 cdclk = new_cdclk_state->actual.cdclk;
2564
2565         /*
2566          * According to "Sequence For Pipe Count Change",
2567          * if pipe count is decreasing, set bits 25:16 to current pipe count,
2568          * after the change(power well is disabled)
2569          * no action if it is increasing, after the change
2570          */
2571         if (update_pipe_count)
2572                 num_active_pipes = hweight8(new_cdclk_state->active_pipes);
2573
2574         intel_pcode_notify(i915, voltage_level, num_active_pipes, cdclk,
2575                            update_cdclk, update_pipe_count);
2576 }
2577
2578 bool intel_cdclk_is_decreasing_later(struct intel_atomic_state *state)
2579 {
2580         const struct intel_cdclk_state *old_cdclk_state =
2581                 intel_atomic_get_old_cdclk_state(state);
2582         const struct intel_cdclk_state *new_cdclk_state =
2583                 intel_atomic_get_new_cdclk_state(state);
2584
2585         return new_cdclk_state && !new_cdclk_state->disable_pipes &&
2586                 new_cdclk_state->actual.cdclk < old_cdclk_state->actual.cdclk;
2587 }
2588
2589 /**
2590  * intel_set_cdclk_pre_plane_update - Push the CDCLK state to the hardware
2591  * @state: intel atomic state
2592  *
2593  * Program the hardware before updating the HW plane state based on the
2594  * new CDCLK state, if necessary.
2595  */
2596 void
2597 intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state)
2598 {
2599         struct drm_i915_private *i915 = to_i915(state->base.dev);
2600         const struct intel_cdclk_state *old_cdclk_state =
2601                 intel_atomic_get_old_cdclk_state(state);
2602         const struct intel_cdclk_state *new_cdclk_state =
2603                 intel_atomic_get_new_cdclk_state(state);
2604         struct intel_cdclk_config cdclk_config;
2605         enum pipe pipe;
2606
2607         if (!intel_cdclk_changed(&old_cdclk_state->actual,
2608                                  &new_cdclk_state->actual))
2609                 return;
2610
2611         if (IS_DG2(i915))
2612                 intel_cdclk_pcode_pre_notify(state);
2613
2614         if (new_cdclk_state->disable_pipes) {
2615                 cdclk_config = new_cdclk_state->actual;
2616                 pipe = INVALID_PIPE;
2617         } else {
2618                 if (new_cdclk_state->actual.cdclk >= old_cdclk_state->actual.cdclk) {
2619                         cdclk_config = new_cdclk_state->actual;
2620                         pipe = new_cdclk_state->pipe;
2621                 } else {
2622                         cdclk_config = old_cdclk_state->actual;
2623                         pipe = INVALID_PIPE;
2624                 }
2625
2626                 cdclk_config.voltage_level = max(new_cdclk_state->actual.voltage_level,
2627                                                  old_cdclk_state->actual.voltage_level);
2628         }
2629
2630         /*
2631          * mbus joining will be changed later by
2632          * intel_dbuf_mbus_{pre,post}_ddb_update()
2633          */
2634         cdclk_config.joined_mbus = old_cdclk_state->actual.joined_mbus;
2635
2636         drm_WARN_ON(&i915->drm, !new_cdclk_state->base.changed);
2637
2638         intel_set_cdclk(i915, &cdclk_config, pipe,
2639                         "Pre changing CDCLK to");
2640 }
2641
2642 /**
2643  * intel_set_cdclk_post_plane_update - Push the CDCLK state to the hardware
2644  * @state: intel atomic state
2645  *
2646  * Program the hardware after updating the HW plane state based on the
2647  * new CDCLK state, if necessary.
2648  */
2649 void
2650 intel_set_cdclk_post_plane_update(struct intel_atomic_state *state)
2651 {
2652         struct drm_i915_private *i915 = to_i915(state->base.dev);
2653         const struct intel_cdclk_state *old_cdclk_state =
2654                 intel_atomic_get_old_cdclk_state(state);
2655         const struct intel_cdclk_state *new_cdclk_state =
2656                 intel_atomic_get_new_cdclk_state(state);
2657         enum pipe pipe;
2658
2659         if (!intel_cdclk_changed(&old_cdclk_state->actual,
2660                                  &new_cdclk_state->actual))
2661                 return;
2662
2663         if (IS_DG2(i915))
2664                 intel_cdclk_pcode_post_notify(state);
2665
2666         if (!new_cdclk_state->disable_pipes &&
2667             new_cdclk_state->actual.cdclk < old_cdclk_state->actual.cdclk)
2668                 pipe = new_cdclk_state->pipe;
2669         else
2670                 pipe = INVALID_PIPE;
2671
2672         drm_WARN_ON(&i915->drm, !new_cdclk_state->base.changed);
2673
2674         intel_set_cdclk(i915, &new_cdclk_state->actual, pipe,
2675                         "Post changing CDCLK to");
2676 }
2677
2678 static int intel_pixel_rate_to_cdclk(const struct intel_crtc_state *crtc_state)
2679 {
2680         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2681         int pixel_rate = crtc_state->pixel_rate;
2682
2683         if (DISPLAY_VER(dev_priv) >= 10)
2684                 return DIV_ROUND_UP(pixel_rate, 2);
2685         else if (DISPLAY_VER(dev_priv) == 9 ||
2686                  IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2687                 return pixel_rate;
2688         else if (IS_CHERRYVIEW(dev_priv))
2689                 return DIV_ROUND_UP(pixel_rate * 100, 95);
2690         else if (crtc_state->double_wide)
2691                 return DIV_ROUND_UP(pixel_rate * 100, 90 * 2);
2692         else
2693                 return DIV_ROUND_UP(pixel_rate * 100, 90);
2694 }
2695
2696 static int intel_planes_min_cdclk(const struct intel_crtc_state *crtc_state)
2697 {
2698         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2699         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2700         struct intel_plane *plane;
2701         int min_cdclk = 0;
2702
2703         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
2704                 min_cdclk = max(crtc_state->min_cdclk[plane->id], min_cdclk);
2705
2706         return min_cdclk;
2707 }
2708
2709 static int intel_vdsc_min_cdclk(const struct intel_crtc_state *crtc_state)
2710 {
2711         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2712         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2713         int num_vdsc_instances = intel_dsc_get_num_vdsc_instances(crtc_state);
2714         int min_cdclk = 0;
2715
2716         /*
2717          * When we decide to use only one VDSC engine, since
2718          * each VDSC operates with 1 ppc throughput, pixel clock
2719          * cannot be higher than the VDSC clock (cdclk)
2720          * If there 2 VDSC engines, then pixel clock can't be higher than
2721          * VDSC clock(cdclk) * 2 and so on.
2722          */
2723         min_cdclk = max_t(int, min_cdclk,
2724                           DIV_ROUND_UP(crtc_state->pixel_rate, num_vdsc_instances));
2725
2726         if (crtc_state->bigjoiner_pipes) {
2727                 int pixel_clock = intel_dp_mode_to_fec_clock(crtc_state->hw.adjusted_mode.clock);
2728
2729                 /*
2730                  * According to Bigjoiner bw check:
2731                  * compressed_bpp <= PPC * CDCLK * Big joiner Interface bits / Pixel clock
2732                  *
2733                  * We have already computed compressed_bpp, so now compute the min CDCLK that
2734                  * is required to support this compressed_bpp.
2735                  *
2736                  * => CDCLK >= compressed_bpp * Pixel clock / (PPC * Bigjoiner Interface bits)
2737                  *
2738                  * Since PPC = 2 with bigjoiner
2739                  * => CDCLK >= compressed_bpp * Pixel clock  / 2 * Bigjoiner Interface bits
2740                  */
2741                 int bigjoiner_interface_bits = DISPLAY_VER(i915) >= 14 ? 36 : 24;
2742                 int min_cdclk_bj =
2743                         (to_bpp_int_roundup(crtc_state->dsc.compressed_bpp_x16) *
2744                          pixel_clock) / (2 * bigjoiner_interface_bits);
2745
2746                 min_cdclk = max(min_cdclk, min_cdclk_bj);
2747         }
2748
2749         return min_cdclk;
2750 }
2751
2752 int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
2753 {
2754         struct drm_i915_private *dev_priv =
2755                 to_i915(crtc_state->uapi.crtc->dev);
2756         int min_cdclk;
2757
2758         if (!crtc_state->hw.enable)
2759                 return 0;
2760
2761         min_cdclk = intel_pixel_rate_to_cdclk(crtc_state);
2762
2763         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
2764         if (IS_BROADWELL(dev_priv) && hsw_crtc_state_ips_capable(crtc_state))
2765                 min_cdclk = DIV_ROUND_UP(min_cdclk * 100, 95);
2766
2767         /* BSpec says "Do not use DisplayPort with CDCLK less than 432 MHz,
2768          * audio enabled, port width x4, and link rate HBR2 (5.4 GHz), or else
2769          * there may be audio corruption or screen corruption." This cdclk
2770          * restriction for GLK is 316.8 MHz.
2771          */
2772         if (intel_crtc_has_dp_encoder(crtc_state) &&
2773             crtc_state->has_audio &&
2774             crtc_state->port_clock >= 540000 &&
2775             crtc_state->lane_count == 4) {
2776                 if (DISPLAY_VER(dev_priv) == 10) {
2777                         /* Display WA #1145: glk */
2778                         min_cdclk = max(316800, min_cdclk);
2779                 } else if (DISPLAY_VER(dev_priv) == 9 || IS_BROADWELL(dev_priv)) {
2780                         /* Display WA #1144: skl,bxt */
2781                         min_cdclk = max(432000, min_cdclk);
2782                 }
2783         }
2784
2785         /*
2786          * According to BSpec, "The CD clock frequency must be at least twice
2787          * the frequency of the Azalia BCLK." and BCLK is 96 MHz by default.
2788          */
2789         if (crtc_state->has_audio && DISPLAY_VER(dev_priv) >= 9)
2790                 min_cdclk = max(2 * 96000, min_cdclk);
2791
2792         /*
2793          * "For DP audio configuration, cdclk frequency shall be set to
2794          *  meet the following requirements:
2795          *  DP Link Frequency(MHz) | Cdclk frequency(MHz)
2796          *  270                    | 320 or higher
2797          *  162                    | 200 or higher"
2798          */
2799         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
2800             intel_crtc_has_dp_encoder(crtc_state) && crtc_state->has_audio)
2801                 min_cdclk = max(crtc_state->port_clock, min_cdclk);
2802
2803         /*
2804          * On Valleyview some DSI panels lose (v|h)sync when the clock is lower
2805          * than 320000KHz.
2806          */
2807         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2808             IS_VALLEYVIEW(dev_priv))
2809                 min_cdclk = max(320000, min_cdclk);
2810
2811         /*
2812          * On Geminilake once the CDCLK gets as low as 79200
2813          * picture gets unstable, despite that values are
2814          * correct for DSI PLL and DE PLL.
2815          */
2816         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2817             IS_GEMINILAKE(dev_priv))
2818                 min_cdclk = max(158400, min_cdclk);
2819
2820         /* Account for additional needs from the planes */
2821         min_cdclk = max(intel_planes_min_cdclk(crtc_state), min_cdclk);
2822
2823         if (crtc_state->dsc.compression_enable)
2824                 min_cdclk = max(min_cdclk, intel_vdsc_min_cdclk(crtc_state));
2825
2826         return min_cdclk;
2827 }
2828
2829 static int intel_compute_min_cdclk(struct intel_cdclk_state *cdclk_state)
2830 {
2831         struct intel_atomic_state *state = cdclk_state->base.state;
2832         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2833         const struct intel_bw_state *bw_state;
2834         struct intel_crtc *crtc;
2835         struct intel_crtc_state *crtc_state;
2836         int min_cdclk, i;
2837         enum pipe pipe;
2838
2839         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2840                 int ret;
2841
2842                 min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
2843                 if (min_cdclk < 0)
2844                         return min_cdclk;
2845
2846                 if (cdclk_state->min_cdclk[crtc->pipe] == min_cdclk)
2847                         continue;
2848
2849                 cdclk_state->min_cdclk[crtc->pipe] = min_cdclk;
2850
2851                 ret = intel_atomic_lock_global_state(&cdclk_state->base);
2852                 if (ret)
2853                         return ret;
2854         }
2855
2856         bw_state = intel_atomic_get_new_bw_state(state);
2857         if (bw_state) {
2858                 min_cdclk = intel_bw_min_cdclk(dev_priv, bw_state);
2859
2860                 if (cdclk_state->bw_min_cdclk != min_cdclk) {
2861                         int ret;
2862
2863                         cdclk_state->bw_min_cdclk = min_cdclk;
2864
2865                         ret = intel_atomic_lock_global_state(&cdclk_state->base);
2866                         if (ret)
2867                                 return ret;
2868                 }
2869         }
2870
2871         min_cdclk = max(cdclk_state->force_min_cdclk,
2872                         cdclk_state->bw_min_cdclk);
2873         for_each_pipe(dev_priv, pipe)
2874                 min_cdclk = max(cdclk_state->min_cdclk[pipe], min_cdclk);
2875
2876         /*
2877          * Avoid glk_force_audio_cdclk() causing excessive screen
2878          * blinking when multiple pipes are active by making sure
2879          * CDCLK frequency is always high enough for audio. With a
2880          * single active pipe we can always change CDCLK frequency
2881          * by changing the cd2x divider (see glk_cdclk_table[]) and
2882          * thus a full modeset won't be needed then.
2883          */
2884         if (IS_GEMINILAKE(dev_priv) && cdclk_state->active_pipes &&
2885             !is_power_of_2(cdclk_state->active_pipes))
2886                 min_cdclk = max(2 * 96000, min_cdclk);
2887
2888         if (min_cdclk > dev_priv->display.cdclk.max_cdclk_freq) {
2889                 drm_dbg_kms(&dev_priv->drm,
2890                             "required cdclk (%d kHz) exceeds max (%d kHz)\n",
2891                             min_cdclk, dev_priv->display.cdclk.max_cdclk_freq);
2892                 return -EINVAL;
2893         }
2894
2895         return min_cdclk;
2896 }
2897
2898 /*
2899  * Account for port clock min voltage level requirements.
2900  * This only really does something on DISPLA_VER >= 11 but can be
2901  * called on earlier platforms as well.
2902  *
2903  * Note that this functions assumes that 0 is
2904  * the lowest voltage value, and higher values
2905  * correspond to increasingly higher voltages.
2906  *
2907  * Should that relationship no longer hold on
2908  * future platforms this code will need to be
2909  * adjusted.
2910  */
2911 static int bxt_compute_min_voltage_level(struct intel_cdclk_state *cdclk_state)
2912 {
2913         struct intel_atomic_state *state = cdclk_state->base.state;
2914         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2915         struct intel_crtc *crtc;
2916         struct intel_crtc_state *crtc_state;
2917         u8 min_voltage_level;
2918         int i;
2919         enum pipe pipe;
2920
2921         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2922                 int ret;
2923
2924                 if (crtc_state->hw.enable)
2925                         min_voltage_level = crtc_state->min_voltage_level;
2926                 else
2927                         min_voltage_level = 0;
2928
2929                 if (cdclk_state->min_voltage_level[crtc->pipe] == min_voltage_level)
2930                         continue;
2931
2932                 cdclk_state->min_voltage_level[crtc->pipe] = min_voltage_level;
2933
2934                 ret = intel_atomic_lock_global_state(&cdclk_state->base);
2935                 if (ret)
2936                         return ret;
2937         }
2938
2939         min_voltage_level = 0;
2940         for_each_pipe(dev_priv, pipe)
2941                 min_voltage_level = max(cdclk_state->min_voltage_level[pipe],
2942                                         min_voltage_level);
2943
2944         return min_voltage_level;
2945 }
2946
2947 static int vlv_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2948 {
2949         struct intel_atomic_state *state = cdclk_state->base.state;
2950         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2951         int min_cdclk, cdclk;
2952
2953         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2954         if (min_cdclk < 0)
2955                 return min_cdclk;
2956
2957         cdclk = vlv_calc_cdclk(dev_priv, min_cdclk);
2958
2959         cdclk_state->logical.cdclk = cdclk;
2960         cdclk_state->logical.voltage_level =
2961                 vlv_calc_voltage_level(dev_priv, cdclk);
2962
2963         if (!cdclk_state->active_pipes) {
2964                 cdclk = vlv_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2965
2966                 cdclk_state->actual.cdclk = cdclk;
2967                 cdclk_state->actual.voltage_level =
2968                         vlv_calc_voltage_level(dev_priv, cdclk);
2969         } else {
2970                 cdclk_state->actual = cdclk_state->logical;
2971         }
2972
2973         return 0;
2974 }
2975
2976 static int bdw_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2977 {
2978         int min_cdclk, cdclk;
2979
2980         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2981         if (min_cdclk < 0)
2982                 return min_cdclk;
2983
2984         cdclk = bdw_calc_cdclk(min_cdclk);
2985
2986         cdclk_state->logical.cdclk = cdclk;
2987         cdclk_state->logical.voltage_level =
2988                 bdw_calc_voltage_level(cdclk);
2989
2990         if (!cdclk_state->active_pipes) {
2991                 cdclk = bdw_calc_cdclk(cdclk_state->force_min_cdclk);
2992
2993                 cdclk_state->actual.cdclk = cdclk;
2994                 cdclk_state->actual.voltage_level =
2995                         bdw_calc_voltage_level(cdclk);
2996         } else {
2997                 cdclk_state->actual = cdclk_state->logical;
2998         }
2999
3000         return 0;
3001 }
3002
3003 static int skl_dpll0_vco(struct intel_cdclk_state *cdclk_state)
3004 {
3005         struct intel_atomic_state *state = cdclk_state->base.state;
3006         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3007         struct intel_crtc *crtc;
3008         struct intel_crtc_state *crtc_state;
3009         int vco, i;
3010
3011         vco = cdclk_state->logical.vco;
3012         if (!vco)
3013                 vco = dev_priv->display.cdclk.skl_preferred_vco_freq;
3014
3015         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
3016                 if (!crtc_state->hw.enable)
3017                         continue;
3018
3019                 if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
3020                         continue;
3021
3022                 /*
3023                  * DPLL0 VCO may need to be adjusted to get the correct
3024                  * clock for eDP. This will affect cdclk as well.
3025                  */
3026                 switch (crtc_state->port_clock / 2) {
3027                 case 108000:
3028                 case 216000:
3029                         vco = 8640000;
3030                         break;
3031                 default:
3032                         vco = 8100000;
3033                         break;
3034                 }
3035         }
3036
3037         return vco;
3038 }
3039
3040 static int skl_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
3041 {
3042         int min_cdclk, cdclk, vco;
3043
3044         min_cdclk = intel_compute_min_cdclk(cdclk_state);
3045         if (min_cdclk < 0)
3046                 return min_cdclk;
3047
3048         vco = skl_dpll0_vco(cdclk_state);
3049
3050         cdclk = skl_calc_cdclk(min_cdclk, vco);
3051
3052         cdclk_state->logical.vco = vco;
3053         cdclk_state->logical.cdclk = cdclk;
3054         cdclk_state->logical.voltage_level =
3055                 skl_calc_voltage_level(cdclk);
3056
3057         if (!cdclk_state->active_pipes) {
3058                 cdclk = skl_calc_cdclk(cdclk_state->force_min_cdclk, vco);
3059
3060                 cdclk_state->actual.vco = vco;
3061                 cdclk_state->actual.cdclk = cdclk;
3062                 cdclk_state->actual.voltage_level =
3063                         skl_calc_voltage_level(cdclk);
3064         } else {
3065                 cdclk_state->actual = cdclk_state->logical;
3066         }
3067
3068         return 0;
3069 }
3070
3071 static int bxt_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
3072 {
3073         struct intel_atomic_state *state = cdclk_state->base.state;
3074         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3075         int min_cdclk, min_voltage_level, cdclk, vco;
3076
3077         min_cdclk = intel_compute_min_cdclk(cdclk_state);
3078         if (min_cdclk < 0)
3079                 return min_cdclk;
3080
3081         min_voltage_level = bxt_compute_min_voltage_level(cdclk_state);
3082         if (min_voltage_level < 0)
3083                 return min_voltage_level;
3084
3085         cdclk = bxt_calc_cdclk(dev_priv, min_cdclk);
3086         vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
3087
3088         cdclk_state->logical.vco = vco;
3089         cdclk_state->logical.cdclk = cdclk;
3090         cdclk_state->logical.voltage_level =
3091                 max_t(int, min_voltage_level,
3092                       intel_cdclk_calc_voltage_level(dev_priv, cdclk));
3093
3094         if (!cdclk_state->active_pipes) {
3095                 cdclk = bxt_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
3096                 vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
3097
3098                 cdclk_state->actual.vco = vco;
3099                 cdclk_state->actual.cdclk = cdclk;
3100                 cdclk_state->actual.voltage_level =
3101                         intel_cdclk_calc_voltage_level(dev_priv, cdclk);
3102         } else {
3103                 cdclk_state->actual = cdclk_state->logical;
3104         }
3105
3106         return 0;
3107 }
3108
3109 static int fixed_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
3110 {
3111         int min_cdclk;
3112
3113         /*
3114          * We can't change the cdclk frequency, but we still want to
3115          * check that the required minimum frequency doesn't exceed
3116          * the actual cdclk frequency.
3117          */
3118         min_cdclk = intel_compute_min_cdclk(cdclk_state);
3119         if (min_cdclk < 0)
3120                 return min_cdclk;
3121
3122         return 0;
3123 }
3124
3125 static struct intel_global_state *intel_cdclk_duplicate_state(struct intel_global_obj *obj)
3126 {
3127         struct intel_cdclk_state *cdclk_state;
3128
3129         cdclk_state = kmemdup(obj->state, sizeof(*cdclk_state), GFP_KERNEL);
3130         if (!cdclk_state)
3131                 return NULL;
3132
3133         cdclk_state->pipe = INVALID_PIPE;
3134         cdclk_state->disable_pipes = false;
3135
3136         return &cdclk_state->base;
3137 }
3138
3139 static void intel_cdclk_destroy_state(struct intel_global_obj *obj,
3140                                       struct intel_global_state *state)
3141 {
3142         kfree(state);
3143 }
3144
3145 static const struct intel_global_state_funcs intel_cdclk_funcs = {
3146         .atomic_duplicate_state = intel_cdclk_duplicate_state,
3147         .atomic_destroy_state = intel_cdclk_destroy_state,
3148 };
3149
3150 struct intel_cdclk_state *
3151 intel_atomic_get_cdclk_state(struct intel_atomic_state *state)
3152 {
3153         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3154         struct intel_global_state *cdclk_state;
3155
3156         cdclk_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.cdclk.obj);
3157         if (IS_ERR(cdclk_state))
3158                 return ERR_CAST(cdclk_state);
3159
3160         return to_intel_cdclk_state(cdclk_state);
3161 }
3162
3163 int intel_cdclk_atomic_check(struct intel_atomic_state *state,
3164                              bool *need_cdclk_calc)
3165 {
3166         const struct intel_cdclk_state *old_cdclk_state;
3167         const struct intel_cdclk_state *new_cdclk_state;
3168         struct intel_plane_state __maybe_unused *plane_state;
3169         struct intel_plane *plane;
3170         int ret;
3171         int i;
3172
3173         /*
3174          * active_planes bitmask has been updated, and potentially affected
3175          * planes are part of the state. We can now compute the minimum cdclk
3176          * for each plane.
3177          */
3178         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
3179                 ret = intel_plane_calc_min_cdclk(state, plane, need_cdclk_calc);
3180                 if (ret)
3181                         return ret;
3182         }
3183
3184         ret = intel_bw_calc_min_cdclk(state, need_cdclk_calc);
3185         if (ret)
3186                 return ret;
3187
3188         old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
3189         new_cdclk_state = intel_atomic_get_new_cdclk_state(state);
3190
3191         if (new_cdclk_state &&
3192             old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk)
3193                 *need_cdclk_calc = true;
3194
3195         return 0;
3196 }
3197
3198 int intel_cdclk_state_set_joined_mbus(struct intel_atomic_state *state, bool joined_mbus)
3199 {
3200         struct intel_cdclk_state *cdclk_state;
3201
3202         cdclk_state = intel_atomic_get_cdclk_state(state);
3203         if (IS_ERR(cdclk_state))
3204                 return PTR_ERR(cdclk_state);
3205
3206         cdclk_state->actual.joined_mbus = joined_mbus;
3207         cdclk_state->logical.joined_mbus = joined_mbus;
3208
3209         return intel_atomic_lock_global_state(&cdclk_state->base);
3210 }
3211
3212 int intel_cdclk_init(struct drm_i915_private *dev_priv)
3213 {
3214         struct intel_cdclk_state *cdclk_state;
3215
3216         cdclk_state = kzalloc(sizeof(*cdclk_state), GFP_KERNEL);
3217         if (!cdclk_state)
3218                 return -ENOMEM;
3219
3220         intel_atomic_global_obj_init(dev_priv, &dev_priv->display.cdclk.obj,
3221                                      &cdclk_state->base, &intel_cdclk_funcs);
3222
3223         return 0;
3224 }
3225
3226 static bool intel_cdclk_need_serialize(struct drm_i915_private *i915,
3227                                        const struct intel_cdclk_state *old_cdclk_state,
3228                                        const struct intel_cdclk_state *new_cdclk_state)
3229 {
3230         bool power_well_cnt_changed = hweight8(old_cdclk_state->active_pipes) !=
3231                                       hweight8(new_cdclk_state->active_pipes);
3232         bool cdclk_changed = intel_cdclk_changed(&old_cdclk_state->actual,
3233                                                  &new_cdclk_state->actual);
3234         /*
3235          * We need to poke hw for gen >= 12, because we notify PCode if
3236          * pipe power well count changes.
3237          */
3238         return cdclk_changed || (IS_DG2(i915) && power_well_cnt_changed);
3239 }
3240
3241 int intel_modeset_calc_cdclk(struct intel_atomic_state *state)
3242 {
3243         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3244         const struct intel_cdclk_state *old_cdclk_state;
3245         struct intel_cdclk_state *new_cdclk_state;
3246         enum pipe pipe = INVALID_PIPE;
3247         int ret;
3248
3249         new_cdclk_state = intel_atomic_get_cdclk_state(state);
3250         if (IS_ERR(new_cdclk_state))
3251                 return PTR_ERR(new_cdclk_state);
3252
3253         old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
3254
3255         new_cdclk_state->active_pipes =
3256                 intel_calc_active_pipes(state, old_cdclk_state->active_pipes);
3257
3258         ret = intel_cdclk_modeset_calc_cdclk(dev_priv, new_cdclk_state);
3259         if (ret)
3260                 return ret;
3261
3262         if (intel_cdclk_need_serialize(dev_priv, old_cdclk_state, new_cdclk_state)) {
3263                 /*
3264                  * Also serialize commits across all crtcs
3265                  * if the actual hw needs to be poked.
3266                  */
3267                 ret = intel_atomic_serialize_global_state(&new_cdclk_state->base);
3268                 if (ret)
3269                         return ret;
3270         } else if (old_cdclk_state->active_pipes != new_cdclk_state->active_pipes ||
3271                    old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk ||
3272                    intel_cdclk_changed(&old_cdclk_state->logical,
3273                                        &new_cdclk_state->logical)) {
3274                 ret = intel_atomic_lock_global_state(&new_cdclk_state->base);
3275                 if (ret)
3276                         return ret;
3277         } else {
3278                 return 0;
3279         }
3280
3281         if (is_power_of_2(new_cdclk_state->active_pipes) &&
3282             intel_cdclk_can_cd2x_update(dev_priv,
3283                                         &old_cdclk_state->actual,
3284                                         &new_cdclk_state->actual)) {
3285                 struct intel_crtc *crtc;
3286                 struct intel_crtc_state *crtc_state;
3287
3288                 pipe = ilog2(new_cdclk_state->active_pipes);
3289                 crtc = intel_crtc_for_pipe(dev_priv, pipe);
3290
3291                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
3292                 if (IS_ERR(crtc_state))
3293                         return PTR_ERR(crtc_state);
3294
3295                 if (intel_crtc_needs_modeset(crtc_state))
3296                         pipe = INVALID_PIPE;
3297         }
3298
3299         if (intel_cdclk_can_crawl_and_squash(dev_priv,
3300                                              &old_cdclk_state->actual,
3301                                              &new_cdclk_state->actual)) {
3302                 drm_dbg_kms(&dev_priv->drm,
3303                             "Can change cdclk via crawling and squashing\n");
3304         } else if (intel_cdclk_can_squash(dev_priv,
3305                                         &old_cdclk_state->actual,
3306                                         &new_cdclk_state->actual)) {
3307                 drm_dbg_kms(&dev_priv->drm,
3308                             "Can change cdclk via squashing\n");
3309         } else if (intel_cdclk_can_crawl(dev_priv,
3310                                          &old_cdclk_state->actual,
3311                                          &new_cdclk_state->actual)) {
3312                 drm_dbg_kms(&dev_priv->drm,
3313                             "Can change cdclk via crawling\n");
3314         } else if (pipe != INVALID_PIPE) {
3315                 new_cdclk_state->pipe = pipe;
3316
3317                 drm_dbg_kms(&dev_priv->drm,
3318                             "Can change cdclk cd2x divider with pipe %c active\n",
3319                             pipe_name(pipe));
3320         } else if (intel_cdclk_clock_changed(&old_cdclk_state->actual,
3321                                              &new_cdclk_state->actual)) {
3322                 /* All pipes must be switched off while we change the cdclk. */
3323                 ret = intel_modeset_all_pipes_late(state, "CDCLK change");
3324                 if (ret)
3325                         return ret;
3326
3327                 new_cdclk_state->disable_pipes = true;
3328
3329                 drm_dbg_kms(&dev_priv->drm,
3330                             "Modeset required for cdclk change\n");
3331         }
3332
3333         if (intel_mdclk_cdclk_ratio(dev_priv, &old_cdclk_state->actual) !=
3334             intel_mdclk_cdclk_ratio(dev_priv, &new_cdclk_state->actual)) {
3335                 int ratio = intel_mdclk_cdclk_ratio(dev_priv, &new_cdclk_state->actual);
3336
3337                 ret = intel_dbuf_state_set_mdclk_cdclk_ratio(state, ratio);
3338                 if (ret)
3339                         return ret;
3340         }
3341
3342         drm_dbg_kms(&dev_priv->drm,
3343                     "New cdclk calculated to be logical %u kHz, actual %u kHz\n",
3344                     new_cdclk_state->logical.cdclk,
3345                     new_cdclk_state->actual.cdclk);
3346         drm_dbg_kms(&dev_priv->drm,
3347                     "New voltage level calculated to be logical %u, actual %u\n",
3348                     new_cdclk_state->logical.voltage_level,
3349                     new_cdclk_state->actual.voltage_level);
3350
3351         return 0;
3352 }
3353
3354 static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
3355 {
3356         int max_cdclk_freq = dev_priv->display.cdclk.max_cdclk_freq;
3357
3358         if (DISPLAY_VER(dev_priv) >= 10)
3359                 return 2 * max_cdclk_freq;
3360         else if (DISPLAY_VER(dev_priv) == 9 ||
3361                  IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
3362                 return max_cdclk_freq;
3363         else if (IS_CHERRYVIEW(dev_priv))
3364                 return max_cdclk_freq*95/100;
3365         else if (DISPLAY_VER(dev_priv) < 4)
3366                 return 2*max_cdclk_freq*90/100;
3367         else
3368                 return max_cdclk_freq*90/100;
3369 }
3370
3371 /**
3372  * intel_update_max_cdclk - Determine the maximum support CDCLK frequency
3373  * @dev_priv: i915 device
3374  *
3375  * Determine the maximum CDCLK frequency the platform supports, and also
3376  * derive the maximum dot clock frequency the maximum CDCLK frequency
3377  * allows.
3378  */
3379 void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
3380 {
3381         if (IS_JASPERLAKE(dev_priv) || IS_ELKHARTLAKE(dev_priv)) {
3382                 if (dev_priv->display.cdclk.hw.ref == 24000)
3383                         dev_priv->display.cdclk.max_cdclk_freq = 552000;
3384                 else
3385                         dev_priv->display.cdclk.max_cdclk_freq = 556800;
3386         } else if (DISPLAY_VER(dev_priv) >= 11) {
3387                 if (dev_priv->display.cdclk.hw.ref == 24000)
3388                         dev_priv->display.cdclk.max_cdclk_freq = 648000;
3389                 else
3390                         dev_priv->display.cdclk.max_cdclk_freq = 652800;
3391         } else if (IS_GEMINILAKE(dev_priv)) {
3392                 dev_priv->display.cdclk.max_cdclk_freq = 316800;
3393         } else if (IS_BROXTON(dev_priv)) {
3394                 dev_priv->display.cdclk.max_cdclk_freq = 624000;
3395         } else if (DISPLAY_VER(dev_priv) == 9) {
3396                 u32 limit = intel_de_read(dev_priv, SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
3397                 int max_cdclk, vco;
3398
3399                 vco = dev_priv->display.cdclk.skl_preferred_vco_freq;
3400                 drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
3401
3402                 /*
3403                  * Use the lower (vco 8640) cdclk values as a
3404                  * first guess. skl_calc_cdclk() will correct it
3405                  * if the preferred vco is 8100 instead.
3406                  */
3407                 if (limit == SKL_DFSM_CDCLK_LIMIT_675)
3408                         max_cdclk = 617143;
3409                 else if (limit == SKL_DFSM_CDCLK_LIMIT_540)
3410                         max_cdclk = 540000;
3411                 else if (limit == SKL_DFSM_CDCLK_LIMIT_450)
3412                         max_cdclk = 432000;
3413                 else
3414                         max_cdclk = 308571;
3415
3416                 dev_priv->display.cdclk.max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
3417         } else if (IS_BROADWELL(dev_priv))  {
3418                 /*
3419                  * FIXME with extra cooling we can allow
3420                  * 540 MHz for ULX and 675 Mhz for ULT.
3421                  * How can we know if extra cooling is
3422                  * available? PCI ID, VTB, something else?
3423                  */
3424                 if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
3425                         dev_priv->display.cdclk.max_cdclk_freq = 450000;
3426                 else if (IS_BROADWELL_ULX(dev_priv))
3427                         dev_priv->display.cdclk.max_cdclk_freq = 450000;
3428                 else if (IS_BROADWELL_ULT(dev_priv))
3429                         dev_priv->display.cdclk.max_cdclk_freq = 540000;
3430                 else
3431                         dev_priv->display.cdclk.max_cdclk_freq = 675000;
3432         } else if (IS_CHERRYVIEW(dev_priv)) {
3433                 dev_priv->display.cdclk.max_cdclk_freq = 320000;
3434         } else if (IS_VALLEYVIEW(dev_priv)) {
3435                 dev_priv->display.cdclk.max_cdclk_freq = 400000;
3436         } else {
3437                 /* otherwise assume cdclk is fixed */
3438                 dev_priv->display.cdclk.max_cdclk_freq = dev_priv->display.cdclk.hw.cdclk;
3439         }
3440
3441         dev_priv->display.cdclk.max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
3442
3443         drm_dbg(&dev_priv->drm, "Max CD clock rate: %d kHz\n",
3444                 dev_priv->display.cdclk.max_cdclk_freq);
3445
3446         drm_dbg(&dev_priv->drm, "Max dotclock rate: %d kHz\n",
3447                 dev_priv->display.cdclk.max_dotclk_freq);
3448 }
3449
3450 /**
3451  * intel_update_cdclk - Determine the current CDCLK frequency
3452  * @dev_priv: i915 device
3453  *
3454  * Determine the current CDCLK frequency.
3455  */
3456 void intel_update_cdclk(struct drm_i915_private *dev_priv)
3457 {
3458         intel_cdclk_get_cdclk(dev_priv, &dev_priv->display.cdclk.hw);
3459
3460         /*
3461          * 9:0 CMBUS [sic] CDCLK frequency (cdfreq):
3462          * Programmng [sic] note: bit[9:2] should be programmed to the number
3463          * of cdclk that generates 4MHz reference clock freq which is used to
3464          * generate GMBus clock. This will vary with the cdclk freq.
3465          */
3466         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
3467                 intel_de_write(dev_priv, GMBUSFREQ_VLV,
3468                                DIV_ROUND_UP(dev_priv->display.cdclk.hw.cdclk, 1000));
3469 }
3470
3471 static int dg1_rawclk(struct drm_i915_private *dev_priv)
3472 {
3473         /*
3474          * DG1 always uses a 38.4 MHz rawclk.  The bspec tells us
3475          * "Program Numerator=2, Denominator=4, Divider=37 decimal."
3476          */
3477         intel_de_write(dev_priv, PCH_RAWCLK_FREQ,
3478                        CNP_RAWCLK_DEN(4) | CNP_RAWCLK_DIV(37) | ICP_RAWCLK_NUM(2));
3479
3480         return 38400;
3481 }
3482
3483 static int cnp_rawclk(struct drm_i915_private *dev_priv)
3484 {
3485         u32 rawclk;
3486         int divider, fraction;
3487
3488         if (intel_de_read(dev_priv, SFUSE_STRAP) & SFUSE_STRAP_RAW_FREQUENCY) {
3489                 /* 24 MHz */
3490                 divider = 24000;
3491                 fraction = 0;
3492         } else {
3493                 /* 19.2 MHz */
3494                 divider = 19000;
3495                 fraction = 200;
3496         }
3497
3498         rawclk = CNP_RAWCLK_DIV(divider / 1000);
3499         if (fraction) {
3500                 int numerator = 1;
3501
3502                 rawclk |= CNP_RAWCLK_DEN(DIV_ROUND_CLOSEST(numerator * 1000,
3503                                                            fraction) - 1);
3504                 if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3505                         rawclk |= ICP_RAWCLK_NUM(numerator);
3506         }
3507
3508         intel_de_write(dev_priv, PCH_RAWCLK_FREQ, rawclk);
3509         return divider + fraction;
3510 }
3511
3512 static int pch_rawclk(struct drm_i915_private *dev_priv)
3513 {
3514         return (intel_de_read(dev_priv, PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK) * 1000;
3515 }
3516
3517 static int vlv_hrawclk(struct drm_i915_private *dev_priv)
3518 {
3519         /* RAWCLK_FREQ_VLV register updated from power well code */
3520         return vlv_get_cck_clock_hpll(dev_priv, "hrawclk",
3521                                       CCK_DISPLAY_REF_CLOCK_CONTROL);
3522 }
3523
3524 static int i9xx_hrawclk(struct drm_i915_private *dev_priv)
3525 {
3526         u32 clkcfg;
3527
3528         /*
3529          * hrawclock is 1/4 the FSB frequency
3530          *
3531          * Note that this only reads the state of the FSB
3532          * straps, not the actual FSB frequency. Some BIOSen
3533          * let you configure each independently. Ideally we'd
3534          * read out the actual FSB frequency but sadly we
3535          * don't know which registers have that information,
3536          * and all the relevant docs have gone to bit heaven :(
3537          */
3538         clkcfg = intel_de_read(dev_priv, CLKCFG) & CLKCFG_FSB_MASK;
3539
3540         if (IS_MOBILE(dev_priv)) {
3541                 switch (clkcfg) {
3542                 case CLKCFG_FSB_400:
3543                         return 100000;
3544                 case CLKCFG_FSB_533:
3545                         return 133333;
3546                 case CLKCFG_FSB_667:
3547                         return 166667;
3548                 case CLKCFG_FSB_800:
3549                         return 200000;
3550                 case CLKCFG_FSB_1067:
3551                         return 266667;
3552                 case CLKCFG_FSB_1333:
3553                         return 333333;
3554                 default:
3555                         MISSING_CASE(clkcfg);
3556                         return 133333;
3557                 }
3558         } else {
3559                 switch (clkcfg) {
3560                 case CLKCFG_FSB_400_ALT:
3561                         return 100000;
3562                 case CLKCFG_FSB_533:
3563                         return 133333;
3564                 case CLKCFG_FSB_667:
3565                         return 166667;
3566                 case CLKCFG_FSB_800:
3567                         return 200000;
3568                 case CLKCFG_FSB_1067_ALT:
3569                         return 266667;
3570                 case CLKCFG_FSB_1333_ALT:
3571                         return 333333;
3572                 case CLKCFG_FSB_1600_ALT:
3573                         return 400000;
3574                 default:
3575                         return 133333;
3576                 }
3577         }
3578 }
3579
3580 /**
3581  * intel_read_rawclk - Determine the current RAWCLK frequency
3582  * @dev_priv: i915 device
3583  *
3584  * Determine the current RAWCLK frequency. RAWCLK is a fixed
3585  * frequency clock so this needs to done only once.
3586  */
3587 u32 intel_read_rawclk(struct drm_i915_private *dev_priv)
3588 {
3589         u32 freq;
3590
3591         if (INTEL_PCH_TYPE(dev_priv) >= PCH_MTL)
3592                 /*
3593                  * MTL always uses a 38.4 MHz rawclk.  The bspec tells us
3594                  * "RAWCLK_FREQ defaults to the values for 38.4 and does
3595                  * not need to be programmed."
3596                  */
3597                 freq = 38400;
3598         else if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1)
3599                 freq = dg1_rawclk(dev_priv);
3600         else if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
3601                 freq = cnp_rawclk(dev_priv);
3602         else if (HAS_PCH_SPLIT(dev_priv))
3603                 freq = pch_rawclk(dev_priv);
3604         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
3605                 freq = vlv_hrawclk(dev_priv);
3606         else if (DISPLAY_VER(dev_priv) >= 3)
3607                 freq = i9xx_hrawclk(dev_priv);
3608         else
3609                 /* no rawclk on other platforms, or no need to know it */
3610                 return 0;
3611
3612         return freq;
3613 }
3614
3615 static int i915_cdclk_info_show(struct seq_file *m, void *unused)
3616 {
3617         struct drm_i915_private *i915 = m->private;
3618
3619         seq_printf(m, "Current CD clock frequency: %d kHz\n", i915->display.cdclk.hw.cdclk);
3620         seq_printf(m, "Max CD clock frequency: %d kHz\n", i915->display.cdclk.max_cdclk_freq);
3621         seq_printf(m, "Max pixel clock frequency: %d kHz\n", i915->display.cdclk.max_dotclk_freq);
3622
3623         return 0;
3624 }
3625
3626 DEFINE_SHOW_ATTRIBUTE(i915_cdclk_info);
3627
3628 void intel_cdclk_debugfs_register(struct drm_i915_private *i915)
3629 {
3630         struct drm_minor *minor = i915->drm.primary;
3631
3632         debugfs_create_file("i915_cdclk_info", 0444, minor->debugfs_root,
3633                             i915, &i915_cdclk_info_fops);
3634 }
3635
3636 static const struct intel_cdclk_funcs rplu_cdclk_funcs = {
3637         .get_cdclk = bxt_get_cdclk,
3638         .set_cdclk = bxt_set_cdclk,
3639         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3640         .calc_voltage_level = rplu_calc_voltage_level,
3641 };
3642
3643 static const struct intel_cdclk_funcs tgl_cdclk_funcs = {
3644         .get_cdclk = bxt_get_cdclk,
3645         .set_cdclk = bxt_set_cdclk,
3646         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3647         .calc_voltage_level = tgl_calc_voltage_level,
3648 };
3649
3650 static const struct intel_cdclk_funcs ehl_cdclk_funcs = {
3651         .get_cdclk = bxt_get_cdclk,
3652         .set_cdclk = bxt_set_cdclk,
3653         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3654         .calc_voltage_level = ehl_calc_voltage_level,
3655 };
3656
3657 static const struct intel_cdclk_funcs icl_cdclk_funcs = {
3658         .get_cdclk = bxt_get_cdclk,
3659         .set_cdclk = bxt_set_cdclk,
3660         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3661         .calc_voltage_level = icl_calc_voltage_level,
3662 };
3663
3664 static const struct intel_cdclk_funcs bxt_cdclk_funcs = {
3665         .get_cdclk = bxt_get_cdclk,
3666         .set_cdclk = bxt_set_cdclk,
3667         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3668         .calc_voltage_level = bxt_calc_voltage_level,
3669 };
3670
3671 static const struct intel_cdclk_funcs skl_cdclk_funcs = {
3672         .get_cdclk = skl_get_cdclk,
3673         .set_cdclk = skl_set_cdclk,
3674         .modeset_calc_cdclk = skl_modeset_calc_cdclk,
3675 };
3676
3677 static const struct intel_cdclk_funcs bdw_cdclk_funcs = {
3678         .get_cdclk = bdw_get_cdclk,
3679         .set_cdclk = bdw_set_cdclk,
3680         .modeset_calc_cdclk = bdw_modeset_calc_cdclk,
3681 };
3682
3683 static const struct intel_cdclk_funcs chv_cdclk_funcs = {
3684         .get_cdclk = vlv_get_cdclk,
3685         .set_cdclk = chv_set_cdclk,
3686         .modeset_calc_cdclk = vlv_modeset_calc_cdclk,
3687 };
3688
3689 static const struct intel_cdclk_funcs vlv_cdclk_funcs = {
3690         .get_cdclk = vlv_get_cdclk,
3691         .set_cdclk = vlv_set_cdclk,
3692         .modeset_calc_cdclk = vlv_modeset_calc_cdclk,
3693 };
3694
3695 static const struct intel_cdclk_funcs hsw_cdclk_funcs = {
3696         .get_cdclk = hsw_get_cdclk,
3697         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3698 };
3699
3700 /* SNB, IVB, 965G, 945G */
3701 static const struct intel_cdclk_funcs fixed_400mhz_cdclk_funcs = {
3702         .get_cdclk = fixed_400mhz_get_cdclk,
3703         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3704 };
3705
3706 static const struct intel_cdclk_funcs ilk_cdclk_funcs = {
3707         .get_cdclk = fixed_450mhz_get_cdclk,
3708         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3709 };
3710
3711 static const struct intel_cdclk_funcs gm45_cdclk_funcs = {
3712         .get_cdclk = gm45_get_cdclk,
3713         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3714 };
3715
3716 /* G45 uses G33 */
3717
3718 static const struct intel_cdclk_funcs i965gm_cdclk_funcs = {
3719         .get_cdclk = i965gm_get_cdclk,
3720         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3721 };
3722
3723 /* i965G uses fixed 400 */
3724
3725 static const struct intel_cdclk_funcs pnv_cdclk_funcs = {
3726         .get_cdclk = pnv_get_cdclk,
3727         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3728 };
3729
3730 static const struct intel_cdclk_funcs g33_cdclk_funcs = {
3731         .get_cdclk = g33_get_cdclk,
3732         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3733 };
3734
3735 static const struct intel_cdclk_funcs i945gm_cdclk_funcs = {
3736         .get_cdclk = i945gm_get_cdclk,
3737         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3738 };
3739
3740 /* i945G uses fixed 400 */
3741
3742 static const struct intel_cdclk_funcs i915gm_cdclk_funcs = {
3743         .get_cdclk = i915gm_get_cdclk,
3744         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3745 };
3746
3747 static const struct intel_cdclk_funcs i915g_cdclk_funcs = {
3748         .get_cdclk = fixed_333mhz_get_cdclk,
3749         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3750 };
3751
3752 static const struct intel_cdclk_funcs i865g_cdclk_funcs = {
3753         .get_cdclk = fixed_266mhz_get_cdclk,
3754         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3755 };
3756
3757 static const struct intel_cdclk_funcs i85x_cdclk_funcs = {
3758         .get_cdclk = i85x_get_cdclk,
3759         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3760 };
3761
3762 static const struct intel_cdclk_funcs i845g_cdclk_funcs = {
3763         .get_cdclk = fixed_200mhz_get_cdclk,
3764         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3765 };
3766
3767 static const struct intel_cdclk_funcs i830_cdclk_funcs = {
3768         .get_cdclk = fixed_133mhz_get_cdclk,
3769         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3770 };
3771
3772 /**
3773  * intel_init_cdclk_hooks - Initialize CDCLK related modesetting hooks
3774  * @dev_priv: i915 device
3775  */
3776 void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
3777 {
3778         if (DISPLAY_VER(dev_priv) >= 20) {
3779                 dev_priv->display.funcs.cdclk = &rplu_cdclk_funcs;
3780                 dev_priv->display.cdclk.table = xe2lpd_cdclk_table;
3781         } else if (DISPLAY_VER(dev_priv) >= 14) {
3782                 dev_priv->display.funcs.cdclk = &rplu_cdclk_funcs;
3783                 dev_priv->display.cdclk.table = mtl_cdclk_table;
3784         } else if (IS_DG2(dev_priv)) {
3785                 dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3786                 dev_priv->display.cdclk.table = dg2_cdclk_table;
3787         } else if (IS_ALDERLAKE_P(dev_priv)) {
3788                 /* Wa_22011320316:adl-p[a0] */
3789                 if (IS_ALDERLAKE_P(dev_priv) && IS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) {
3790                         dev_priv->display.cdclk.table = adlp_a_step_cdclk_table;
3791                         dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3792                 } else if (IS_RAPTORLAKE_U(dev_priv)) {
3793                         dev_priv->display.cdclk.table = rplu_cdclk_table;
3794                         dev_priv->display.funcs.cdclk = &rplu_cdclk_funcs;
3795                 } else {
3796                         dev_priv->display.cdclk.table = adlp_cdclk_table;
3797                         dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3798                 }
3799         } else if (IS_ROCKETLAKE(dev_priv)) {
3800                 dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3801                 dev_priv->display.cdclk.table = rkl_cdclk_table;
3802         } else if (DISPLAY_VER(dev_priv) >= 12) {
3803                 dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3804                 dev_priv->display.cdclk.table = icl_cdclk_table;
3805         } else if (IS_JASPERLAKE(dev_priv) || IS_ELKHARTLAKE(dev_priv)) {
3806                 dev_priv->display.funcs.cdclk = &ehl_cdclk_funcs;
3807                 dev_priv->display.cdclk.table = icl_cdclk_table;
3808         } else if (DISPLAY_VER(dev_priv) >= 11) {
3809                 dev_priv->display.funcs.cdclk = &icl_cdclk_funcs;
3810                 dev_priv->display.cdclk.table = icl_cdclk_table;
3811         } else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
3812                 dev_priv->display.funcs.cdclk = &bxt_cdclk_funcs;
3813                 if (IS_GEMINILAKE(dev_priv))
3814                         dev_priv->display.cdclk.table = glk_cdclk_table;
3815                 else
3816                         dev_priv->display.cdclk.table = bxt_cdclk_table;
3817         } else if (DISPLAY_VER(dev_priv) == 9) {
3818                 dev_priv->display.funcs.cdclk = &skl_cdclk_funcs;
3819         } else if (IS_BROADWELL(dev_priv)) {
3820                 dev_priv->display.funcs.cdclk = &bdw_cdclk_funcs;
3821         } else if (IS_HASWELL(dev_priv)) {
3822                 dev_priv->display.funcs.cdclk = &hsw_cdclk_funcs;
3823         } else if (IS_CHERRYVIEW(dev_priv)) {
3824                 dev_priv->display.funcs.cdclk = &chv_cdclk_funcs;
3825         } else if (IS_VALLEYVIEW(dev_priv)) {
3826                 dev_priv->display.funcs.cdclk = &vlv_cdclk_funcs;
3827         } else if (IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
3828                 dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3829         } else if (IS_IRONLAKE(dev_priv)) {
3830                 dev_priv->display.funcs.cdclk = &ilk_cdclk_funcs;
3831         } else if (IS_GM45(dev_priv)) {
3832                 dev_priv->display.funcs.cdclk = &gm45_cdclk_funcs;
3833         } else if (IS_G45(dev_priv)) {
3834                 dev_priv->display.funcs.cdclk = &g33_cdclk_funcs;
3835         } else if (IS_I965GM(dev_priv)) {
3836                 dev_priv->display.funcs.cdclk = &i965gm_cdclk_funcs;
3837         } else if (IS_I965G(dev_priv)) {
3838                 dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3839         } else if (IS_PINEVIEW(dev_priv)) {
3840                 dev_priv->display.funcs.cdclk = &pnv_cdclk_funcs;
3841         } else if (IS_G33(dev_priv)) {
3842                 dev_priv->display.funcs.cdclk = &g33_cdclk_funcs;
3843         } else if (IS_I945GM(dev_priv)) {
3844                 dev_priv->display.funcs.cdclk = &i945gm_cdclk_funcs;
3845         } else if (IS_I945G(dev_priv)) {
3846                 dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3847         } else if (IS_I915GM(dev_priv)) {
3848                 dev_priv->display.funcs.cdclk = &i915gm_cdclk_funcs;
3849         } else if (IS_I915G(dev_priv)) {
3850                 dev_priv->display.funcs.cdclk = &i915g_cdclk_funcs;
3851         } else if (IS_I865G(dev_priv)) {
3852                 dev_priv->display.funcs.cdclk = &i865g_cdclk_funcs;
3853         } else if (IS_I85X(dev_priv)) {
3854                 dev_priv->display.funcs.cdclk = &i85x_cdclk_funcs;
3855         } else if (IS_I845G(dev_priv)) {
3856                 dev_priv->display.funcs.cdclk = &i845g_cdclk_funcs;
3857         } else if (IS_I830(dev_priv)) {
3858                 dev_priv->display.funcs.cdclk = &i830_cdclk_funcs;
3859         }
3860
3861         if (drm_WARN(&dev_priv->drm, !dev_priv->display.funcs.cdclk,
3862                      "Unknown platform. Assuming i830\n"))
3863                 dev_priv->display.funcs.cdclk = &i830_cdclk_funcs;
3864 }