drivers/gpu/drm/i915/display/intel_cdclk.c

   1 /*
   2  * Copyright © 2006-2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include <linux/time.h>
  25
  26 #include "hsw_ips.h"
  27 #include "intel_atomic.h"
  28 #include "intel_atomic_plane.h"
  29 #include "intel_audio.h"
  30 #include "intel_bw.h"
  31 #include "intel_cdclk.h"
  32 #include "intel_crtc.h"
  33 #include "intel_de.h"
  34 #include "intel_display_types.h"
  35 #include "intel_mchbar_regs.h"
  36 #include "intel_pci_config.h"
  37 #include "intel_pcode.h"
  38 #include "intel_psr.h"
  39 #include "vlv_sideband.h"
  40
  41 /**
  42  * DOC: CDCLK / RAWCLK
  43  *
  44  * The display engine uses several different clocks to do its work. There
  45  * are two main clocks involved that aren't directly related to the actual
  46  * pixel clock or any symbol/bit clock of the actual output port. These
  47  * are the core display clock (CDCLK) and RAWCLK.
  48  *
  49  * CDCLK clocks most of the display pipe logic, and thus its frequency
  50  * must be high enough to support the rate at which pixels are flowing
  51  * through the pipes. Downscaling must also be accounted as that increases
  52  * the effective pixel rate.
  53  *
  54  * On several platforms the CDCLK frequency can be changed dynamically
  55  * to minimize power consumption for a given display configuration.
  56  * Typically changes to the CDCLK frequency require all the display pipes
  57  * to be shut down while the frequency is being changed.
  58  *
  59  * On SKL+ the DMC will toggle the CDCLK off/on during DC5/6 entry/exit.
  60  * DMC will not change the active CDCLK frequency however, so that part
  61  * will still be performed by the driver directly.
  62  *
  63  * RAWCLK is a fixed frequency clock, often used by various auxiliary
  64  * blocks such as AUX CH or backlight PWM. Hence the only thing we
  65  * really need to know about RAWCLK is its frequency so that various
  66  * dividers can be programmed correctly.
  67  */
  68
  69 struct intel_cdclk_funcs {
  70         void (*get_cdclk)(struct drm_i915_private *i915,
  71                           struct intel_cdclk_config *cdclk_config);
  72         void (*set_cdclk)(struct drm_i915_private *i915,
  73                           const struct intel_cdclk_config *cdclk_config,
  74                           enum pipe pipe);
  75         int (*modeset_calc_cdclk)(struct intel_cdclk_state *state);
  76         u8 (*calc_voltage_level)(int cdclk);
  77 };
  78
  79 void intel_cdclk_get_cdclk(struct drm_i915_private *dev_priv,
  80                            struct intel_cdclk_config *cdclk_config)
  81 {
  82         dev_priv->cdclk_funcs->get_cdclk(dev_priv, cdclk_config);
  83 }
  84
  85 static void intel_cdclk_set_cdclk(struct drm_i915_private *dev_priv,
  86                                   const struct intel_cdclk_config *cdclk_config,
  87                                   enum pipe pipe)
  88 {
  89         dev_priv->cdclk_funcs->set_cdclk(dev_priv, cdclk_config, pipe);
  90 }
  91
  92 static int intel_cdclk_modeset_calc_cdclk(struct drm_i915_private *dev_priv,
  93                                           struct intel_cdclk_state *cdclk_config)
  94 {
  95         return dev_priv->cdclk_funcs->modeset_calc_cdclk(cdclk_config);
  96 }
  97
  98 static u8 intel_cdclk_calc_voltage_level(struct drm_i915_private *dev_priv,
  99                                          int cdclk)
 100 {
 101         return dev_priv->cdclk_funcs->calc_voltage_level(cdclk);
 102 }
 103
 104 static void fixed_133mhz_get_cdclk(struct drm_i915_private *dev_priv,
 105                                    struct intel_cdclk_config *cdclk_config)
 106 {
 107         cdclk_config->cdclk = 133333;
 108 }
 109
 110 static void fixed_200mhz_get_cdclk(struct drm_i915_private *dev_priv,
 111                                    struct intel_cdclk_config *cdclk_config)
 112 {
 113         cdclk_config->cdclk = 200000;
 114 }
 115
 116 static void fixed_266mhz_get_cdclk(struct drm_i915_private *dev_priv,
 117                                    struct intel_cdclk_config *cdclk_config)
 118 {
 119         cdclk_config->cdclk = 266667;
 120 }
 121
 122 static void fixed_333mhz_get_cdclk(struct drm_i915_private *dev_priv,
 123                                    struct intel_cdclk_config *cdclk_config)
 124 {
 125         cdclk_config->cdclk = 333333;
 126 }
 127
 128 static void fixed_400mhz_get_cdclk(struct drm_i915_private *dev_priv,
 129                                    struct intel_cdclk_config *cdclk_config)
 130 {
 131         cdclk_config->cdclk = 400000;
 132 }
 133
 134 static void fixed_450mhz_get_cdclk(struct drm_i915_private *dev_priv,
 135                                    struct intel_cdclk_config *cdclk_config)
 136 {
 137         cdclk_config->cdclk = 450000;
 138 }
 139
 140 static void i85x_get_cdclk(struct drm_i915_private *dev_priv,
 141                            struct intel_cdclk_config *cdclk_config)
 142 {
 143         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 144         u16 hpllcc = 0;
 145
 146         /*
 147          * 852GM/852GMV only supports 133 MHz and the HPLLCC
 148          * encoding is different :(
 149          * FIXME is this the right way to detect 852GM/852GMV?
 150          */
 151         if (pdev->revision == 0x1) {
 152                 cdclk_config->cdclk = 133333;
 153                 return;
 154         }
 155
 156         pci_bus_read_config_word(pdev->bus,
 157                                  PCI_DEVFN(0, 3), HPLLCC, &hpllcc);
 158
 159         /* Assume that the hardware is in the high speed state.  This
 160          * should be the default.
 161          */
 162         switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
 163         case GC_CLOCK_133_200:
 164         case GC_CLOCK_133_200_2:
 165         case GC_CLOCK_100_200:
 166                 cdclk_config->cdclk = 200000;
 167                 break;
 168         case GC_CLOCK_166_250:
 169                 cdclk_config->cdclk = 250000;
 170                 break;
 171         case GC_CLOCK_100_133:
 172                 cdclk_config->cdclk = 133333;
 173                 break;
 174         case GC_CLOCK_133_266:
 175         case GC_CLOCK_133_266_2:
 176         case GC_CLOCK_166_266:
 177                 cdclk_config->cdclk = 266667;
 178                 break;
 179         }
 180 }
 181
 182 static void i915gm_get_cdclk(struct drm_i915_private *dev_priv,
 183                              struct intel_cdclk_config *cdclk_config)
 184 {
 185         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 186         u16 gcfgc = 0;
 187
 188         pci_read_config_word(pdev, GCFGC, &gcfgc);
 189
 190         if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 191                 cdclk_config->cdclk = 133333;
 192                 return;
 193         }
 194
 195         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 196         case GC_DISPLAY_CLOCK_333_320_MHZ:
 197                 cdclk_config->cdclk = 333333;
 198                 break;
 199         default:
 200         case GC_DISPLAY_CLOCK_190_200_MHZ:
 201                 cdclk_config->cdclk = 190000;
 202                 break;
 203         }
 204 }
 205
 206 static void i945gm_get_cdclk(struct drm_i915_private *dev_priv,
 207                              struct intel_cdclk_config *cdclk_config)
 208 {
 209         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 210         u16 gcfgc = 0;
 211
 212         pci_read_config_word(pdev, GCFGC, &gcfgc);
 213
 214         if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 215                 cdclk_config->cdclk = 133333;
 216                 return;
 217         }
 218
 219         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 220         case GC_DISPLAY_CLOCK_333_320_MHZ:
 221                 cdclk_config->cdclk = 320000;
 222                 break;
 223         default:
 224         case GC_DISPLAY_CLOCK_190_200_MHZ:
 225                 cdclk_config->cdclk = 200000;
 226                 break;
 227         }
 228 }
 229
 230 static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv)
 231 {
 232         static const unsigned int blb_vco[8] = {
 233                 [0] = 3200000,
 234                 [1] = 4000000,
 235                 [2] = 5333333,
 236                 [3] = 4800000,
 237                 [4] = 6400000,
 238         };
 239         static const unsigned int pnv_vco[8] = {
 240                 [0] = 3200000,
 241                 [1] = 4000000,
 242                 [2] = 5333333,
 243                 [3] = 4800000,
 244                 [4] = 2666667,
 245         };
 246         static const unsigned int cl_vco[8] = {
 247                 [0] = 3200000,
 248                 [1] = 4000000,
 249                 [2] = 5333333,
 250                 [3] = 6400000,
 251                 [4] = 3333333,
 252                 [5] = 3566667,
 253                 [6] = 4266667,
 254         };
 255         static const unsigned int elk_vco[8] = {
 256                 [0] = 3200000,
 257                 [1] = 4000000,
 258                 [2] = 5333333,
 259                 [3] = 4800000,
 260         };
 261         static const unsigned int ctg_vco[8] = {
 262                 [0] = 3200000,
 263                 [1] = 4000000,
 264                 [2] = 5333333,
 265                 [3] = 6400000,
 266                 [4] = 2666667,
 267                 [5] = 4266667,
 268         };
 269         const unsigned int *vco_table;
 270         unsigned int vco;
 271         u8 tmp = 0;
 272
 273         /* FIXME other chipsets? */
 274         if (IS_GM45(dev_priv))
 275                 vco_table = ctg_vco;
 276         else if (IS_G45(dev_priv))
 277                 vco_table = elk_vco;
 278         else if (IS_I965GM(dev_priv))
 279                 vco_table = cl_vco;
 280         else if (IS_PINEVIEW(dev_priv))
 281                 vco_table = pnv_vco;
 282         else if (IS_G33(dev_priv))
 283                 vco_table = blb_vco;
 284         else
 285                 return 0;
 286
 287         tmp = intel_de_read(dev_priv,
 288                             IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv) ? HPLLVCO_MOBILE : HPLLVCO);
 289
 290         vco = vco_table[tmp & 0x7];
 291         if (vco == 0)
 292                 drm_err(&dev_priv->drm, "Bad HPLL VCO (HPLLVCO=0x%02x)\n",
 293                         tmp);
 294         else
 295                 drm_dbg_kms(&dev_priv->drm, "HPLL VCO %u kHz\n", vco);
 296
 297         return vco;
 298 }
 299
 300 static void g33_get_cdclk(struct drm_i915_private *dev_priv,
 301                           struct intel_cdclk_config *cdclk_config)
 302 {
 303         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 304         static const u8 div_3200[] = { 12, 10,  8,  7, 5, 16 };
 305         static const u8 div_4000[] = { 14, 12, 10,  8, 6, 20 };
 306         static const u8 div_4800[] = { 20, 14, 12, 10, 8, 24 };
 307         static const u8 div_5333[] = { 20, 16, 12, 12, 8, 28 };
 308         const u8 *div_table;
 309         unsigned int cdclk_sel;
 310         u16 tmp = 0;
 311
 312         cdclk_config->vco = intel_hpll_vco(dev_priv);
 313
 314         pci_read_config_word(pdev, GCFGC, &tmp);
 315
 316         cdclk_sel = (tmp >> 4) & 0x7;
 317
 318         if (cdclk_sel >= ARRAY_SIZE(div_3200))
 319                 goto fail;
 320
 321         switch (cdclk_config->vco) {
 322         case 3200000:
 323                 div_table = div_3200;
 324                 break;
 325         case 4000000:
 326                 div_table = div_4000;
 327                 break;
 328         case 4800000:
 329                 div_table = div_4800;
 330                 break;
 331         case 5333333:
 332                 div_table = div_5333;
 333                 break;
 334         default:
 335                 goto fail;
 336         }
 337
 338         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 339                                                 div_table[cdclk_sel]);
 340         return;
 341
 342 fail:
 343         drm_err(&dev_priv->drm,
 344                 "Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n",
 345                 cdclk_config->vco, tmp);
 346         cdclk_config->cdclk = 190476;
 347 }
 348
 349 static void pnv_get_cdclk(struct drm_i915_private *dev_priv,
 350                           struct intel_cdclk_config *cdclk_config)
 351 {
 352         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 353         u16 gcfgc = 0;
 354
 355         pci_read_config_word(pdev, GCFGC, &gcfgc);
 356
 357         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 358         case GC_DISPLAY_CLOCK_267_MHZ_PNV:
 359                 cdclk_config->cdclk = 266667;
 360                 break;
 361         case GC_DISPLAY_CLOCK_333_MHZ_PNV:
 362                 cdclk_config->cdclk = 333333;
 363                 break;
 364         case GC_DISPLAY_CLOCK_444_MHZ_PNV:
 365                 cdclk_config->cdclk = 444444;
 366                 break;
 367         case GC_DISPLAY_CLOCK_200_MHZ_PNV:
 368                 cdclk_config->cdclk = 200000;
 369                 break;
 370         default:
 371                 drm_err(&dev_priv->drm,
 372                         "Unknown pnv display core clock 0x%04x\n", gcfgc);
 373                 fallthrough;
 374         case GC_DISPLAY_CLOCK_133_MHZ_PNV:
 375                 cdclk_config->cdclk = 133333;
 376                 break;
 377         case GC_DISPLAY_CLOCK_167_MHZ_PNV:
 378                 cdclk_config->cdclk = 166667;
 379                 break;
 380         }
 381 }
 382
 383 static void i965gm_get_cdclk(struct drm_i915_private *dev_priv,
 384                              struct intel_cdclk_config *cdclk_config)
 385 {
 386         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 387         static const u8 div_3200[] = { 16, 10,  8 };
 388         static const u8 div_4000[] = { 20, 12, 10 };
 389         static const u8 div_5333[] = { 24, 16, 14 };
 390         const u8 *div_table;
 391         unsigned int cdclk_sel;
 392         u16 tmp = 0;
 393
 394         cdclk_config->vco = intel_hpll_vco(dev_priv);
 395
 396         pci_read_config_word(pdev, GCFGC, &tmp);
 397
 398         cdclk_sel = ((tmp >> 8) & 0x1f) - 1;
 399
 400         if (cdclk_sel >= ARRAY_SIZE(div_3200))
 401                 goto fail;
 402
 403         switch (cdclk_config->vco) {
 404         case 3200000:
 405                 div_table = div_3200;
 406                 break;
 407         case 4000000:
 408                 div_table = div_4000;
 409                 break;
 410         case 5333333:
 411                 div_table = div_5333;
 412                 break;
 413         default:
 414                 goto fail;
 415         }
 416
 417         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 418                                                 div_table[cdclk_sel]);
 419         return;
 420
 421 fail:
 422         drm_err(&dev_priv->drm,
 423                 "Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n",
 424                 cdclk_config->vco, tmp);
 425         cdclk_config->cdclk = 200000;
 426 }
 427
 428 static void gm45_get_cdclk(struct drm_i915_private *dev_priv,
 429                            struct intel_cdclk_config *cdclk_config)
 430 {
 431         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 432         unsigned int cdclk_sel;
 433         u16 tmp = 0;
 434
 435         cdclk_config->vco = intel_hpll_vco(dev_priv);
 436
 437         pci_read_config_word(pdev, GCFGC, &tmp);
 438
 439         cdclk_sel = (tmp >> 12) & 0x1;
 440
 441         switch (cdclk_config->vco) {
 442         case 2666667:
 443         case 4000000:
 444         case 5333333:
 445                 cdclk_config->cdclk = cdclk_sel ? 333333 : 222222;
 446                 break;
 447         case 3200000:
 448                 cdclk_config->cdclk = cdclk_sel ? 320000 : 228571;
 449                 break;
 450         default:
 451                 drm_err(&dev_priv->drm,
 452                         "Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n",
 453                         cdclk_config->vco, tmp);
 454                 cdclk_config->cdclk = 222222;
 455                 break;
 456         }
 457 }
 458
 459 static void hsw_get_cdclk(struct drm_i915_private *dev_priv,
 460                           struct intel_cdclk_config *cdclk_config)
 461 {
 462         u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 463         u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 464
 465         if (lcpll & LCPLL_CD_SOURCE_FCLK)
 466                 cdclk_config->cdclk = 800000;
 467         else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 468                 cdclk_config->cdclk = 450000;
 469         else if (freq == LCPLL_CLK_FREQ_450)
 470                 cdclk_config->cdclk = 450000;
 471         else if (IS_HSW_ULT(dev_priv))
 472                 cdclk_config->cdclk = 337500;
 473         else
 474                 cdclk_config->cdclk = 540000;
 475 }
 476
 477 static int vlv_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
 478 {
 479         int freq_320 = (dev_priv->hpll_freq <<  1) % 320000 != 0 ?
 480                 333333 : 320000;
 481
 482         /*
 483          * We seem to get an unstable or solid color picture at 200MHz.
 484          * Not sure what's wrong. For now use 200MHz only when all pipes
 485          * are off.
 486          */
 487         if (IS_VALLEYVIEW(dev_priv) && min_cdclk > freq_320)
 488                 return 400000;
 489         else if (min_cdclk > 266667)
 490                 return freq_320;
 491         else if (min_cdclk > 0)
 492                 return 266667;
 493         else
 494                 return 200000;
 495 }
 496
 497 static u8 vlv_calc_voltage_level(struct drm_i915_private *dev_priv, int cdclk)
 498 {
 499         if (IS_VALLEYVIEW(dev_priv)) {
 500                 if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
 501                         return 2;
 502                 else if (cdclk >= 266667)
 503                         return 1;
 504                 else
 505                         return 0;
 506         } else {
 507                 /*
 508                  * Specs are full of misinformation, but testing on actual
 509                  * hardware has shown that we just need to write the desired
 510                  * CCK divider into the Punit register.
 511                  */
 512                 return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
 513         }
 514 }
 515
 516 static void vlv_get_cdclk(struct drm_i915_private *dev_priv,
 517                           struct intel_cdclk_config *cdclk_config)
 518 {
 519         u32 val;
 520
 521         vlv_iosf_sb_get(dev_priv,
 522                         BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 523
 524         cdclk_config->vco = vlv_get_hpll_vco(dev_priv);
 525         cdclk_config->cdclk = vlv_get_cck_clock(dev_priv, "cdclk",
 526                                                 CCK_DISPLAY_CLOCK_CONTROL,
 527                                                 cdclk_config->vco);
 528
 529         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 530
 531         vlv_iosf_sb_put(dev_priv,
 532                         BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 533
 534         if (IS_VALLEYVIEW(dev_priv))
 535                 cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK) >>
 536                         DSPFREQGUAR_SHIFT;
 537         else
 538                 cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK_CHV) >>
 539                         DSPFREQGUAR_SHIFT_CHV;
 540 }
 541
 542 static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
 543 {
 544         unsigned int credits, default_credits;
 545
 546         if (IS_CHERRYVIEW(dev_priv))
 547                 default_credits = PFI_CREDIT(12);
 548         else
 549                 default_credits = PFI_CREDIT(8);
 550
 551         if (dev_priv->cdclk.hw.cdclk >= dev_priv->czclk_freq) {
 552                 /* CHV suggested value is 31 or 63 */
 553                 if (IS_CHERRYVIEW(dev_priv))
 554                         credits = PFI_CREDIT_63;
 555                 else
 556                         credits = PFI_CREDIT(15);
 557         } else {
 558                 credits = default_credits;
 559         }
 560
 561         /*
 562          * WA - write default credits before re-programming
 563          * FIXME: should we also set the resend bit here?
 564          */
 565         intel_de_write(dev_priv, GCI_CONTROL,
 566                        VGA_FAST_MODE_DISABLE | default_credits);
 567
 568         intel_de_write(dev_priv, GCI_CONTROL,
 569                        VGA_FAST_MODE_DISABLE | credits | PFI_CREDIT_RESEND);
 570
 571         /*
 572          * FIXME is this guaranteed to clear
 573          * immediately or should we poll for it?
 574          */
 575         drm_WARN_ON(&dev_priv->drm,
 576                     intel_de_read(dev_priv, GCI_CONTROL) & PFI_CREDIT_RESEND);
 577 }
 578
 579 static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
 580                           const struct intel_cdclk_config *cdclk_config,
 581                           enum pipe pipe)
 582 {
 583         int cdclk = cdclk_config->cdclk;
 584         u32 val, cmd = cdclk_config->voltage_level;
 585         intel_wakeref_t wakeref;
 586
 587         switch (cdclk) {
 588         case 400000:
 589         case 333333:
 590         case 320000:
 591         case 266667:
 592         case 200000:
 593                 break;
 594         default:
 595                 MISSING_CASE(cdclk);
 596                 return;
 597         }
 598
 599         /* There are cases where we can end up here with power domains
 600          * off and a CDCLK frequency other than the minimum, like when
 601          * issuing a modeset without actually changing any display after
 602          * a system suspend.  So grab the display core domain, which covers
 603          * the HW blocks needed for the following programming.
 604          */
 605         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 606
 607         vlv_iosf_sb_get(dev_priv,
 608                         BIT(VLV_IOSF_SB_CCK) |
 609                         BIT(VLV_IOSF_SB_BUNIT) |
 610                         BIT(VLV_IOSF_SB_PUNIT));
 611
 612         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 613         val &= ~DSPFREQGUAR_MASK;
 614         val |= (cmd << DSPFREQGUAR_SHIFT);
 615         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 616         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 617                       DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
 618                      50)) {
 619                 drm_err(&dev_priv->drm,
 620                         "timed out waiting for CDclk change\n");
 621         }
 622
 623         if (cdclk == 400000) {
 624                 u32 divider;
 625
 626                 divider = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1,
 627                                             cdclk) - 1;
 628
 629                 /* adjust cdclk divider */
 630                 val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
 631                 val &= ~CCK_FREQUENCY_VALUES;
 632                 val |= divider;
 633                 vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val);
 634
 635                 if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) &
 636                               CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT),
 637                              50))
 638                         drm_err(&dev_priv->drm,
 639                                 "timed out waiting for CDclk change\n");
 640         }
 641
 642         /* adjust self-refresh exit latency value */
 643         val = vlv_bunit_read(dev_priv, BUNIT_REG_BISOC);
 644         val &= ~0x7f;
 645
 646         /*
 647          * For high bandwidth configs, we set a higher latency in the bunit
 648          * so that the core display fetch happens in time to avoid underruns.
 649          */
 650         if (cdclk == 400000)
 651                 val |= 4500 / 250; /* 4.5 usec */
 652         else
 653                 val |= 3000 / 250; /* 3.0 usec */
 654         vlv_bunit_write(dev_priv, BUNIT_REG_BISOC, val);
 655
 656         vlv_iosf_sb_put(dev_priv,
 657                         BIT(VLV_IOSF_SB_CCK) |
 658                         BIT(VLV_IOSF_SB_BUNIT) |
 659                         BIT(VLV_IOSF_SB_PUNIT));
 660
 661         intel_update_cdclk(dev_priv);
 662
 663         vlv_program_pfi_credits(dev_priv);
 664
 665         intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 666 }
 667
 668 static void chv_set_cdclk(struct drm_i915_private *dev_priv,
 669                           const struct intel_cdclk_config *cdclk_config,
 670                           enum pipe pipe)
 671 {
 672         int cdclk = cdclk_config->cdclk;
 673         u32 val, cmd = cdclk_config->voltage_level;
 674         intel_wakeref_t wakeref;
 675
 676         switch (cdclk) {
 677         case 333333:
 678         case 320000:
 679         case 266667:
 680         case 200000:
 681                 break;
 682         default:
 683                 MISSING_CASE(cdclk);
 684                 return;
 685         }
 686
 687         /* There are cases where we can end up here with power domains
 688          * off and a CDCLK frequency other than the minimum, like when
 689          * issuing a modeset without actually changing any display after
 690          * a system suspend.  So grab the display core domain, which covers
 691          * the HW blocks needed for the following programming.
 692          */
 693         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 694
 695         vlv_punit_get(dev_priv);
 696         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 697         val &= ~DSPFREQGUAR_MASK_CHV;
 698         val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
 699         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 700         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 701                       DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
 702                      50)) {
 703                 drm_err(&dev_priv->drm,
 704                         "timed out waiting for CDclk change\n");
 705         }
 706
 707         vlv_punit_put(dev_priv);
 708
 709         intel_update_cdclk(dev_priv);
 710
 711         vlv_program_pfi_credits(dev_priv);
 712
 713         intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 714 }
 715
 716 static int bdw_calc_cdclk(int min_cdclk)
 717 {
 718         if (min_cdclk > 540000)
 719                 return 675000;
 720         else if (min_cdclk > 450000)
 721                 return 540000;
 722         else if (min_cdclk > 337500)
 723                 return 450000;
 724         else
 725                 return 337500;
 726 }
 727
 728 static u8 bdw_calc_voltage_level(int cdclk)
 729 {
 730         switch (cdclk) {
 731         default:
 732         case 337500:
 733                 return 2;
 734         case 450000:
 735                 return 0;
 736         case 540000:
 737                 return 1;
 738         case 675000:
 739                 return 3;
 740         }
 741 }
 742
 743 static void bdw_get_cdclk(struct drm_i915_private *dev_priv,
 744                           struct intel_cdclk_config *cdclk_config)
 745 {
 746         u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 747         u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 748
 749         if (lcpll & LCPLL_CD_SOURCE_FCLK)
 750                 cdclk_config->cdclk = 800000;
 751         else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 752                 cdclk_config->cdclk = 450000;
 753         else if (freq == LCPLL_CLK_FREQ_450)
 754                 cdclk_config->cdclk = 450000;
 755         else if (freq == LCPLL_CLK_FREQ_54O_BDW)
 756                 cdclk_config->cdclk = 540000;
 757         else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
 758                 cdclk_config->cdclk = 337500;
 759         else
 760                 cdclk_config->cdclk = 675000;
 761
 762         /*
 763          * Can't read this out :( Let's assume it's
 764          * at least what the CDCLK frequency requires.
 765          */
 766         cdclk_config->voltage_level =
 767                 bdw_calc_voltage_level(cdclk_config->cdclk);
 768 }
 769
 770 static u32 bdw_cdclk_freq_sel(int cdclk)
 771 {
 772         switch (cdclk) {
 773         default:
 774                 MISSING_CASE(cdclk);
 775                 fallthrough;
 776         case 337500:
 777                 return LCPLL_CLK_FREQ_337_5_BDW;
 778         case 450000:
 779                 return LCPLL_CLK_FREQ_450;
 780         case 540000:
 781                 return LCPLL_CLK_FREQ_54O_BDW;
 782         case 675000:
 783                 return LCPLL_CLK_FREQ_675_BDW;
 784         }
 785 }
 786
 787 static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
 788                           const struct intel_cdclk_config *cdclk_config,
 789                           enum pipe pipe)
 790 {
 791         int cdclk = cdclk_config->cdclk;
 792         int ret;
 793
 794         if (drm_WARN(&dev_priv->drm,
 795                      (intel_de_read(dev_priv, LCPLL_CTL) &
 796                       (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK |
 797                        LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE |
 798                        LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW |
 799                        LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK,
 800                      "trying to change cdclk frequency with cdclk not enabled\n"))
 801                 return;
 802
 803         ret = snb_pcode_write(dev_priv, BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
 804         if (ret) {
 805                 drm_err(&dev_priv->drm,
 806                         "failed to inform pcode about cdclk change\n");
 807                 return;
 808         }
 809
 810         intel_de_rmw(dev_priv, LCPLL_CTL,
 811                      0, LCPLL_CD_SOURCE_FCLK);
 812
 813         /*
 814          * According to the spec, it should be enough to poll for this 1 us.
 815          * However, extensive testing shows that this can take longer.
 816          */
 817         if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
 818                         LCPLL_CD_SOURCE_FCLK_DONE, 100))
 819                 drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
 820
 821         intel_de_rmw(dev_priv, LCPLL_CTL,
 822                      LCPLL_CLK_FREQ_MASK, bdw_cdclk_freq_sel(cdclk));
 823
 824         intel_de_rmw(dev_priv, LCPLL_CTL,
 825                      LCPLL_CD_SOURCE_FCLK, 0);
 826
 827         if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
 828                          LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
 829                 drm_err(&dev_priv->drm, "Switching back to LCPLL failed\n");
 830
 831         snb_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
 832                         cdclk_config->voltage_level);
 833
 834         intel_de_write(dev_priv, CDCLK_FREQ,
 835                        DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
 836
 837         intel_update_cdclk(dev_priv);
 838 }
 839
 840 static int skl_calc_cdclk(int min_cdclk, int vco)
 841 {
 842         if (vco == 8640000) {
 843                 if (min_cdclk > 540000)
 844                         return 617143;
 845                 else if (min_cdclk > 432000)
 846                         return 540000;
 847                 else if (min_cdclk > 308571)
 848                         return 432000;
 849                 else
 850                         return 308571;
 851         } else {
 852                 if (min_cdclk > 540000)
 853                         return 675000;
 854                 else if (min_cdclk > 450000)
 855                         return 540000;
 856                 else if (min_cdclk > 337500)
 857                         return 450000;
 858                 else
 859                         return 337500;
 860         }
 861 }
 862
 863 static u8 skl_calc_voltage_level(int cdclk)
 864 {
 865         if (cdclk > 540000)
 866                 return 3;
 867         else if (cdclk > 450000)
 868                 return 2;
 869         else if (cdclk > 337500)
 870                 return 1;
 871         else
 872                 return 0;
 873 }
 874
 875 static void skl_dpll0_update(struct drm_i915_private *dev_priv,
 876                              struct intel_cdclk_config *cdclk_config)
 877 {
 878         u32 val;
 879
 880         cdclk_config->ref = 24000;
 881         cdclk_config->vco = 0;
 882
 883         val = intel_de_read(dev_priv, LCPLL1_CTL);
 884         if ((val & LCPLL_PLL_ENABLE) == 0)
 885                 return;
 886
 887         if (drm_WARN_ON(&dev_priv->drm, (val & LCPLL_PLL_LOCK) == 0))
 888                 return;
 889
 890         val = intel_de_read(dev_priv, DPLL_CTRL1);
 891
 892         if (drm_WARN_ON(&dev_priv->drm,
 893                         (val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
 894                                 DPLL_CTRL1_SSC(SKL_DPLL0) |
 895                                 DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) !=
 896                         DPLL_CTRL1_OVERRIDE(SKL_DPLL0)))
 897                 return;
 898
 899         switch (val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) {
 900         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0):
 901         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, SKL_DPLL0):
 902         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, SKL_DPLL0):
 903         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, SKL_DPLL0):
 904                 cdclk_config->vco = 8100000;
 905                 break;
 906         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0):
 907         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, SKL_DPLL0):
 908                 cdclk_config->vco = 8640000;
 909                 break;
 910         default:
 911                 MISSING_CASE(val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
 912                 break;
 913         }
 914 }
 915
 916 static void skl_get_cdclk(struct drm_i915_private *dev_priv,
 917                           struct intel_cdclk_config *cdclk_config)
 918 {
 919         u32 cdctl;
 920
 921         skl_dpll0_update(dev_priv, cdclk_config);
 922
 923         cdclk_config->cdclk = cdclk_config->bypass = cdclk_config->ref;
 924
 925         if (cdclk_config->vco == 0)
 926                 goto out;
 927
 928         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
 929
 930         if (cdclk_config->vco == 8640000) {
 931                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 932                 case CDCLK_FREQ_450_432:
 933                         cdclk_config->cdclk = 432000;
 934                         break;
 935                 case CDCLK_FREQ_337_308:
 936                         cdclk_config->cdclk = 308571;
 937                         break;
 938                 case CDCLK_FREQ_540:
 939                         cdclk_config->cdclk = 540000;
 940                         break;
 941                 case CDCLK_FREQ_675_617:
 942                         cdclk_config->cdclk = 617143;
 943                         break;
 944                 default:
 945                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
 946                         break;
 947                 }
 948         } else {
 949                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 950                 case CDCLK_FREQ_450_432:
 951                         cdclk_config->cdclk = 450000;
 952                         break;
 953                 case CDCLK_FREQ_337_308:
 954                         cdclk_config->cdclk = 337500;
 955                         break;
 956                 case CDCLK_FREQ_540:
 957                         cdclk_config->cdclk = 540000;
 958                         break;
 959                 case CDCLK_FREQ_675_617:
 960                         cdclk_config->cdclk = 675000;
 961                         break;
 962                 default:
 963                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
 964                         break;
 965                 }
 966         }
 967
 968  out:
 969         /*
 970          * Can't read this out :( Let's assume it's
 971          * at least what the CDCLK frequency requires.
 972          */
 973         cdclk_config->voltage_level =
 974                 skl_calc_voltage_level(cdclk_config->cdclk);
 975 }
 976
 977 /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
 978 static int skl_cdclk_decimal(int cdclk)
 979 {
 980         return DIV_ROUND_CLOSEST(cdclk - 1000, 500);
 981 }
 982
 983 static void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv,
 984                                         int vco)
 985 {
 986         bool changed = dev_priv->skl_preferred_vco_freq != vco;
 987
 988         dev_priv->skl_preferred_vco_freq = vco;
 989
 990         if (changed)
 991                 intel_update_max_cdclk(dev_priv);
 992 }
 993
 994 static u32 skl_dpll0_link_rate(struct drm_i915_private *dev_priv, int vco)
 995 {
 996         drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
 997
 998         /*
 999          * We always enable DPLL0 with the lowest link rate possible, but still
1000          * taking into account the VCO required to operate the eDP panel at the
1001          * desired frequency. The usual DP link rates operate with a VCO of
1002          * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640.
1003          * The modeset code is responsible for the selection of the exact link
1004          * rate later on, with the constraint of choosing a frequency that
1005          * works with vco.
1006          */
1007         if (vco == 8640000)
1008                 return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0);
1009         else
1010                 return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0);
1011 }
1012
1013 static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco)
1014 {
1015         intel_de_rmw(dev_priv, DPLL_CTRL1,
1016                      DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
1017                      DPLL_CTRL1_SSC(SKL_DPLL0) |
1018                      DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0),
1019                      DPLL_CTRL1_OVERRIDE(SKL_DPLL0) |
1020                      skl_dpll0_link_rate(dev_priv, vco));
1021         intel_de_posting_read(dev_priv, DPLL_CTRL1);
1022
1023         intel_de_rmw(dev_priv, LCPLL1_CTL,
1024                      0, LCPLL_PLL_ENABLE);
1025
1026         if (intel_de_wait_for_set(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 5))
1027                 drm_err(&dev_priv->drm, "DPLL0 not locked\n");
1028
1029         dev_priv->cdclk.hw.vco = vco;
1030
1031         /* We'll want to keep using the current vco from now on. */
1032         skl_set_preferred_cdclk_vco(dev_priv, vco);
1033 }
1034
1035 static void skl_dpll0_disable(struct drm_i915_private *dev_priv)
1036 {
1037         intel_de_rmw(dev_priv, LCPLL1_CTL,
1038                      LCPLL_PLL_ENABLE, 0);
1039
1040         if (intel_de_wait_for_clear(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 1))
1041                 drm_err(&dev_priv->drm, "Couldn't disable DPLL0\n");
1042
1043         dev_priv->cdclk.hw.vco = 0;
1044 }
1045
1046 static u32 skl_cdclk_freq_sel(struct drm_i915_private *dev_priv,
1047                               int cdclk, int vco)
1048 {
1049         switch (cdclk) {
1050         default:
1051                 drm_WARN_ON(&dev_priv->drm,
1052                             cdclk != dev_priv->cdclk.hw.bypass);
1053                 drm_WARN_ON(&dev_priv->drm, vco != 0);
1054                 fallthrough;
1055         case 308571:
1056         case 337500:
1057                 return CDCLK_FREQ_337_308;
1058         case 450000:
1059         case 432000:
1060                 return CDCLK_FREQ_450_432;
1061         case 540000:
1062                 return CDCLK_FREQ_540;
1063         case 617143:
1064         case 675000:
1065                 return CDCLK_FREQ_675_617;
1066         }
1067 }
1068
1069 static void skl_set_cdclk(struct drm_i915_private *dev_priv,
1070                           const struct intel_cdclk_config *cdclk_config,
1071                           enum pipe pipe)
1072 {
1073         int cdclk = cdclk_config->cdclk;
1074         int vco = cdclk_config->vco;
1075         u32 freq_select, cdclk_ctl;
1076         int ret;
1077
1078         /*
1079          * Based on WA#1183 CDCLK rates 308 and 617MHz CDCLK rates are
1080          * unsupported on SKL. In theory this should never happen since only
1081          * the eDP1.4 2.16 and 4.32Gbps rates require it, but eDP1.4 is not
1082          * supported on SKL either, see the above WA. WARN whenever trying to
1083          * use the corresponding VCO freq as that always leads to using the
1084          * minimum 308MHz CDCLK.
1085          */
1086         drm_WARN_ON_ONCE(&dev_priv->drm,
1087                          IS_SKYLAKE(dev_priv) && vco == 8640000);
1088
1089         ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1090                                 SKL_CDCLK_PREPARE_FOR_CHANGE,
1091                                 SKL_CDCLK_READY_FOR_CHANGE,
1092                                 SKL_CDCLK_READY_FOR_CHANGE, 3);
1093         if (ret) {
1094                 drm_err(&dev_priv->drm,
1095                         "Failed to inform PCU about cdclk change (%d)\n", ret);
1096                 return;
1097         }
1098
1099         freq_select = skl_cdclk_freq_sel(dev_priv, cdclk, vco);
1100
1101         if (dev_priv->cdclk.hw.vco != 0 &&
1102             dev_priv->cdclk.hw.vco != vco)
1103                 skl_dpll0_disable(dev_priv);
1104
1105         cdclk_ctl = intel_de_read(dev_priv, CDCLK_CTL);
1106
1107         if (dev_priv->cdclk.hw.vco != vco) {
1108                 /* Wa Display #1183: skl,kbl,cfl */
1109                 cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1110                 cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1111                 intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1112         }
1113
1114         /* Wa Display #1183: skl,kbl,cfl */
1115         cdclk_ctl |= CDCLK_DIVMUX_CD_OVERRIDE;
1116         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1117         intel_de_posting_read(dev_priv, CDCLK_CTL);
1118
1119         if (dev_priv->cdclk.hw.vco != vco)
1120                 skl_dpll0_enable(dev_priv, vco);
1121
1122         /* Wa Display #1183: skl,kbl,cfl */
1123         cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1124         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1125
1126         cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1127         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1128
1129         /* Wa Display #1183: skl,kbl,cfl */
1130         cdclk_ctl &= ~CDCLK_DIVMUX_CD_OVERRIDE;
1131         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1132         intel_de_posting_read(dev_priv, CDCLK_CTL);
1133
1134         /* inform PCU of the change */
1135         snb_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1136                         cdclk_config->voltage_level);
1137
1138         intel_update_cdclk(dev_priv);
1139 }
1140
1141 static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
1142 {
1143         u32 cdctl, expected;
1144
1145         /*
1146          * check if the pre-os initialized the display
1147          * There is SWF18 scratchpad register defined which is set by the
1148          * pre-os which can be used by the OS drivers to check the status
1149          */
1150         if ((intel_de_read(dev_priv, SWF_ILK(0x18)) & 0x00FFFFFF) == 0)
1151                 goto sanitize;
1152
1153         intel_update_cdclk(dev_priv);
1154         intel_cdclk_dump_config(dev_priv, &dev_priv->cdclk.hw, "Current CDCLK");
1155
1156         /* Is PLL enabled and locked ? */
1157         if (dev_priv->cdclk.hw.vco == 0 ||
1158             dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass)
1159                 goto sanitize;
1160
1161         /* DPLL okay; verify the cdclock
1162          *
1163          * Noticed in some instances that the freq selection is correct but
1164          * decimal part is programmed wrong from BIOS where pre-os does not
1165          * enable display. Verify the same as well.
1166          */
1167         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1168         expected = (cdctl & CDCLK_FREQ_SEL_MASK) |
1169                 skl_cdclk_decimal(dev_priv->cdclk.hw.cdclk);
1170         if (cdctl == expected)
1171                 /* All well; nothing to sanitize */
1172                 return;
1173
1174 sanitize:
1175         drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1176
1177         /* force cdclk programming */
1178         dev_priv->cdclk.hw.cdclk = 0;
1179         /* force full PLL disable + enable */
1180         dev_priv->cdclk.hw.vco = -1;
1181 }
1182
1183 static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv)
1184 {
1185         struct intel_cdclk_config cdclk_config;
1186
1187         skl_sanitize_cdclk(dev_priv);
1188
1189         if (dev_priv->cdclk.hw.cdclk != 0 &&
1190             dev_priv->cdclk.hw.vco != 0) {
1191                 /*
1192                  * Use the current vco as our initial
1193                  * guess as to what the preferred vco is.
1194                  */
1195                 if (dev_priv->skl_preferred_vco_freq == 0)
1196                         skl_set_preferred_cdclk_vco(dev_priv,
1197                                                     dev_priv->cdclk.hw.vco);
1198                 return;
1199         }
1200
1201         cdclk_config = dev_priv->cdclk.hw;
1202
1203         cdclk_config.vco = dev_priv->skl_preferred_vco_freq;
1204         if (cdclk_config.vco == 0)
1205                 cdclk_config.vco = 8100000;
1206         cdclk_config.cdclk = skl_calc_cdclk(0, cdclk_config.vco);
1207         cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1208
1209         skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1210 }
1211
1212 static void skl_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
1213 {
1214         struct intel_cdclk_config cdclk_config = dev_priv->cdclk.hw;
1215
1216         cdclk_config.cdclk = cdclk_config.bypass;
1217         cdclk_config.vco = 0;
1218         cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1219
1220         skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1221 }
1222
1223 static bool has_cdclk_squasher(struct drm_i915_private *i915)
1224 {
1225         return IS_DG2(i915);
1226 }
1227
1228 struct intel_cdclk_vals {
1229         u32 cdclk;
1230         u16 refclk;
1231         u16 waveform;
1232         u8 divider;     /* CD2X divider * 2 */
1233         u8 ratio;
1234 };
1235
1236 static const struct intel_cdclk_vals bxt_cdclk_table[] = {
1237         { .refclk = 19200, .cdclk = 144000, .divider = 8, .ratio = 60 },
1238         { .refclk = 19200, .cdclk = 288000, .divider = 4, .ratio = 60 },
1239         { .refclk = 19200, .cdclk = 384000, .divider = 3, .ratio = 60 },
1240         { .refclk = 19200, .cdclk = 576000, .divider = 2, .ratio = 60 },
1241         { .refclk = 19200, .cdclk = 624000, .divider = 2, .ratio = 65 },
1242         {}
1243 };
1244
1245 static const struct intel_cdclk_vals glk_cdclk_table[] = {
1246         { .refclk = 19200, .cdclk =  79200, .divider = 8, .ratio = 33 },
1247         { .refclk = 19200, .cdclk = 158400, .divider = 4, .ratio = 33 },
1248         { .refclk = 19200, .cdclk = 316800, .divider = 2, .ratio = 33 },
1249         {}
1250 };
1251
1252 static const struct intel_cdclk_vals icl_cdclk_table[] = {
1253         { .refclk = 19200, .cdclk = 172800, .divider = 2, .ratio = 18 },
1254         { .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1255         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1256         { .refclk = 19200, .cdclk = 326400, .divider = 4, .ratio = 68 },
1257         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1258         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1259
1260         { .refclk = 24000, .cdclk = 180000, .divider = 2, .ratio = 15 },
1261         { .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1262         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1263         { .refclk = 24000, .cdclk = 324000, .divider = 4, .ratio = 54 },
1264         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1265         { .refclk = 24000, .cdclk = 648000, .divider = 2, .ratio = 54 },
1266
1267         { .refclk = 38400, .cdclk = 172800, .divider = 2, .ratio =  9 },
1268         { .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1269         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1270         { .refclk = 38400, .cdclk = 326400, .divider = 4, .ratio = 34 },
1271         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1272         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1273         {}
1274 };
1275
1276 static const struct intel_cdclk_vals rkl_cdclk_table[] = {
1277         { .refclk = 19200, .cdclk = 172800, .divider = 4, .ratio =  36 },
1278         { .refclk = 19200, .cdclk = 192000, .divider = 4, .ratio =  40 },
1279         { .refclk = 19200, .cdclk = 307200, .divider = 4, .ratio =  64 },
1280         { .refclk = 19200, .cdclk = 326400, .divider = 8, .ratio = 136 },
1281         { .refclk = 19200, .cdclk = 556800, .divider = 4, .ratio = 116 },
1282         { .refclk = 19200, .cdclk = 652800, .divider = 4, .ratio = 136 },
1283
1284         { .refclk = 24000, .cdclk = 180000, .divider = 4, .ratio =  30 },
1285         { .refclk = 24000, .cdclk = 192000, .divider = 4, .ratio =  32 },
1286         { .refclk = 24000, .cdclk = 312000, .divider = 4, .ratio =  52 },
1287         { .refclk = 24000, .cdclk = 324000, .divider = 8, .ratio = 108 },
1288         { .refclk = 24000, .cdclk = 552000, .divider = 4, .ratio =  92 },
1289         { .refclk = 24000, .cdclk = 648000, .divider = 4, .ratio = 108 },
1290
1291         { .refclk = 38400, .cdclk = 172800, .divider = 4, .ratio = 18 },
1292         { .refclk = 38400, .cdclk = 192000, .divider = 4, .ratio = 20 },
1293         { .refclk = 38400, .cdclk = 307200, .divider = 4, .ratio = 32 },
1294         { .refclk = 38400, .cdclk = 326400, .divider = 8, .ratio = 68 },
1295         { .refclk = 38400, .cdclk = 556800, .divider = 4, .ratio = 58 },
1296         { .refclk = 38400, .cdclk = 652800, .divider = 4, .ratio = 68 },
1297         {}
1298 };
1299
1300 static const struct intel_cdclk_vals adlp_a_step_cdclk_table[] = {
1301         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1302         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1303         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1304
1305         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1306         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1307         { .refclk = 24400, .cdclk = 648000, .divider = 2, .ratio = 54 },
1308
1309         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1310         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1311         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1312         {}
1313 };
1314
1315 static const struct intel_cdclk_vals adlp_cdclk_table[] = {
1316         { .refclk = 19200, .cdclk = 172800, .divider = 3, .ratio = 27 },
1317         { .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1318         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1319         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1320         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1321
1322         { .refclk = 24000, .cdclk = 176000, .divider = 3, .ratio = 22 },
1323         { .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1324         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1325         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1326         { .refclk = 24400, .cdclk = 648000, .divider = 2, .ratio = 54 },
1327
1328         { .refclk = 38400, .cdclk = 179200, .divider = 3, .ratio = 14 },
1329         { .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1330         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1331         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1332         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1333         {}
1334 };
1335
1336 static const struct intel_cdclk_vals dg2_cdclk_table[] = {
1337         { .refclk = 38400, .cdclk = 163200, .divider = 2, .ratio = 34, .waveform = 0x8888 },
1338         { .refclk = 38400, .cdclk = 204000, .divider = 2, .ratio = 34, .waveform = 0x9248 },
1339         { .refclk = 38400, .cdclk = 244800, .divider = 2, .ratio = 34, .waveform = 0xa4a4 },
1340         { .refclk = 38400, .cdclk = 285600, .divider = 2, .ratio = 34, .waveform = 0xa54a },
1341         { .refclk = 38400, .cdclk = 326400, .divider = 2, .ratio = 34, .waveform = 0xaaaa },
1342         { .refclk = 38400, .cdclk = 367200, .divider = 2, .ratio = 34, .waveform = 0xad5a },
1343         { .refclk = 38400, .cdclk = 408000, .divider = 2, .ratio = 34, .waveform = 0xb6b6 },
1344         { .refclk = 38400, .cdclk = 448800, .divider = 2, .ratio = 34, .waveform = 0xdbb6 },
1345         { .refclk = 38400, .cdclk = 489600, .divider = 2, .ratio = 34, .waveform = 0xeeee },
1346         { .refclk = 38400, .cdclk = 530400, .divider = 2, .ratio = 34, .waveform = 0xf7de },
1347         { .refclk = 38400, .cdclk = 571200, .divider = 2, .ratio = 34, .waveform = 0xfefe },
1348         { .refclk = 38400, .cdclk = 612000, .divider = 2, .ratio = 34, .waveform = 0xfffe },
1349         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34, .waveform = 0xffff },
1350         {}
1351 };
1352
1353 static int bxt_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
1354 {
1355         const struct intel_cdclk_vals *table = dev_priv->cdclk.table;
1356         int i;
1357
1358         for (i = 0; table[i].refclk; i++)
1359                 if (table[i].refclk == dev_priv->cdclk.hw.ref &&
1360                     table[i].cdclk >= min_cdclk)
1361                         return table[i].cdclk;
1362
1363         drm_WARN(&dev_priv->drm, 1,
1364                  "Cannot satisfy minimum cdclk %d with refclk %u\n",
1365                  min_cdclk, dev_priv->cdclk.hw.ref);
1366         return 0;
1367 }
1368
1369 static int bxt_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
1370 {
1371         const struct intel_cdclk_vals *table = dev_priv->cdclk.table;
1372         int i;
1373
1374         if (cdclk == dev_priv->cdclk.hw.bypass)
1375                 return 0;
1376
1377         for (i = 0; table[i].refclk; i++)
1378                 if (table[i].refclk == dev_priv->cdclk.hw.ref &&
1379                     table[i].cdclk == cdclk)
1380                         return dev_priv->cdclk.hw.ref * table[i].ratio;
1381
1382         drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1383                  cdclk, dev_priv->cdclk.hw.ref);
1384         return 0;
1385 }
1386
1387 static u8 bxt_calc_voltage_level(int cdclk)
1388 {
1389         return DIV_ROUND_UP(cdclk, 25000);
1390 }
1391
1392 static u8 icl_calc_voltage_level(int cdclk)
1393 {
1394         if (cdclk > 556800)
1395                 return 2;
1396         else if (cdclk > 312000)
1397                 return 1;
1398         else
1399                 return 0;
1400 }
1401
1402 static u8 ehl_calc_voltage_level(int cdclk)
1403 {
1404         if (cdclk > 326400)
1405                 return 3;
1406         else if (cdclk > 312000)
1407                 return 2;
1408         else if (cdclk > 180000)
1409                 return 1;
1410         else
1411                 return 0;
1412 }
1413
1414 static u8 tgl_calc_voltage_level(int cdclk)
1415 {
1416         if (cdclk > 556800)
1417                 return 3;
1418         else if (cdclk > 326400)
1419                 return 2;
1420         else if (cdclk > 312000)
1421                 return 1;
1422         else
1423                 return 0;
1424 }
1425
1426 static void icl_readout_refclk(struct drm_i915_private *dev_priv,
1427                                struct intel_cdclk_config *cdclk_config)
1428 {
1429         u32 dssm = intel_de_read(dev_priv, SKL_DSSM) & ICL_DSSM_CDCLK_PLL_REFCLK_MASK;
1430
1431         switch (dssm) {
1432         default:
1433                 MISSING_CASE(dssm);
1434                 fallthrough;
1435         case ICL_DSSM_CDCLK_PLL_REFCLK_24MHz:
1436                 cdclk_config->ref = 24000;
1437                 break;
1438         case ICL_DSSM_CDCLK_PLL_REFCLK_19_2MHz:
1439                 cdclk_config->ref = 19200;
1440                 break;
1441         case ICL_DSSM_CDCLK_PLL_REFCLK_38_4MHz:
1442                 cdclk_config->ref = 38400;
1443                 break;
1444         }
1445 }
1446
1447 static void bxt_de_pll_readout(struct drm_i915_private *dev_priv,
1448                                struct intel_cdclk_config *cdclk_config)
1449 {
1450         u32 val, ratio;
1451
1452         if (IS_DG2(dev_priv))
1453                 cdclk_config->ref = 38400;
1454         else if (DISPLAY_VER(dev_priv) >= 11)
1455                 icl_readout_refclk(dev_priv, cdclk_config);
1456         else
1457                 cdclk_config->ref = 19200;
1458
1459         val = intel_de_read(dev_priv, BXT_DE_PLL_ENABLE);
1460         if ((val & BXT_DE_PLL_PLL_ENABLE) == 0 ||
1461             (val & BXT_DE_PLL_LOCK) == 0) {
1462                 /*
1463                  * CDCLK PLL is disabled, the VCO/ratio doesn't matter, but
1464                  * setting it to zero is a way to signal that.
1465                  */
1466                 cdclk_config->vco = 0;
1467                 return;
1468         }
1469
1470         /*
1471          * DISPLAY_VER >= 11 have the ratio directly in the PLL enable register,
1472          * gen9lp had it in a separate PLL control register.
1473          */
1474         if (DISPLAY_VER(dev_priv) >= 11)
1475                 ratio = val & ICL_CDCLK_PLL_RATIO_MASK;
1476         else
1477                 ratio = intel_de_read(dev_priv, BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK;
1478
1479         cdclk_config->vco = ratio * cdclk_config->ref;
1480 }
1481
1482 static void bxt_get_cdclk(struct drm_i915_private *dev_priv,
1483                           struct intel_cdclk_config *cdclk_config)
1484 {
1485         u32 squash_ctl = 0;
1486         u32 divider;
1487         int div;
1488
1489         bxt_de_pll_readout(dev_priv, cdclk_config);
1490
1491         if (DISPLAY_VER(dev_priv) >= 12)
1492                 cdclk_config->bypass = cdclk_config->ref / 2;
1493         else if (DISPLAY_VER(dev_priv) >= 11)
1494                 cdclk_config->bypass = 50000;
1495         else
1496                 cdclk_config->bypass = cdclk_config->ref;
1497
1498         if (cdclk_config->vco == 0) {
1499                 cdclk_config->cdclk = cdclk_config->bypass;
1500                 goto out;
1501         }
1502
1503         divider = intel_de_read(dev_priv, CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
1504
1505         switch (divider) {
1506         case BXT_CDCLK_CD2X_DIV_SEL_1:
1507                 div = 2;
1508                 break;
1509         case BXT_CDCLK_CD2X_DIV_SEL_1_5:
1510                 div = 3;
1511                 break;
1512         case BXT_CDCLK_CD2X_DIV_SEL_2:
1513                 div = 4;
1514                 break;
1515         case BXT_CDCLK_CD2X_DIV_SEL_4:
1516                 div = 8;
1517                 break;
1518         default:
1519                 MISSING_CASE(divider);
1520                 return;
1521         }
1522
1523         if (has_cdclk_squasher(dev_priv))
1524                 squash_ctl = intel_de_read(dev_priv, CDCLK_SQUASH_CTL);
1525
1526         if (squash_ctl & CDCLK_SQUASH_ENABLE) {
1527                 u16 waveform;
1528                 int size;
1529
1530                 size = REG_FIELD_GET(CDCLK_SQUASH_WINDOW_SIZE_MASK, squash_ctl) + 1;
1531                 waveform = REG_FIELD_GET(CDCLK_SQUASH_WAVEFORM_MASK, squash_ctl) >> (16 - size);
1532
1533                 cdclk_config->cdclk = DIV_ROUND_CLOSEST(hweight16(waveform) *
1534                                                         cdclk_config->vco, size * div);
1535         } else {
1536                 cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco, div);
1537         }
1538
1539  out:
1540         /*
1541          * Can't read this out :( Let's assume it's
1542          * at least what the CDCLK frequency requires.
1543          */
1544         cdclk_config->voltage_level =
1545                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config->cdclk);
1546 }
1547
1548 static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
1549 {
1550         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, 0);
1551
1552         /* Timeout 200us */
1553         if (intel_de_wait_for_clear(dev_priv,
1554                                     BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1555                 drm_err(&dev_priv->drm, "timeout waiting for DE PLL unlock\n");
1556
1557         dev_priv->cdclk.hw.vco = 0;
1558 }
1559
1560 static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
1561 {
1562         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
1563
1564         intel_de_rmw(dev_priv, BXT_DE_PLL_CTL,
1565                      BXT_DE_PLL_RATIO_MASK, BXT_DE_PLL_RATIO(ratio));
1566
1567         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
1568
1569         /* Timeout 200us */
1570         if (intel_de_wait_for_set(dev_priv,
1571                                   BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1572                 drm_err(&dev_priv->drm, "timeout waiting for DE PLL lock\n");
1573
1574         dev_priv->cdclk.hw.vco = vco;
1575 }
1576
1577 static void icl_cdclk_pll_disable(struct drm_i915_private *dev_priv)
1578 {
1579         intel_de_rmw(dev_priv, BXT_DE_PLL_ENABLE,
1580                      BXT_DE_PLL_PLL_ENABLE, 0);
1581
1582         /* Timeout 200us */
1583         if (intel_de_wait_for_clear(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1584                 drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL unlock\n");
1585
1586         dev_priv->cdclk.hw.vco = 0;
1587 }
1588
1589 static void icl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco)
1590 {
1591         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
1592         u32 val;
1593
1594         val = ICL_CDCLK_PLL_RATIO(ratio);
1595         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1596
1597         val |= BXT_DE_PLL_PLL_ENABLE;
1598         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1599
1600         /* Timeout 200us */
1601         if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1602                 drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL lock\n");
1603
1604         dev_priv->cdclk.hw.vco = vco;
1605 }
1606
1607 static void adlp_cdclk_pll_crawl(struct drm_i915_private *dev_priv, int vco)
1608 {
1609         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
1610         u32 val;
1611
1612         /* Write PLL ratio without disabling */
1613         val = ICL_CDCLK_PLL_RATIO(ratio) | BXT_DE_PLL_PLL_ENABLE;
1614         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1615
1616         /* Submit freq change request */
1617         val |= BXT_DE_PLL_FREQ_REQ;
1618         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1619
1620         /* Timeout 200us */
1621         if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE,
1622                                   BXT_DE_PLL_LOCK | BXT_DE_PLL_FREQ_REQ_ACK, 1))
1623                 drm_err(&dev_priv->drm, "timeout waiting for FREQ change request ack\n");
1624
1625         val &= ~BXT_DE_PLL_FREQ_REQ;
1626         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1627
1628         dev_priv->cdclk.hw.vco = vco;
1629 }
1630
1631 static u32 bxt_cdclk_cd2x_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
1632 {
1633         if (DISPLAY_VER(dev_priv) >= 12) {
1634                 if (pipe == INVALID_PIPE)
1635                         return TGL_CDCLK_CD2X_PIPE_NONE;
1636                 else
1637                         return TGL_CDCLK_CD2X_PIPE(pipe);
1638         } else if (DISPLAY_VER(dev_priv) >= 11) {
1639                 if (pipe == INVALID_PIPE)
1640                         return ICL_CDCLK_CD2X_PIPE_NONE;
1641                 else
1642                         return ICL_CDCLK_CD2X_PIPE(pipe);
1643         } else {
1644                 if (pipe == INVALID_PIPE)
1645                         return BXT_CDCLK_CD2X_PIPE_NONE;
1646                 else
1647                         return BXT_CDCLK_CD2X_PIPE(pipe);
1648         }
1649 }
1650
1651 static u32 bxt_cdclk_cd2x_div_sel(struct drm_i915_private *dev_priv,
1652                                   int cdclk, int vco)
1653 {
1654         /* cdclk = vco / 2 / div{1,1.5,2,4} */
1655         switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
1656         default:
1657                 drm_WARN_ON(&dev_priv->drm,
1658                             cdclk != dev_priv->cdclk.hw.bypass);
1659                 drm_WARN_ON(&dev_priv->drm, vco != 0);
1660                 fallthrough;
1661         case 2:
1662                 return BXT_CDCLK_CD2X_DIV_SEL_1;
1663         case 3:
1664                 return BXT_CDCLK_CD2X_DIV_SEL_1_5;
1665         case 4:
1666                 return BXT_CDCLK_CD2X_DIV_SEL_2;
1667         case 8:
1668                 return BXT_CDCLK_CD2X_DIV_SEL_4;
1669         }
1670 }
1671
1672 static u32 cdclk_squash_waveform(struct drm_i915_private *dev_priv,
1673                                  int cdclk)
1674 {
1675         const struct intel_cdclk_vals *table = dev_priv->cdclk.table;
1676         int i;
1677
1678         if (cdclk == dev_priv->cdclk.hw.bypass)
1679                 return 0;
1680
1681         for (i = 0; table[i].refclk; i++)
1682                 if (table[i].refclk == dev_priv->cdclk.hw.ref &&
1683                     table[i].cdclk == cdclk)
1684                         return table[i].waveform;
1685
1686         drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1687                  cdclk, dev_priv->cdclk.hw.ref);
1688
1689         return 0xffff;
1690 }
1691
1692 static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
1693                           const struct intel_cdclk_config *cdclk_config,
1694                           enum pipe pipe)
1695 {
1696         int cdclk = cdclk_config->cdclk;
1697         int vco = cdclk_config->vco;
1698         u32 val;
1699         u16 waveform;
1700         int clock;
1701         int ret;
1702
1703         /* Inform power controller of upcoming frequency change. */
1704         if (DISPLAY_VER(dev_priv) >= 11)
1705                 ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1706                                         SKL_CDCLK_PREPARE_FOR_CHANGE,
1707                                         SKL_CDCLK_READY_FOR_CHANGE,
1708                                         SKL_CDCLK_READY_FOR_CHANGE, 3);
1709         else
1710                 /*
1711                  * BSpec requires us to wait up to 150usec, but that leads to
1712                  * timeouts; the 2ms used here is based on experiment.
1713                  */
1714                 ret = snb_pcode_write_timeout(dev_priv,
1715                                               HSW_PCODE_DE_WRITE_FREQ_REQ,
1716                                               0x80000000, 150, 2);
1717         if (ret) {
1718                 drm_err(&dev_priv->drm,
1719                         "Failed to inform PCU about cdclk change (err %d, freq %d)\n",
1720                         ret, cdclk);
1721                 return;
1722         }
1723
1724         if (HAS_CDCLK_CRAWL(dev_priv) && dev_priv->cdclk.hw.vco > 0 && vco > 0) {
1725                 if (dev_priv->cdclk.hw.vco != vco)
1726                         adlp_cdclk_pll_crawl(dev_priv, vco);
1727         } else if (DISPLAY_VER(dev_priv) >= 11) {
1728                 if (dev_priv->cdclk.hw.vco != 0 &&
1729                     dev_priv->cdclk.hw.vco != vco)
1730                         icl_cdclk_pll_disable(dev_priv);
1731
1732                 if (dev_priv->cdclk.hw.vco != vco)
1733                         icl_cdclk_pll_enable(dev_priv, vco);
1734         } else {
1735                 if (dev_priv->cdclk.hw.vco != 0 &&
1736                     dev_priv->cdclk.hw.vco != vco)
1737                         bxt_de_pll_disable(dev_priv);
1738
1739                 if (dev_priv->cdclk.hw.vco != vco)
1740                         bxt_de_pll_enable(dev_priv, vco);
1741         }
1742
1743         waveform = cdclk_squash_waveform(dev_priv, cdclk);
1744
1745         if (waveform)
1746                 clock = vco / 2;
1747         else
1748                 clock = cdclk;
1749
1750         if (has_cdclk_squasher(dev_priv)) {
1751                 u32 squash_ctl = 0;
1752
1753                 if (waveform)
1754                         squash_ctl = CDCLK_SQUASH_ENABLE |
1755                                 CDCLK_SQUASH_WINDOW_SIZE(0xf) | waveform;
1756
1757                 intel_de_write(dev_priv, CDCLK_SQUASH_CTL, squash_ctl);
1758         }
1759
1760         val = bxt_cdclk_cd2x_div_sel(dev_priv, clock, vco) |
1761                 bxt_cdclk_cd2x_pipe(dev_priv, pipe) |
1762                 skl_cdclk_decimal(cdclk);
1763
1764         /*
1765          * Disable SSA Precharge when CD clock frequency < 500 MHz,
1766          * enable otherwise.
1767          */
1768         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
1769             cdclk >= 500000)
1770                 val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
1771         intel_de_write(dev_priv, CDCLK_CTL, val);
1772
1773         if (pipe != INVALID_PIPE)
1774                 intel_crtc_wait_for_next_vblank(intel_crtc_for_pipe(dev_priv, pipe));
1775
1776         if (DISPLAY_VER(dev_priv) >= 11) {
1777                 ret = snb_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1778                                       cdclk_config->voltage_level);
1779         } else {
1780                 /*
1781                  * The timeout isn't specified, the 2ms used here is based on
1782                  * experiment.
1783                  * FIXME: Waiting for the request completion could be delayed
1784                  * until the next PCODE request based on BSpec.
1785                  */
1786                 ret = snb_pcode_write_timeout(dev_priv,
1787                                               HSW_PCODE_DE_WRITE_FREQ_REQ,
1788                                               cdclk_config->voltage_level,
1789                                               150, 2);
1790         }
1791
1792         if (ret) {
1793                 drm_err(&dev_priv->drm,
1794                         "PCode CDCLK freq set failed, (err %d, freq %d)\n",
1795                         ret, cdclk);
1796                 return;
1797         }
1798
1799         intel_update_cdclk(dev_priv);
1800
1801         if (DISPLAY_VER(dev_priv) >= 11)
1802                 /*
1803                  * Can't read out the voltage level :(
1804                  * Let's just assume everything is as expected.
1805                  */
1806                 dev_priv->cdclk.hw.voltage_level = cdclk_config->voltage_level;
1807 }
1808
1809 static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
1810 {
1811         u32 cdctl, expected;
1812         int cdclk, clock, vco;
1813
1814         intel_update_cdclk(dev_priv);
1815         intel_cdclk_dump_config(dev_priv, &dev_priv->cdclk.hw, "Current CDCLK");
1816
1817         if (dev_priv->cdclk.hw.vco == 0 ||
1818             dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass)
1819                 goto sanitize;
1820
1821         /* DPLL okay; verify the cdclock
1822          *
1823          * Some BIOS versions leave an incorrect decimal frequency value and
1824          * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4,
1825          * so sanitize this register.
1826          */
1827         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1828         /*
1829          * Let's ignore the pipe field, since BIOS could have configured the
1830          * dividers both synching to an active pipe, or asynchronously
1831          * (PIPE_NONE).
1832          */
1833         cdctl &= ~bxt_cdclk_cd2x_pipe(dev_priv, INVALID_PIPE);
1834
1835         /* Make sure this is a legal cdclk value for the platform */
1836         cdclk = bxt_calc_cdclk(dev_priv, dev_priv->cdclk.hw.cdclk);
1837         if (cdclk != dev_priv->cdclk.hw.cdclk)
1838                 goto sanitize;
1839
1840         /* Make sure the VCO is correct for the cdclk */
1841         vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
1842         if (vco != dev_priv->cdclk.hw.vco)
1843                 goto sanitize;
1844
1845         expected = skl_cdclk_decimal(cdclk);
1846
1847         /* Figure out what CD2X divider we should be using for this cdclk */
1848         if (has_cdclk_squasher(dev_priv))
1849                 clock = dev_priv->cdclk.hw.vco / 2;
1850         else
1851                 clock = dev_priv->cdclk.hw.cdclk;
1852
1853         expected |= bxt_cdclk_cd2x_div_sel(dev_priv, clock,
1854                                            dev_priv->cdclk.hw.vco);
1855
1856         /*
1857          * Disable SSA Precharge when CD clock frequency < 500 MHz,
1858          * enable otherwise.
1859          */
1860         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
1861             dev_priv->cdclk.hw.cdclk >= 500000)
1862                 expected |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
1863
1864         if (cdctl == expected)
1865                 /* All well; nothing to sanitize */
1866                 return;
1867
1868 sanitize:
1869         drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1870
1871         /* force cdclk programming */
1872         dev_priv->cdclk.hw.cdclk = 0;
1873
1874         /* force full PLL disable + enable */
1875         dev_priv->cdclk.hw.vco = -1;
1876 }
1877
1878 static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv)
1879 {
1880         struct intel_cdclk_config cdclk_config;
1881
1882         bxt_sanitize_cdclk(dev_priv);
1883
1884         if (dev_priv->cdclk.hw.cdclk != 0 &&
1885             dev_priv->cdclk.hw.vco != 0)
1886                 return;
1887
1888         cdclk_config = dev_priv->cdclk.hw;
1889
1890         /*
1891          * FIXME:
1892          * - The initial CDCLK needs to be read from VBT.
1893          *   Need to make this change after VBT has changes for BXT.
1894          */
1895         cdclk_config.cdclk = bxt_calc_cdclk(dev_priv, 0);
1896         cdclk_config.vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk_config.cdclk);
1897         cdclk_config.voltage_level =
1898                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk);
1899
1900         bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1901 }
1902
1903 static void bxt_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
1904 {
1905         struct intel_cdclk_config cdclk_config = dev_priv->cdclk.hw;
1906
1907         cdclk_config.cdclk = cdclk_config.bypass;
1908         cdclk_config.vco = 0;
1909         cdclk_config.voltage_level =
1910                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk);
1911
1912         bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1913 }
1914
1915 /**
1916  * intel_cdclk_init_hw - Initialize CDCLK hardware
1917  * @i915: i915 device
1918  *
1919  * Initialize CDCLK. This consists mainly of initializing dev_priv->cdclk.hw and
1920  * sanitizing the state of the hardware if needed. This is generally done only
1921  * during the display core initialization sequence, after which the DMC will
1922  * take care of turning CDCLK off/on as needed.
1923  */
1924 void intel_cdclk_init_hw(struct drm_i915_private *i915)
1925 {
1926         if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
1927                 bxt_cdclk_init_hw(i915);
1928         else if (DISPLAY_VER(i915) == 9)
1929                 skl_cdclk_init_hw(i915);
1930 }
1931
1932 /**
1933  * intel_cdclk_uninit_hw - Uninitialize CDCLK hardware
1934  * @i915: i915 device
1935  *
1936  * Uninitialize CDCLK. This is done only during the display core
1937  * uninitialization sequence.
1938  */
1939 void intel_cdclk_uninit_hw(struct drm_i915_private *i915)
1940 {
1941         if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
1942                 bxt_cdclk_uninit_hw(i915);
1943         else if (DISPLAY_VER(i915) == 9)
1944                 skl_cdclk_uninit_hw(i915);
1945 }
1946
1947 static bool intel_cdclk_can_crawl(struct drm_i915_private *dev_priv,
1948                                   const struct intel_cdclk_config *a,
1949                                   const struct intel_cdclk_config *b)
1950 {
1951         int a_div, b_div;
1952
1953         if (!HAS_CDCLK_CRAWL(dev_priv))
1954                 return false;
1955
1956         /*
1957          * The vco and cd2x divider will change independently
1958          * from each, so we disallow cd2x change when crawling.
1959          */
1960         a_div = DIV_ROUND_CLOSEST(a->vco, a->cdclk);
1961         b_div = DIV_ROUND_CLOSEST(b->vco, b->cdclk);
1962
1963         return a->vco != 0 && b->vco != 0 &&
1964                 a->vco != b->vco &&
1965                 a_div == b_div &&
1966                 a->ref == b->ref;
1967 }
1968
1969 static bool intel_cdclk_can_squash(struct drm_i915_private *dev_priv,
1970                                    const struct intel_cdclk_config *a,
1971                                    const struct intel_cdclk_config *b)
1972 {
1973         /*
1974          * FIXME should store a bit more state in intel_cdclk_config
1975          * to differentiate squasher vs. cd2x divider properly. For
1976          * the moment all platforms with squasher use a fixed cd2x
1977          * divider.
1978          */
1979         if (!has_cdclk_squasher(dev_priv))
1980                 return false;
1981
1982         return a->cdclk != b->cdclk &&
1983                 a->vco != 0 &&
1984                 a->vco == b->vco &&
1985                 a->ref == b->ref;
1986 }
1987
1988 /**
1989  * intel_cdclk_needs_modeset - Determine if changong between the CDCLK
1990  *                             configurations requires a modeset on all pipes
1991  * @a: first CDCLK configuration
1992  * @b: second CDCLK configuration
1993  *
1994  * Returns:
1995  * True if changing between the two CDCLK configurations
1996  * requires all pipes to be off, false if not.
1997  */
1998 bool intel_cdclk_needs_modeset(const struct intel_cdclk_config *a,
1999                                const struct intel_cdclk_config *b)
2000 {
2001         return a->cdclk != b->cdclk ||
2002                 a->vco != b->vco ||
2003                 a->ref != b->ref;
2004 }
2005
2006 /**
2007  * intel_cdclk_can_cd2x_update - Determine if changing between the two CDCLK
2008  *                               configurations requires only a cd2x divider update
2009  * @dev_priv: i915 device
2010  * @a: first CDCLK configuration
2011  * @b: second CDCLK configuration
2012  *
2013  * Returns:
2014  * True if changing between the two CDCLK configurations
2015  * can be done with just a cd2x divider update, false if not.
2016  */
2017 static bool intel_cdclk_can_cd2x_update(struct drm_i915_private *dev_priv,
2018                                         const struct intel_cdclk_config *a,
2019                                         const struct intel_cdclk_config *b)
2020 {
2021         /* Older hw doesn't have the capability */
2022         if (DISPLAY_VER(dev_priv) < 10 && !IS_BROXTON(dev_priv))
2023                 return false;
2024
2025         /*
2026          * FIXME should store a bit more state in intel_cdclk_config
2027          * to differentiate squasher vs. cd2x divider properly. For
2028          * the moment all platforms with squasher use a fixed cd2x
2029          * divider.
2030          */
2031         if (has_cdclk_squasher(dev_priv))
2032                 return false;
2033
2034         return a->cdclk != b->cdclk &&
2035                 a->vco != 0 &&
2036                 a->vco == b->vco &&
2037                 a->ref == b->ref;
2038 }
2039
2040 /**
2041  * intel_cdclk_changed - Determine if two CDCLK configurations are different
2042  * @a: first CDCLK configuration
2043  * @b: second CDCLK configuration
2044  *
2045  * Returns:
2046  * True if the CDCLK configurations don't match, false if they do.
2047  */
2048 static bool intel_cdclk_changed(const struct intel_cdclk_config *a,
2049                                 const struct intel_cdclk_config *b)
2050 {
2051         return intel_cdclk_needs_modeset(a, b) ||
2052                 a->voltage_level != b->voltage_level;
2053 }
2054
2055 void intel_cdclk_dump_config(struct drm_i915_private *i915,
2056                              const struct intel_cdclk_config *cdclk_config,
2057                              const char *context)
2058 {
2059         drm_dbg_kms(&i915->drm, "%s %d kHz, VCO %d kHz, ref %d kHz, bypass %d kHz, voltage level %d\n",
2060                     context, cdclk_config->cdclk, cdclk_config->vco,
2061                     cdclk_config->ref, cdclk_config->bypass,
2062                     cdclk_config->voltage_level);
2063 }
2064
2065 /**
2066  * intel_set_cdclk - Push the CDCLK configuration to the hardware
2067  * @dev_priv: i915 device
2068  * @cdclk_config: new CDCLK configuration
2069  * @pipe: pipe with which to synchronize the update
2070  *
2071  * Program the hardware based on the passed in CDCLK state,
2072  * if necessary.
2073  */
2074 static void intel_set_cdclk(struct drm_i915_private *dev_priv,
2075                             const struct intel_cdclk_config *cdclk_config,
2076                             enum pipe pipe)
2077 {
2078         struct intel_encoder *encoder;
2079
2080         if (!intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config))
2081                 return;
2082
2083         if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->cdclk_funcs->set_cdclk))
2084                 return;
2085
2086         intel_cdclk_dump_config(dev_priv, cdclk_config, "Changing CDCLK to");
2087
2088         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2089                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2090
2091                 intel_psr_pause(intel_dp);
2092         }
2093
2094         intel_audio_cdclk_change_pre(dev_priv);
2095
2096         /*
2097          * Lock aux/gmbus while we change cdclk in case those
2098          * functions use cdclk. Not all platforms/ports do,
2099          * but we'll lock them all for simplicity.
2100          */
2101         mutex_lock(&dev_priv->gmbus_mutex);
2102         for_each_intel_dp(&dev_priv->drm, encoder) {
2103                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2104
2105                 mutex_lock_nest_lock(&intel_dp->aux.hw_mutex,
2106                                      &dev_priv->gmbus_mutex);
2107         }
2108
2109         intel_cdclk_set_cdclk(dev_priv, cdclk_config, pipe);
2110
2111         for_each_intel_dp(&dev_priv->drm, encoder) {
2112                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2113
2114                 mutex_unlock(&intel_dp->aux.hw_mutex);
2115         }
2116         mutex_unlock(&dev_priv->gmbus_mutex);
2117
2118         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2119                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2120
2121                 intel_psr_resume(intel_dp);
2122         }
2123
2124         intel_audio_cdclk_change_post(dev_priv);
2125
2126         if (drm_WARN(&dev_priv->drm,
2127                      intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config),
2128                      "cdclk state doesn't match!\n")) {
2129                 intel_cdclk_dump_config(dev_priv, &dev_priv->cdclk.hw, "[hw state]");
2130                 intel_cdclk_dump_config(dev_priv, cdclk_config, "[sw state]");
2131         }
2132 }
2133
2134 /**
2135  * intel_set_cdclk_pre_plane_update - Push the CDCLK state to the hardware
2136  * @state: intel atomic state
2137  *
2138  * Program the hardware before updating the HW plane state based on the
2139  * new CDCLK state, if necessary.
2140  */
2141 void
2142 intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state)
2143 {
2144         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2145         const struct intel_cdclk_state *old_cdclk_state =
2146                 intel_atomic_get_old_cdclk_state(state);
2147         const struct intel_cdclk_state *new_cdclk_state =
2148                 intel_atomic_get_new_cdclk_state(state);
2149         enum pipe pipe = new_cdclk_state->pipe;
2150
2151         if (!intel_cdclk_changed(&old_cdclk_state->actual,
2152                                  &new_cdclk_state->actual))
2153                 return;
2154
2155         if (pipe == INVALID_PIPE ||
2156             old_cdclk_state->actual.cdclk <= new_cdclk_state->actual.cdclk) {
2157                 drm_WARN_ON(&dev_priv->drm, !new_cdclk_state->base.changed);
2158
2159                 intel_set_cdclk(dev_priv, &new_cdclk_state->actual, pipe);
2160         }
2161 }
2162
2163 /**
2164  * intel_set_cdclk_post_plane_update - Push the CDCLK state to the hardware
2165  * @state: intel atomic state
2166  *
2167  * Program the hardware after updating the HW plane state based on the
2168  * new CDCLK state, if necessary.
2169  */
2170 void
2171 intel_set_cdclk_post_plane_update(struct intel_atomic_state *state)
2172 {
2173         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2174         const struct intel_cdclk_state *old_cdclk_state =
2175                 intel_atomic_get_old_cdclk_state(state);
2176         const struct intel_cdclk_state *new_cdclk_state =
2177                 intel_atomic_get_new_cdclk_state(state);
2178         enum pipe pipe = new_cdclk_state->pipe;
2179
2180         if (!intel_cdclk_changed(&old_cdclk_state->actual,
2181                                  &new_cdclk_state->actual))
2182                 return;
2183
2184         if (pipe != INVALID_PIPE &&
2185             old_cdclk_state->actual.cdclk > new_cdclk_state->actual.cdclk) {
2186                 drm_WARN_ON(&dev_priv->drm, !new_cdclk_state->base.changed);
2187
2188                 intel_set_cdclk(dev_priv, &new_cdclk_state->actual, pipe);
2189         }
2190 }
2191
2192 static int intel_pixel_rate_to_cdclk(const struct intel_crtc_state *crtc_state)
2193 {
2194         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2195         int pixel_rate = crtc_state->pixel_rate;
2196
2197         if (DISPLAY_VER(dev_priv) >= 10)
2198                 return DIV_ROUND_UP(pixel_rate, 2);
2199         else if (DISPLAY_VER(dev_priv) == 9 ||
2200                  IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2201                 return pixel_rate;
2202         else if (IS_CHERRYVIEW(dev_priv))
2203                 return DIV_ROUND_UP(pixel_rate * 100, 95);
2204         else if (crtc_state->double_wide)
2205                 return DIV_ROUND_UP(pixel_rate * 100, 90 * 2);
2206         else
2207                 return DIV_ROUND_UP(pixel_rate * 100, 90);
2208 }
2209
2210 static int intel_planes_min_cdclk(const struct intel_crtc_state *crtc_state)
2211 {
2212         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2213         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2214         struct intel_plane *plane;
2215         int min_cdclk = 0;
2216
2217         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
2218                 min_cdclk = max(crtc_state->min_cdclk[plane->id], min_cdclk);
2219
2220         return min_cdclk;
2221 }
2222
2223 int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
2224 {
2225         struct drm_i915_private *dev_priv =
2226                 to_i915(crtc_state->uapi.crtc->dev);
2227         int min_cdclk;
2228
2229         if (!crtc_state->hw.enable)
2230                 return 0;
2231
2232         min_cdclk = intel_pixel_rate_to_cdclk(crtc_state);
2233
2234         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
2235         if (IS_BROADWELL(dev_priv) && hsw_crtc_state_ips_capable(crtc_state))
2236                 min_cdclk = DIV_ROUND_UP(min_cdclk * 100, 95);
2237
2238         /* BSpec says "Do not use DisplayPort with CDCLK less than 432 MHz,
2239          * audio enabled, port width x4, and link rate HBR2 (5.4 GHz), or else
2240          * there may be audio corruption or screen corruption." This cdclk
2241          * restriction for GLK is 316.8 MHz.
2242          */
2243         if (intel_crtc_has_dp_encoder(crtc_state) &&
2244             crtc_state->has_audio &&
2245             crtc_state->port_clock >= 540000 &&
2246             crtc_state->lane_count == 4) {
2247                 if (DISPLAY_VER(dev_priv) == 10) {
2248                         /* Display WA #1145: glk */
2249                         min_cdclk = max(316800, min_cdclk);
2250                 } else if (DISPLAY_VER(dev_priv) == 9 || IS_BROADWELL(dev_priv)) {
2251                         /* Display WA #1144: skl,bxt */
2252                         min_cdclk = max(432000, min_cdclk);
2253                 }
2254         }
2255
2256         /*
2257          * According to BSpec, "The CD clock frequency must be at least twice
2258          * the frequency of the Azalia BCLK." and BCLK is 96 MHz by default.
2259          */
2260         if (crtc_state->has_audio && DISPLAY_VER(dev_priv) >= 9)
2261                 min_cdclk = max(2 * 96000, min_cdclk);
2262
2263         /*
2264          * "For DP audio configuration, cdclk frequency shall be set to
2265          *  meet the following requirements:
2266          *  DP Link Frequency(MHz) | Cdclk frequency(MHz)
2267          *  270                    | 320 or higher
2268          *  162                    | 200 or higher"
2269          */
2270         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
2271             intel_crtc_has_dp_encoder(crtc_state) && crtc_state->has_audio)
2272                 min_cdclk = max(crtc_state->port_clock, min_cdclk);
2273
2274         /*
2275          * On Valleyview some DSI panels lose (v|h)sync when the clock is lower
2276          * than 320000KHz.
2277          */
2278         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2279             IS_VALLEYVIEW(dev_priv))
2280                 min_cdclk = max(320000, min_cdclk);
2281
2282         /*
2283          * On Geminilake once the CDCLK gets as low as 79200
2284          * picture gets unstable, despite that values are
2285          * correct for DSI PLL and DE PLL.
2286          */
2287         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2288             IS_GEMINILAKE(dev_priv))
2289                 min_cdclk = max(158400, min_cdclk);
2290
2291         /* Account for additional needs from the planes */
2292         min_cdclk = max(intel_planes_min_cdclk(crtc_state), min_cdclk);
2293
2294         /*
2295          * When we decide to use only one VDSC engine, since
2296          * each VDSC operates with 1 ppc throughput, pixel clock
2297          * cannot be higher than the VDSC clock (cdclk)
2298          */
2299         if (crtc_state->dsc.compression_enable && !crtc_state->dsc.dsc_split)
2300                 min_cdclk = max(min_cdclk, (int)crtc_state->pixel_rate);
2301
2302         /*
2303          * HACK. Currently for TGL platforms we calculate
2304          * min_cdclk initially based on pixel_rate divided
2305          * by 2, accounting for also plane requirements,
2306          * however in some cases the lowest possible CDCLK
2307          * doesn't work and causing the underruns.
2308          * Explicitly stating here that this seems to be currently
2309          * rather a Hack, than final solution.
2310          */
2311         if (IS_TIGERLAKE(dev_priv)) {
2312                 /*
2313                  * Clamp to max_cdclk_freq in case pixel rate is higher,
2314                  * in order not to break an 8K, but still leave W/A at place.
2315                  */
2316                 min_cdclk = max_t(int, min_cdclk,
2317                                   min_t(int, crtc_state->pixel_rate,
2318                                         dev_priv->max_cdclk_freq));
2319         }
2320
2321         return min_cdclk;
2322 }
2323
2324 static int intel_compute_min_cdclk(struct intel_cdclk_state *cdclk_state)
2325 {
2326         struct intel_atomic_state *state = cdclk_state->base.state;
2327         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2328         const struct intel_bw_state *bw_state;
2329         struct intel_crtc *crtc;
2330         struct intel_crtc_state *crtc_state;
2331         int min_cdclk, i;
2332         enum pipe pipe;
2333
2334         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2335                 int ret;
2336
2337                 min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
2338                 if (min_cdclk < 0)
2339                         return min_cdclk;
2340
2341                 if (cdclk_state->min_cdclk[crtc->pipe] == min_cdclk)
2342                         continue;
2343
2344                 cdclk_state->min_cdclk[crtc->pipe] = min_cdclk;
2345
2346                 ret = intel_atomic_lock_global_state(&cdclk_state->base);
2347                 if (ret)
2348                         return ret;
2349         }
2350
2351         bw_state = intel_atomic_get_new_bw_state(state);
2352         if (bw_state) {
2353                 min_cdclk = intel_bw_min_cdclk(dev_priv, bw_state);
2354
2355                 if (cdclk_state->bw_min_cdclk != min_cdclk) {
2356                         int ret;
2357
2358                         cdclk_state->bw_min_cdclk = min_cdclk;
2359
2360                         ret = intel_atomic_lock_global_state(&cdclk_state->base);
2361                         if (ret)
2362                                 return ret;
2363                 }
2364         }
2365
2366         min_cdclk = max(cdclk_state->force_min_cdclk,
2367                         cdclk_state->bw_min_cdclk);
2368         for_each_pipe(dev_priv, pipe)
2369                 min_cdclk = max(cdclk_state->min_cdclk[pipe], min_cdclk);
2370
2371         if (min_cdclk > dev_priv->max_cdclk_freq) {
2372                 drm_dbg_kms(&dev_priv->drm,
2373                             "required cdclk (%d kHz) exceeds max (%d kHz)\n",
2374                             min_cdclk, dev_priv->max_cdclk_freq);
2375                 return -EINVAL;
2376         }
2377
2378         return min_cdclk;
2379 }
2380
2381 /*
2382  * Account for port clock min voltage level requirements.
2383  * This only really does something on DISPLA_VER >= 11 but can be
2384  * called on earlier platforms as well.
2385  *
2386  * Note that this functions assumes that 0 is
2387  * the lowest voltage value, and higher values
2388  * correspond to increasingly higher voltages.
2389  *
2390  * Should that relationship no longer hold on
2391  * future platforms this code will need to be
2392  * adjusted.
2393  */
2394 static int bxt_compute_min_voltage_level(struct intel_cdclk_state *cdclk_state)
2395 {
2396         struct intel_atomic_state *state = cdclk_state->base.state;
2397         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2398         struct intel_crtc *crtc;
2399         struct intel_crtc_state *crtc_state;
2400         u8 min_voltage_level;
2401         int i;
2402         enum pipe pipe;
2403
2404         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2405                 int ret;
2406
2407                 if (crtc_state->hw.enable)
2408                         min_voltage_level = crtc_state->min_voltage_level;
2409                 else
2410                         min_voltage_level = 0;
2411
2412                 if (cdclk_state->min_voltage_level[crtc->pipe] == min_voltage_level)
2413                         continue;
2414
2415                 cdclk_state->min_voltage_level[crtc->pipe] = min_voltage_level;
2416
2417                 ret = intel_atomic_lock_global_state(&cdclk_state->base);
2418                 if (ret)
2419                         return ret;
2420         }
2421
2422         min_voltage_level = 0;
2423         for_each_pipe(dev_priv, pipe)
2424                 min_voltage_level = max(cdclk_state->min_voltage_level[pipe],
2425                                         min_voltage_level);
2426
2427         return min_voltage_level;
2428 }
2429
2430 static int vlv_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2431 {
2432         struct intel_atomic_state *state = cdclk_state->base.state;
2433         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2434         int min_cdclk, cdclk;
2435
2436         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2437         if (min_cdclk < 0)
2438                 return min_cdclk;
2439
2440         cdclk = vlv_calc_cdclk(dev_priv, min_cdclk);
2441
2442         cdclk_state->logical.cdclk = cdclk;
2443         cdclk_state->logical.voltage_level =
2444                 vlv_calc_voltage_level(dev_priv, cdclk);
2445
2446         if (!cdclk_state->active_pipes) {
2447                 cdclk = vlv_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2448
2449                 cdclk_state->actual.cdclk = cdclk;
2450                 cdclk_state->actual.voltage_level =
2451                         vlv_calc_voltage_level(dev_priv, cdclk);
2452         } else {
2453                 cdclk_state->actual = cdclk_state->logical;
2454         }
2455
2456         return 0;
2457 }
2458
2459 static int bdw_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2460 {
2461         int min_cdclk, cdclk;
2462
2463         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2464         if (min_cdclk < 0)
2465                 return min_cdclk;
2466
2467         /*
2468          * FIXME should also account for plane ratio
2469          * once 64bpp pixel formats are supported.
2470          */
2471         cdclk = bdw_calc_cdclk(min_cdclk);
2472
2473         cdclk_state->logical.cdclk = cdclk;
2474         cdclk_state->logical.voltage_level =
2475                 bdw_calc_voltage_level(cdclk);
2476
2477         if (!cdclk_state->active_pipes) {
2478                 cdclk = bdw_calc_cdclk(cdclk_state->force_min_cdclk);
2479
2480                 cdclk_state->actual.cdclk = cdclk;
2481                 cdclk_state->actual.voltage_level =
2482                         bdw_calc_voltage_level(cdclk);
2483         } else {
2484                 cdclk_state->actual = cdclk_state->logical;
2485         }
2486
2487         return 0;
2488 }
2489
2490 static int skl_dpll0_vco(struct intel_cdclk_state *cdclk_state)
2491 {
2492         struct intel_atomic_state *state = cdclk_state->base.state;
2493         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2494         struct intel_crtc *crtc;
2495         struct intel_crtc_state *crtc_state;
2496         int vco, i;
2497
2498         vco = cdclk_state->logical.vco;
2499         if (!vco)
2500                 vco = dev_priv->skl_preferred_vco_freq;
2501
2502         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2503                 if (!crtc_state->hw.enable)
2504                         continue;
2505
2506                 if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
2507                         continue;
2508
2509                 /*
2510                  * DPLL0 VCO may need to be adjusted to get the correct
2511                  * clock for eDP. This will affect cdclk as well.
2512                  */
2513                 switch (crtc_state->port_clock / 2) {
2514                 case 108000:
2515                 case 216000:
2516                         vco = 8640000;
2517                         break;
2518                 default:
2519                         vco = 8100000;
2520                         break;
2521                 }
2522         }
2523
2524         return vco;
2525 }
2526
2527 static int skl_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2528 {
2529         int min_cdclk, cdclk, vco;
2530
2531         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2532         if (min_cdclk < 0)
2533                 return min_cdclk;
2534
2535         vco = skl_dpll0_vco(cdclk_state);
2536
2537         /*
2538          * FIXME should also account for plane ratio
2539          * once 64bpp pixel formats are supported.
2540          */
2541         cdclk = skl_calc_cdclk(min_cdclk, vco);
2542
2543         cdclk_state->logical.vco = vco;
2544         cdclk_state->logical.cdclk = cdclk;
2545         cdclk_state->logical.voltage_level =
2546                 skl_calc_voltage_level(cdclk);
2547
2548         if (!cdclk_state->active_pipes) {
2549                 cdclk = skl_calc_cdclk(cdclk_state->force_min_cdclk, vco);
2550
2551                 cdclk_state->actual.vco = vco;
2552                 cdclk_state->actual.cdclk = cdclk;
2553                 cdclk_state->actual.voltage_level =
2554                         skl_calc_voltage_level(cdclk);
2555         } else {
2556                 cdclk_state->actual = cdclk_state->logical;
2557         }
2558
2559         return 0;
2560 }
2561
2562 static int bxt_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2563 {
2564         struct intel_atomic_state *state = cdclk_state->base.state;
2565         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2566         int min_cdclk, min_voltage_level, cdclk, vco;
2567
2568         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2569         if (min_cdclk < 0)
2570                 return min_cdclk;
2571
2572         min_voltage_level = bxt_compute_min_voltage_level(cdclk_state);
2573         if (min_voltage_level < 0)
2574                 return min_voltage_level;
2575
2576         cdclk = bxt_calc_cdclk(dev_priv, min_cdclk);
2577         vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2578
2579         cdclk_state->logical.vco = vco;
2580         cdclk_state->logical.cdclk = cdclk;
2581         cdclk_state->logical.voltage_level =
2582                 max_t(int, min_voltage_level,
2583                       intel_cdclk_calc_voltage_level(dev_priv, cdclk));
2584
2585         if (!cdclk_state->active_pipes) {
2586                 cdclk = bxt_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2587                 vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2588
2589                 cdclk_state->actual.vco = vco;
2590                 cdclk_state->actual.cdclk = cdclk;
2591                 cdclk_state->actual.voltage_level =
2592                         intel_cdclk_calc_voltage_level(dev_priv, cdclk);
2593         } else {
2594                 cdclk_state->actual = cdclk_state->logical;
2595         }
2596
2597         return 0;
2598 }
2599
2600 static int fixed_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2601 {
2602         int min_cdclk;
2603
2604         /*
2605          * We can't change the cdclk frequency, but we still want to
2606          * check that the required minimum frequency doesn't exceed
2607          * the actual cdclk frequency.
2608          */
2609         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2610         if (min_cdclk < 0)
2611                 return min_cdclk;
2612
2613         return 0;
2614 }
2615
2616 static struct intel_global_state *intel_cdclk_duplicate_state(struct intel_global_obj *obj)
2617 {
2618         struct intel_cdclk_state *cdclk_state;
2619
2620         cdclk_state = kmemdup(obj->state, sizeof(*cdclk_state), GFP_KERNEL);
2621         if (!cdclk_state)
2622                 return NULL;
2623
2624         cdclk_state->pipe = INVALID_PIPE;
2625
2626         return &cdclk_state->base;
2627 }
2628
2629 static void intel_cdclk_destroy_state(struct intel_global_obj *obj,
2630                                       struct intel_global_state *state)
2631 {
2632         kfree(state);
2633 }
2634
2635 static const struct intel_global_state_funcs intel_cdclk_funcs = {
2636         .atomic_duplicate_state = intel_cdclk_duplicate_state,
2637         .atomic_destroy_state = intel_cdclk_destroy_state,
2638 };
2639
2640 struct intel_cdclk_state *
2641 intel_atomic_get_cdclk_state(struct intel_atomic_state *state)
2642 {
2643         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2644         struct intel_global_state *cdclk_state;
2645
2646         cdclk_state = intel_atomic_get_global_obj_state(state, &dev_priv->cdclk.obj);
2647         if (IS_ERR(cdclk_state))
2648                 return ERR_CAST(cdclk_state);
2649
2650         return to_intel_cdclk_state(cdclk_state);
2651 }
2652
2653 int intel_cdclk_atomic_check(struct intel_atomic_state *state,
2654                              bool *need_cdclk_calc)
2655 {
2656         const struct intel_cdclk_state *old_cdclk_state;
2657         const struct intel_cdclk_state *new_cdclk_state;
2658         struct intel_plane_state *plane_state;
2659         struct intel_plane *plane;
2660         int ret;
2661         int i;
2662
2663         /*
2664          * active_planes bitmask has been updated, and potentially affected
2665          * planes are part of the state. We can now compute the minimum cdclk
2666          * for each plane.
2667          */
2668         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2669                 ret = intel_plane_calc_min_cdclk(state, plane, need_cdclk_calc);
2670                 if (ret)
2671                         return ret;
2672         }
2673
2674         ret = intel_bw_calc_min_cdclk(state, need_cdclk_calc);
2675         if (ret)
2676                 return ret;
2677
2678         old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
2679         new_cdclk_state = intel_atomic_get_new_cdclk_state(state);
2680
2681         if (new_cdclk_state &&
2682             old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk)
2683                 *need_cdclk_calc = true;
2684
2685         return 0;
2686 }
2687
2688 int intel_cdclk_init(struct drm_i915_private *dev_priv)
2689 {
2690         struct intel_cdclk_state *cdclk_state;
2691
2692         cdclk_state = kzalloc(sizeof(*cdclk_state), GFP_KERNEL);
2693         if (!cdclk_state)
2694                 return -ENOMEM;
2695
2696         intel_atomic_global_obj_init(dev_priv, &dev_priv->cdclk.obj,
2697                                      &cdclk_state->base, &intel_cdclk_funcs);
2698
2699         return 0;
2700 }
2701
2702 int intel_modeset_calc_cdclk(struct intel_atomic_state *state)
2703 {
2704         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2705         const struct intel_cdclk_state *old_cdclk_state;
2706         struct intel_cdclk_state *new_cdclk_state;
2707         enum pipe pipe = INVALID_PIPE;
2708         int ret;
2709
2710         new_cdclk_state = intel_atomic_get_cdclk_state(state);
2711         if (IS_ERR(new_cdclk_state))
2712                 return PTR_ERR(new_cdclk_state);
2713
2714         old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
2715
2716         new_cdclk_state->active_pipes =
2717                 intel_calc_active_pipes(state, old_cdclk_state->active_pipes);
2718
2719         ret = intel_cdclk_modeset_calc_cdclk(dev_priv, new_cdclk_state);
2720         if (ret)
2721                 return ret;
2722
2723         if (intel_cdclk_changed(&old_cdclk_state->actual,
2724                                 &new_cdclk_state->actual)) {
2725                 /*
2726                  * Also serialize commits across all crtcs
2727                  * if the actual hw needs to be poked.
2728                  */
2729                 ret = intel_atomic_serialize_global_state(&new_cdclk_state->base);
2730                 if (ret)
2731                         return ret;
2732         } else if (old_cdclk_state->active_pipes != new_cdclk_state->active_pipes ||
2733                    old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk ||
2734                    intel_cdclk_changed(&old_cdclk_state->logical,
2735                                        &new_cdclk_state->logical)) {
2736                 ret = intel_atomic_lock_global_state(&new_cdclk_state->base);
2737                 if (ret)
2738                         return ret;
2739         } else {
2740                 return 0;
2741         }
2742
2743         if (is_power_of_2(new_cdclk_state->active_pipes) &&
2744             intel_cdclk_can_cd2x_update(dev_priv,
2745                                         &old_cdclk_state->actual,
2746                                         &new_cdclk_state->actual)) {
2747                 struct intel_crtc *crtc;
2748                 struct intel_crtc_state *crtc_state;
2749
2750                 pipe = ilog2(new_cdclk_state->active_pipes);
2751                 crtc = intel_crtc_for_pipe(dev_priv, pipe);
2752
2753                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
2754                 if (IS_ERR(crtc_state))
2755                         return PTR_ERR(crtc_state);
2756
2757                 if (drm_atomic_crtc_needs_modeset(&crtc_state->uapi))
2758                         pipe = INVALID_PIPE;
2759         }
2760
2761         if (intel_cdclk_can_squash(dev_priv,
2762                                    &old_cdclk_state->actual,
2763                                    &new_cdclk_state->actual)) {
2764                 drm_dbg_kms(&dev_priv->drm,
2765                             "Can change cdclk via squasher\n");
2766         } else if (intel_cdclk_can_crawl(dev_priv,
2767                                          &old_cdclk_state->actual,
2768                                          &new_cdclk_state->actual)) {
2769                 drm_dbg_kms(&dev_priv->drm,
2770                             "Can change cdclk via crawl\n");
2771         } else if (pipe != INVALID_PIPE) {
2772                 new_cdclk_state->pipe = pipe;
2773
2774                 drm_dbg_kms(&dev_priv->drm,
2775                             "Can change cdclk cd2x divider with pipe %c active\n",
2776                             pipe_name(pipe));
2777         } else if (intel_cdclk_needs_modeset(&old_cdclk_state->actual,
2778                                              &new_cdclk_state->actual)) {
2779                 /* All pipes must be switched off while we change the cdclk. */
2780                 ret = intel_modeset_all_pipes(state);
2781                 if (ret)
2782                         return ret;
2783
2784                 drm_dbg_kms(&dev_priv->drm,
2785                             "Modeset required for cdclk change\n");
2786         }
2787
2788         drm_dbg_kms(&dev_priv->drm,
2789                     "New cdclk calculated to be logical %u kHz, actual %u kHz\n",
2790                     new_cdclk_state->logical.cdclk,
2791                     new_cdclk_state->actual.cdclk);
2792         drm_dbg_kms(&dev_priv->drm,
2793                     "New voltage level calculated to be logical %u, actual %u\n",
2794                     new_cdclk_state->logical.voltage_level,
2795                     new_cdclk_state->actual.voltage_level);
2796
2797         return 0;
2798 }
2799
2800 static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
2801 {
2802         int max_cdclk_freq = dev_priv->max_cdclk_freq;
2803
2804         if (DISPLAY_VER(dev_priv) >= 10)
2805                 return 2 * max_cdclk_freq;
2806         else if (DISPLAY_VER(dev_priv) == 9 ||
2807                  IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2808                 return max_cdclk_freq;
2809         else if (IS_CHERRYVIEW(dev_priv))
2810                 return max_cdclk_freq*95/100;
2811         else if (DISPLAY_VER(dev_priv) < 4)
2812                 return 2*max_cdclk_freq*90/100;
2813         else
2814                 return max_cdclk_freq*90/100;
2815 }
2816
2817 /**
2818  * intel_update_max_cdclk - Determine the maximum support CDCLK frequency
2819  * @dev_priv: i915 device
2820  *
2821  * Determine the maximum CDCLK frequency the platform supports, and also
2822  * derive the maximum dot clock frequency the maximum CDCLK frequency
2823  * allows.
2824  */
2825 void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
2826 {
2827         if (IS_JSL_EHL(dev_priv)) {
2828                 if (dev_priv->cdclk.hw.ref == 24000)
2829                         dev_priv->max_cdclk_freq = 552000;
2830                 else
2831                         dev_priv->max_cdclk_freq = 556800;
2832         } else if (DISPLAY_VER(dev_priv) >= 11) {
2833                 if (dev_priv->cdclk.hw.ref == 24000)
2834                         dev_priv->max_cdclk_freq = 648000;
2835                 else
2836                         dev_priv->max_cdclk_freq = 652800;
2837         } else if (IS_GEMINILAKE(dev_priv)) {
2838                 dev_priv->max_cdclk_freq = 316800;
2839         } else if (IS_BROXTON(dev_priv)) {
2840                 dev_priv->max_cdclk_freq = 624000;
2841         } else if (DISPLAY_VER(dev_priv) == 9) {
2842                 u32 limit = intel_de_read(dev_priv, SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
2843                 int max_cdclk, vco;
2844
2845                 vco = dev_priv->skl_preferred_vco_freq;
2846                 drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
2847
2848                 /*
2849                  * Use the lower (vco 8640) cdclk values as a
2850                  * first guess. skl_calc_cdclk() will correct it
2851                  * if the preferred vco is 8100 instead.
2852                  */
2853                 if (limit == SKL_DFSM_CDCLK_LIMIT_675)
2854                         max_cdclk = 617143;
2855                 else if (limit == SKL_DFSM_CDCLK_LIMIT_540)
2856                         max_cdclk = 540000;
2857                 else if (limit == SKL_DFSM_CDCLK_LIMIT_450)
2858                         max_cdclk = 432000;
2859                 else
2860                         max_cdclk = 308571;
2861
2862                 dev_priv->max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
2863         } else if (IS_BROADWELL(dev_priv))  {
2864                 /*
2865                  * FIXME with extra cooling we can allow
2866                  * 540 MHz for ULX and 675 Mhz for ULT.
2867                  * How can we know if extra cooling is
2868                  * available? PCI ID, VTB, something else?
2869                  */
2870                 if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
2871                         dev_priv->max_cdclk_freq = 450000;
2872                 else if (IS_BDW_ULX(dev_priv))
2873                         dev_priv->max_cdclk_freq = 450000;
2874                 else if (IS_BDW_ULT(dev_priv))
2875                         dev_priv->max_cdclk_freq = 540000;
2876                 else
2877                         dev_priv->max_cdclk_freq = 675000;
2878         } else if (IS_CHERRYVIEW(dev_priv)) {
2879                 dev_priv->max_cdclk_freq = 320000;
2880         } else if (IS_VALLEYVIEW(dev_priv)) {
2881                 dev_priv->max_cdclk_freq = 400000;
2882         } else {
2883                 /* otherwise assume cdclk is fixed */
2884                 dev_priv->max_cdclk_freq = dev_priv->cdclk.hw.cdclk;
2885         }
2886
2887         dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
2888
2889         drm_dbg(&dev_priv->drm, "Max CD clock rate: %d kHz\n",
2890                 dev_priv->max_cdclk_freq);
2891
2892         drm_dbg(&dev_priv->drm, "Max dotclock rate: %d kHz\n",
2893                 dev_priv->max_dotclk_freq);
2894 }
2895
2896 /**
2897  * intel_update_cdclk - Determine the current CDCLK frequency
2898  * @dev_priv: i915 device
2899  *
2900  * Determine the current CDCLK frequency.
2901  */
2902 void intel_update_cdclk(struct drm_i915_private *dev_priv)
2903 {
2904         intel_cdclk_get_cdclk(dev_priv, &dev_priv->cdclk.hw);
2905
2906         /*
2907          * 9:0 CMBUS [sic] CDCLK frequency (cdfreq):
2908          * Programmng [sic] note: bit[9:2] should be programmed to the number
2909          * of cdclk that generates 4MHz reference clock freq which is used to
2910          * generate GMBus clock. This will vary with the cdclk freq.
2911          */
2912         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2913                 intel_de_write(dev_priv, GMBUSFREQ_VLV,
2914                                DIV_ROUND_UP(dev_priv->cdclk.hw.cdclk, 1000));
2915 }
2916
2917 static int dg1_rawclk(struct drm_i915_private *dev_priv)
2918 {
2919         /*
2920          * DG1 always uses a 38.4 MHz rawclk.  The bspec tells us
2921          * "Program Numerator=2, Denominator=4, Divider=37 decimal."
2922          */
2923         intel_de_write(dev_priv, PCH_RAWCLK_FREQ,
2924                        CNP_RAWCLK_DEN(4) | CNP_RAWCLK_DIV(37) | ICP_RAWCLK_NUM(2));
2925
2926         return 38400;
2927 }
2928
2929 static int cnp_rawclk(struct drm_i915_private *dev_priv)
2930 {
2931         u32 rawclk;
2932         int divider, fraction;
2933
2934         if (intel_de_read(dev_priv, SFUSE_STRAP) & SFUSE_STRAP_RAW_FREQUENCY) {
2935                 /* 24 MHz */
2936                 divider = 24000;
2937                 fraction = 0;
2938         } else {
2939                 /* 19.2 MHz */
2940                 divider = 19000;
2941                 fraction = 200;
2942         }
2943
2944         rawclk = CNP_RAWCLK_DIV(divider / 1000);
2945         if (fraction) {
2946                 int numerator = 1;
2947
2948                 rawclk |= CNP_RAWCLK_DEN(DIV_ROUND_CLOSEST(numerator * 1000,
2949                                                            fraction) - 1);
2950                 if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
2951                         rawclk |= ICP_RAWCLK_NUM(numerator);
2952         }
2953
2954         intel_de_write(dev_priv, PCH_RAWCLK_FREQ, rawclk);
2955         return divider + fraction;
2956 }
2957
2958 static int pch_rawclk(struct drm_i915_private *dev_priv)
2959 {
2960         return (intel_de_read(dev_priv, PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK) * 1000;
2961 }
2962
2963 static int vlv_hrawclk(struct drm_i915_private *dev_priv)
2964 {
2965         /* RAWCLK_FREQ_VLV register updated from power well code */
2966         return vlv_get_cck_clock_hpll(dev_priv, "hrawclk",
2967                                       CCK_DISPLAY_REF_CLOCK_CONTROL);
2968 }
2969
2970 static int i9xx_hrawclk(struct drm_i915_private *dev_priv)
2971 {
2972         u32 clkcfg;
2973
2974         /*
2975          * hrawclock is 1/4 the FSB frequency
2976          *
2977          * Note that this only reads the state of the FSB
2978          * straps, not the actual FSB frequency. Some BIOSen
2979          * let you configure each independently. Ideally we'd
2980          * read out the actual FSB frequency but sadly we
2981          * don't know which registers have that information,
2982          * and all the relevant docs have gone to bit heaven :(
2983          */
2984         clkcfg = intel_de_read(dev_priv, CLKCFG) & CLKCFG_FSB_MASK;
2985
2986         if (IS_MOBILE(dev_priv)) {
2987                 switch (clkcfg) {
2988                 case CLKCFG_FSB_400:
2989                         return 100000;
2990                 case CLKCFG_FSB_533:
2991                         return 133333;
2992                 case CLKCFG_FSB_667:
2993                         return 166667;
2994                 case CLKCFG_FSB_800:
2995                         return 200000;
2996                 case CLKCFG_FSB_1067:
2997                         return 266667;
2998                 case CLKCFG_FSB_1333:
2999                         return 333333;
3000                 default:
3001                         MISSING_CASE(clkcfg);
3002                         return 133333;
3003                 }
3004         } else {
3005                 switch (clkcfg) {
3006                 case CLKCFG_FSB_400_ALT:
3007                         return 100000;
3008                 case CLKCFG_FSB_533:
3009                         return 133333;
3010                 case CLKCFG_FSB_667:
3011                         return 166667;
3012                 case CLKCFG_FSB_800:
3013                         return 200000;
3014                 case CLKCFG_FSB_1067_ALT:
3015                         return 266667;
3016                 case CLKCFG_FSB_1333_ALT:
3017                         return 333333;
3018                 case CLKCFG_FSB_1600_ALT:
3019                         return 400000;
3020                 default:
3021                         return 133333;
3022                 }
3023         }
3024 }
3025
3026 /**
3027  * intel_read_rawclk - Determine the current RAWCLK frequency
3028  * @dev_priv: i915 device
3029  *
3030  * Determine the current RAWCLK frequency. RAWCLK is a fixed
3031  * frequency clock so this needs to done only once.
3032  */
3033 u32 intel_read_rawclk(struct drm_i915_private *dev_priv)
3034 {
3035         u32 freq;
3036
3037         if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1)
3038                 freq = dg1_rawclk(dev_priv);
3039         else if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
3040                 freq = cnp_rawclk(dev_priv);
3041         else if (HAS_PCH_SPLIT(dev_priv))
3042                 freq = pch_rawclk(dev_priv);
3043         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
3044                 freq = vlv_hrawclk(dev_priv);
3045         else if (DISPLAY_VER(dev_priv) >= 3)
3046                 freq = i9xx_hrawclk(dev_priv);
3047         else
3048                 /* no rawclk on other platforms, or no need to know it */
3049                 return 0;
3050
3051         return freq;
3052 }
3053
3054 static const struct intel_cdclk_funcs tgl_cdclk_funcs = {
3055         .get_cdclk = bxt_get_cdclk,
3056         .set_cdclk = bxt_set_cdclk,
3057         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3058         .calc_voltage_level = tgl_calc_voltage_level,
3059 };
3060
3061 static const struct intel_cdclk_funcs ehl_cdclk_funcs = {
3062         .get_cdclk = bxt_get_cdclk,
3063         .set_cdclk = bxt_set_cdclk,
3064         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3065         .calc_voltage_level = ehl_calc_voltage_level,
3066 };
3067
3068 static const struct intel_cdclk_funcs icl_cdclk_funcs = {
3069         .get_cdclk = bxt_get_cdclk,
3070         .set_cdclk = bxt_set_cdclk,
3071         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3072         .calc_voltage_level = icl_calc_voltage_level,
3073 };
3074
3075 static const struct intel_cdclk_funcs bxt_cdclk_funcs = {
3076         .get_cdclk = bxt_get_cdclk,
3077         .set_cdclk = bxt_set_cdclk,
3078         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3079         .calc_voltage_level = bxt_calc_voltage_level,
3080 };
3081
3082 static const struct intel_cdclk_funcs skl_cdclk_funcs = {
3083         .get_cdclk = skl_get_cdclk,
3084         .set_cdclk = skl_set_cdclk,
3085         .modeset_calc_cdclk = skl_modeset_calc_cdclk,
3086 };
3087
3088 static const struct intel_cdclk_funcs bdw_cdclk_funcs = {
3089         .get_cdclk = bdw_get_cdclk,
3090         .set_cdclk = bdw_set_cdclk,
3091         .modeset_calc_cdclk = bdw_modeset_calc_cdclk,
3092 };
3093
3094 static const struct intel_cdclk_funcs chv_cdclk_funcs = {
3095         .get_cdclk = vlv_get_cdclk,
3096         .set_cdclk = chv_set_cdclk,
3097         .modeset_calc_cdclk = vlv_modeset_calc_cdclk,
3098 };
3099
3100 static const struct intel_cdclk_funcs vlv_cdclk_funcs = {
3101         .get_cdclk = vlv_get_cdclk,
3102         .set_cdclk = vlv_set_cdclk,
3103         .modeset_calc_cdclk = vlv_modeset_calc_cdclk,
3104 };
3105
3106 static const struct intel_cdclk_funcs hsw_cdclk_funcs = {
3107         .get_cdclk = hsw_get_cdclk,
3108         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3109 };
3110
3111 /* SNB, IVB, 965G, 945G */
3112 static const struct intel_cdclk_funcs fixed_400mhz_cdclk_funcs = {
3113         .get_cdclk = fixed_400mhz_get_cdclk,
3114         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3115 };
3116
3117 static const struct intel_cdclk_funcs ilk_cdclk_funcs = {
3118         .get_cdclk = fixed_450mhz_get_cdclk,
3119         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3120 };
3121
3122 static const struct intel_cdclk_funcs gm45_cdclk_funcs = {
3123         .get_cdclk = gm45_get_cdclk,
3124         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3125 };
3126
3127 /* G45 uses G33 */
3128
3129 static const struct intel_cdclk_funcs i965gm_cdclk_funcs = {
3130         .get_cdclk = i965gm_get_cdclk,
3131         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3132 };
3133
3134 /* i965G uses fixed 400 */
3135
3136 static const struct intel_cdclk_funcs pnv_cdclk_funcs = {
3137         .get_cdclk = pnv_get_cdclk,
3138         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3139 };
3140
3141 static const struct intel_cdclk_funcs g33_cdclk_funcs = {
3142         .get_cdclk = g33_get_cdclk,
3143         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3144 };
3145
3146 static const struct intel_cdclk_funcs i945gm_cdclk_funcs = {
3147         .get_cdclk = i945gm_get_cdclk,
3148         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3149 };
3150
3151 /* i945G uses fixed 400 */
3152
3153 static const struct intel_cdclk_funcs i915gm_cdclk_funcs = {
3154         .get_cdclk = i915gm_get_cdclk,
3155         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3156 };
3157
3158 static const struct intel_cdclk_funcs i915g_cdclk_funcs = {
3159         .get_cdclk = fixed_333mhz_get_cdclk,
3160         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3161 };
3162
3163 static const struct intel_cdclk_funcs i865g_cdclk_funcs = {
3164         .get_cdclk = fixed_266mhz_get_cdclk,
3165         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3166 };
3167
3168 static const struct intel_cdclk_funcs i85x_cdclk_funcs = {
3169         .get_cdclk = i85x_get_cdclk,
3170         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3171 };
3172
3173 static const struct intel_cdclk_funcs i845g_cdclk_funcs = {
3174         .get_cdclk = fixed_200mhz_get_cdclk,
3175         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3176 };
3177
3178 static const struct intel_cdclk_funcs i830_cdclk_funcs = {
3179         .get_cdclk = fixed_133mhz_get_cdclk,
3180         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3181 };
3182
3183 /**
3184  * intel_init_cdclk_hooks - Initialize CDCLK related modesetting hooks
3185  * @dev_priv: i915 device
3186  */
3187 void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
3188 {
3189         if (IS_DG2(dev_priv)) {
3190                 dev_priv->cdclk_funcs = &tgl_cdclk_funcs;
3191                 dev_priv->cdclk.table = dg2_cdclk_table;
3192         } else if (IS_ALDERLAKE_P(dev_priv)) {
3193                 dev_priv->cdclk_funcs = &tgl_cdclk_funcs;
3194                 /* Wa_22011320316:adl-p[a0] */
3195                 if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
3196                         dev_priv->cdclk.table = adlp_a_step_cdclk_table;
3197                 else
3198                         dev_priv->cdclk.table = adlp_cdclk_table;
3199         } else if (IS_ROCKETLAKE(dev_priv)) {
3200                 dev_priv->cdclk_funcs = &tgl_cdclk_funcs;
3201                 dev_priv->cdclk.table = rkl_cdclk_table;
3202         } else if (DISPLAY_VER(dev_priv) >= 12) {
3203                 dev_priv->cdclk_funcs = &tgl_cdclk_funcs;
3204                 dev_priv->cdclk.table = icl_cdclk_table;
3205         } else if (IS_JSL_EHL(dev_priv)) {
3206                 dev_priv->cdclk_funcs = &ehl_cdclk_funcs;
3207                 dev_priv->cdclk.table = icl_cdclk_table;
3208         } else if (DISPLAY_VER(dev_priv) >= 11) {
3209                 dev_priv->cdclk_funcs = &icl_cdclk_funcs;
3210                 dev_priv->cdclk.table = icl_cdclk_table;
3211         } else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
3212                 dev_priv->cdclk_funcs = &bxt_cdclk_funcs;
3213                 if (IS_GEMINILAKE(dev_priv))
3214                         dev_priv->cdclk.table = glk_cdclk_table;
3215                 else
3216                         dev_priv->cdclk.table = bxt_cdclk_table;
3217         } else if (DISPLAY_VER(dev_priv) == 9) {
3218                 dev_priv->cdclk_funcs = &skl_cdclk_funcs;
3219         } else if (IS_BROADWELL(dev_priv)) {
3220                 dev_priv->cdclk_funcs = &bdw_cdclk_funcs;
3221         } else if (IS_HASWELL(dev_priv)) {
3222                 dev_priv->cdclk_funcs = &hsw_cdclk_funcs;
3223         } else if (IS_CHERRYVIEW(dev_priv)) {
3224                 dev_priv->cdclk_funcs = &chv_cdclk_funcs;
3225         } else if (IS_VALLEYVIEW(dev_priv)) {
3226                 dev_priv->cdclk_funcs = &vlv_cdclk_funcs;
3227         } else if (IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
3228                 dev_priv->cdclk_funcs = &fixed_400mhz_cdclk_funcs;
3229         } else if (IS_IRONLAKE(dev_priv)) {
3230                 dev_priv->cdclk_funcs = &ilk_cdclk_funcs;
3231         } else if (IS_GM45(dev_priv)) {
3232                 dev_priv->cdclk_funcs = &gm45_cdclk_funcs;
3233         } else if (IS_G45(dev_priv)) {
3234                 dev_priv->cdclk_funcs = &g33_cdclk_funcs;
3235         } else if (IS_I965GM(dev_priv)) {
3236                 dev_priv->cdclk_funcs = &i965gm_cdclk_funcs;
3237         } else if (IS_I965G(dev_priv)) {
3238                 dev_priv->cdclk_funcs = &fixed_400mhz_cdclk_funcs;
3239         } else if (IS_PINEVIEW(dev_priv)) {
3240                 dev_priv->cdclk_funcs = &pnv_cdclk_funcs;
3241         } else if (IS_G33(dev_priv)) {
3242                 dev_priv->cdclk_funcs = &g33_cdclk_funcs;
3243         } else if (IS_I945GM(dev_priv)) {
3244                 dev_priv->cdclk_funcs = &i945gm_cdclk_funcs;
3245         } else if (IS_I945G(dev_priv)) {
3246                 dev_priv->cdclk_funcs = &fixed_400mhz_cdclk_funcs;
3247         } else if (IS_I915GM(dev_priv)) {
3248                 dev_priv->cdclk_funcs = &i915gm_cdclk_funcs;
3249         } else if (IS_I915G(dev_priv)) {
3250                 dev_priv->cdclk_funcs = &i915g_cdclk_funcs;
3251         } else if (IS_I865G(dev_priv)) {
3252                 dev_priv->cdclk_funcs = &i865g_cdclk_funcs;
3253         } else if (IS_I85X(dev_priv)) {
3254                 dev_priv->cdclk_funcs = &i85x_cdclk_funcs;
3255         } else if (IS_I845G(dev_priv)) {
3256                 dev_priv->cdclk_funcs = &i845g_cdclk_funcs;
3257         } else if (IS_I830(dev_priv)) {
3258                 dev_priv->cdclk_funcs = &i830_cdclk_funcs;
3259         }
3260
3261         if (drm_WARN(&dev_priv->drm, !dev_priv->cdclk_funcs,
3262                      "Unknown platform. Assuming i830\n"))
3263                 dev_priv->cdclk_funcs = &i830_cdclk_funcs;
3264 }