drivers/gpu/drm/i915/display/intel_cdclk.c

   1 /*
   2  * Copyright © 2006-2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include <linux/time.h>
  25
  26 #include "hsw_ips.h"
  27 #include "i915_reg.h"
  28 #include "intel_atomic.h"
  29 #include "intel_atomic_plane.h"
  30 #include "intel_audio.h"
  31 #include "intel_bw.h"
  32 #include "intel_cdclk.h"
  33 #include "intel_crtc.h"
  34 #include "intel_de.h"
  35 #include "intel_dp.h"
  36 #include "intel_display_types.h"
  37 #include "intel_mchbar_regs.h"
  38 #include "intel_pci_config.h"
  39 #include "intel_pcode.h"
  40 #include "intel_psr.h"
  41 #include "intel_vdsc.h"
  42 #include "vlv_sideband.h"
  43
  44 /**
  45  * DOC: CDCLK / RAWCLK
  46  *
  47  * The display engine uses several different clocks to do its work. There
  48  * are two main clocks involved that aren't directly related to the actual
  49  * pixel clock or any symbol/bit clock of the actual output port. These
  50  * are the core display clock (CDCLK) and RAWCLK.
  51  *
  52  * CDCLK clocks most of the display pipe logic, and thus its frequency
  53  * must be high enough to support the rate at which pixels are flowing
  54  * through the pipes. Downscaling must also be accounted as that increases
  55  * the effective pixel rate.
  56  *
  57  * On several platforms the CDCLK frequency can be changed dynamically
  58  * to minimize power consumption for a given display configuration.
  59  * Typically changes to the CDCLK frequency require all the display pipes
  60  * to be shut down while the frequency is being changed.
  61  *
  62  * On SKL+ the DMC will toggle the CDCLK off/on during DC5/6 entry/exit.
  63  * DMC will not change the active CDCLK frequency however, so that part
  64  * will still be performed by the driver directly.
  65  *
  66  * RAWCLK is a fixed frequency clock, often used by various auxiliary
  67  * blocks such as AUX CH or backlight PWM. Hence the only thing we
  68  * really need to know about RAWCLK is its frequency so that various
  69  * dividers can be programmed correctly.
  70  */
  71
  72 struct intel_cdclk_funcs {
  73         void (*get_cdclk)(struct drm_i915_private *i915,
  74                           struct intel_cdclk_config *cdclk_config);
  75         void (*set_cdclk)(struct drm_i915_private *i915,
  76                           const struct intel_cdclk_config *cdclk_config,
  77                           enum pipe pipe);
  78         int (*modeset_calc_cdclk)(struct intel_cdclk_state *state);
  79         u8 (*calc_voltage_level)(int cdclk);
  80 };
  81
  82 void intel_cdclk_get_cdclk(struct drm_i915_private *dev_priv,
  83                            struct intel_cdclk_config *cdclk_config)
  84 {
  85         dev_priv->display.funcs.cdclk->get_cdclk(dev_priv, cdclk_config);
  86 }
  87
  88 static void intel_cdclk_set_cdclk(struct drm_i915_private *dev_priv,
  89                                   const struct intel_cdclk_config *cdclk_config,
  90                                   enum pipe pipe)
  91 {
  92         dev_priv->display.funcs.cdclk->set_cdclk(dev_priv, cdclk_config, pipe);
  93 }
  94
  95 static int intel_cdclk_modeset_calc_cdclk(struct drm_i915_private *dev_priv,
  96                                           struct intel_cdclk_state *cdclk_config)
  97 {
  98         return dev_priv->display.funcs.cdclk->modeset_calc_cdclk(cdclk_config);
  99 }
 100
 101 static u8 intel_cdclk_calc_voltage_level(struct drm_i915_private *dev_priv,
 102                                          int cdclk)
 103 {
 104         return dev_priv->display.funcs.cdclk->calc_voltage_level(cdclk);
 105 }
 106
 107 static void fixed_133mhz_get_cdclk(struct drm_i915_private *dev_priv,
 108                                    struct intel_cdclk_config *cdclk_config)
 109 {
 110         cdclk_config->cdclk = 133333;
 111 }
 112
 113 static void fixed_200mhz_get_cdclk(struct drm_i915_private *dev_priv,
 114                                    struct intel_cdclk_config *cdclk_config)
 115 {
 116         cdclk_config->cdclk = 200000;
 117 }
 118
 119 static void fixed_266mhz_get_cdclk(struct drm_i915_private *dev_priv,
 120                                    struct intel_cdclk_config *cdclk_config)
 121 {
 122         cdclk_config->cdclk = 266667;
 123 }
 124
 125 static void fixed_333mhz_get_cdclk(struct drm_i915_private *dev_priv,
 126                                    struct intel_cdclk_config *cdclk_config)
 127 {
 128         cdclk_config->cdclk = 333333;
 129 }
 130
 131 static void fixed_400mhz_get_cdclk(struct drm_i915_private *dev_priv,
 132                                    struct intel_cdclk_config *cdclk_config)
 133 {
 134         cdclk_config->cdclk = 400000;
 135 }
 136
 137 static void fixed_450mhz_get_cdclk(struct drm_i915_private *dev_priv,
 138                                    struct intel_cdclk_config *cdclk_config)
 139 {
 140         cdclk_config->cdclk = 450000;
 141 }
 142
 143 static void i85x_get_cdclk(struct drm_i915_private *dev_priv,
 144                            struct intel_cdclk_config *cdclk_config)
 145 {
 146         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 147         u16 hpllcc = 0;
 148
 149         /*
 150          * 852GM/852GMV only supports 133 MHz and the HPLLCC
 151          * encoding is different :(
 152          * FIXME is this the right way to detect 852GM/852GMV?
 153          */
 154         if (pdev->revision == 0x1) {
 155                 cdclk_config->cdclk = 133333;
 156                 return;
 157         }
 158
 159         pci_bus_read_config_word(pdev->bus,
 160                                  PCI_DEVFN(0, 3), HPLLCC, &hpllcc);
 161
 162         /* Assume that the hardware is in the high speed state.  This
 163          * should be the default.
 164          */
 165         switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
 166         case GC_CLOCK_133_200:
 167         case GC_CLOCK_133_200_2:
 168         case GC_CLOCK_100_200:
 169                 cdclk_config->cdclk = 200000;
 170                 break;
 171         case GC_CLOCK_166_250:
 172                 cdclk_config->cdclk = 250000;
 173                 break;
 174         case GC_CLOCK_100_133:
 175                 cdclk_config->cdclk = 133333;
 176                 break;
 177         case GC_CLOCK_133_266:
 178         case GC_CLOCK_133_266_2:
 179         case GC_CLOCK_166_266:
 180                 cdclk_config->cdclk = 266667;
 181                 break;
 182         }
 183 }
 184
 185 static void i915gm_get_cdclk(struct drm_i915_private *dev_priv,
 186                              struct intel_cdclk_config *cdclk_config)
 187 {
 188         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 189         u16 gcfgc = 0;
 190
 191         pci_read_config_word(pdev, GCFGC, &gcfgc);
 192
 193         if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 194                 cdclk_config->cdclk = 133333;
 195                 return;
 196         }
 197
 198         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 199         case GC_DISPLAY_CLOCK_333_320_MHZ:
 200                 cdclk_config->cdclk = 333333;
 201                 break;
 202         default:
 203         case GC_DISPLAY_CLOCK_190_200_MHZ:
 204                 cdclk_config->cdclk = 190000;
 205                 break;
 206         }
 207 }
 208
 209 static void i945gm_get_cdclk(struct drm_i915_private *dev_priv,
 210                              struct intel_cdclk_config *cdclk_config)
 211 {
 212         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 213         u16 gcfgc = 0;
 214
 215         pci_read_config_word(pdev, GCFGC, &gcfgc);
 216
 217         if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 218                 cdclk_config->cdclk = 133333;
 219                 return;
 220         }
 221
 222         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 223         case GC_DISPLAY_CLOCK_333_320_MHZ:
 224                 cdclk_config->cdclk = 320000;
 225                 break;
 226         default:
 227         case GC_DISPLAY_CLOCK_190_200_MHZ:
 228                 cdclk_config->cdclk = 200000;
 229                 break;
 230         }
 231 }
 232
 233 static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv)
 234 {
 235         static const unsigned int blb_vco[8] = {
 236                 [0] = 3200000,
 237                 [1] = 4000000,
 238                 [2] = 5333333,
 239                 [3] = 4800000,
 240                 [4] = 6400000,
 241         };
 242         static const unsigned int pnv_vco[8] = {
 243                 [0] = 3200000,
 244                 [1] = 4000000,
 245                 [2] = 5333333,
 246                 [3] = 4800000,
 247                 [4] = 2666667,
 248         };
 249         static const unsigned int cl_vco[8] = {
 250                 [0] = 3200000,
 251                 [1] = 4000000,
 252                 [2] = 5333333,
 253                 [3] = 6400000,
 254                 [4] = 3333333,
 255                 [5] = 3566667,
 256                 [6] = 4266667,
 257         };
 258         static const unsigned int elk_vco[8] = {
 259                 [0] = 3200000,
 260                 [1] = 4000000,
 261                 [2] = 5333333,
 262                 [3] = 4800000,
 263         };
 264         static const unsigned int ctg_vco[8] = {
 265                 [0] = 3200000,
 266                 [1] = 4000000,
 267                 [2] = 5333333,
 268                 [3] = 6400000,
 269                 [4] = 2666667,
 270                 [5] = 4266667,
 271         };
 272         const unsigned int *vco_table;
 273         unsigned int vco;
 274         u8 tmp = 0;
 275
 276         /* FIXME other chipsets? */
 277         if (IS_GM45(dev_priv))
 278                 vco_table = ctg_vco;
 279         else if (IS_G45(dev_priv))
 280                 vco_table = elk_vco;
 281         else if (IS_I965GM(dev_priv))
 282                 vco_table = cl_vco;
 283         else if (IS_PINEVIEW(dev_priv))
 284                 vco_table = pnv_vco;
 285         else if (IS_G33(dev_priv))
 286                 vco_table = blb_vco;
 287         else
 288                 return 0;
 289
 290         tmp = intel_de_read(dev_priv,
 291                             IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv) ? HPLLVCO_MOBILE : HPLLVCO);
 292
 293         vco = vco_table[tmp & 0x7];
 294         if (vco == 0)
 295                 drm_err(&dev_priv->drm, "Bad HPLL VCO (HPLLVCO=0x%02x)\n",
 296                         tmp);
 297         else
 298                 drm_dbg_kms(&dev_priv->drm, "HPLL VCO %u kHz\n", vco);
 299
 300         return vco;
 301 }
 302
 303 static void g33_get_cdclk(struct drm_i915_private *dev_priv,
 304                           struct intel_cdclk_config *cdclk_config)
 305 {
 306         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 307         static const u8 div_3200[] = { 12, 10,  8,  7, 5, 16 };
 308         static const u8 div_4000[] = { 14, 12, 10,  8, 6, 20 };
 309         static const u8 div_4800[] = { 20, 14, 12, 10, 8, 24 };
 310         static const u8 div_5333[] = { 20, 16, 12, 12, 8, 28 };
 311         const u8 *div_table;
 312         unsigned int cdclk_sel;
 313         u16 tmp = 0;
 314
 315         cdclk_config->vco = intel_hpll_vco(dev_priv);
 316
 317         pci_read_config_word(pdev, GCFGC, &tmp);
 318
 319         cdclk_sel = (tmp >> 4) & 0x7;
 320
 321         if (cdclk_sel >= ARRAY_SIZE(div_3200))
 322                 goto fail;
 323
 324         switch (cdclk_config->vco) {
 325         case 3200000:
 326                 div_table = div_3200;
 327                 break;
 328         case 4000000:
 329                 div_table = div_4000;
 330                 break;
 331         case 4800000:
 332                 div_table = div_4800;
 333                 break;
 334         case 5333333:
 335                 div_table = div_5333;
 336                 break;
 337         default:
 338                 goto fail;
 339         }
 340
 341         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 342                                                 div_table[cdclk_sel]);
 343         return;
 344
 345 fail:
 346         drm_err(&dev_priv->drm,
 347                 "Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n",
 348                 cdclk_config->vco, tmp);
 349         cdclk_config->cdclk = 190476;
 350 }
 351
 352 static void pnv_get_cdclk(struct drm_i915_private *dev_priv,
 353                           struct intel_cdclk_config *cdclk_config)
 354 {
 355         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 356         u16 gcfgc = 0;
 357
 358         pci_read_config_word(pdev, GCFGC, &gcfgc);
 359
 360         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 361         case GC_DISPLAY_CLOCK_267_MHZ_PNV:
 362                 cdclk_config->cdclk = 266667;
 363                 break;
 364         case GC_DISPLAY_CLOCK_333_MHZ_PNV:
 365                 cdclk_config->cdclk = 333333;
 366                 break;
 367         case GC_DISPLAY_CLOCK_444_MHZ_PNV:
 368                 cdclk_config->cdclk = 444444;
 369                 break;
 370         case GC_DISPLAY_CLOCK_200_MHZ_PNV:
 371                 cdclk_config->cdclk = 200000;
 372                 break;
 373         default:
 374                 drm_err(&dev_priv->drm,
 375                         "Unknown pnv display core clock 0x%04x\n", gcfgc);
 376                 fallthrough;
 377         case GC_DISPLAY_CLOCK_133_MHZ_PNV:
 378                 cdclk_config->cdclk = 133333;
 379                 break;
 380         case GC_DISPLAY_CLOCK_167_MHZ_PNV:
 381                 cdclk_config->cdclk = 166667;
 382                 break;
 383         }
 384 }
 385
 386 static void i965gm_get_cdclk(struct drm_i915_private *dev_priv,
 387                              struct intel_cdclk_config *cdclk_config)
 388 {
 389         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 390         static const u8 div_3200[] = { 16, 10,  8 };
 391         static const u8 div_4000[] = { 20, 12, 10 };
 392         static const u8 div_5333[] = { 24, 16, 14 };
 393         const u8 *div_table;
 394         unsigned int cdclk_sel;
 395         u16 tmp = 0;
 396
 397         cdclk_config->vco = intel_hpll_vco(dev_priv);
 398
 399         pci_read_config_word(pdev, GCFGC, &tmp);
 400
 401         cdclk_sel = ((tmp >> 8) & 0x1f) - 1;
 402
 403         if (cdclk_sel >= ARRAY_SIZE(div_3200))
 404                 goto fail;
 405
 406         switch (cdclk_config->vco) {
 407         case 3200000:
 408                 div_table = div_3200;
 409                 break;
 410         case 4000000:
 411                 div_table = div_4000;
 412                 break;
 413         case 5333333:
 414                 div_table = div_5333;
 415                 break;
 416         default:
 417                 goto fail;
 418         }
 419
 420         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 421                                                 div_table[cdclk_sel]);
 422         return;
 423
 424 fail:
 425         drm_err(&dev_priv->drm,
 426                 "Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n",
 427                 cdclk_config->vco, tmp);
 428         cdclk_config->cdclk = 200000;
 429 }
 430
 431 static void gm45_get_cdclk(struct drm_i915_private *dev_priv,
 432                            struct intel_cdclk_config *cdclk_config)
 433 {
 434         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 435         unsigned int cdclk_sel;
 436         u16 tmp = 0;
 437
 438         cdclk_config->vco = intel_hpll_vco(dev_priv);
 439
 440         pci_read_config_word(pdev, GCFGC, &tmp);
 441
 442         cdclk_sel = (tmp >> 12) & 0x1;
 443
 444         switch (cdclk_config->vco) {
 445         case 2666667:
 446         case 4000000:
 447         case 5333333:
 448                 cdclk_config->cdclk = cdclk_sel ? 333333 : 222222;
 449                 break;
 450         case 3200000:
 451                 cdclk_config->cdclk = cdclk_sel ? 320000 : 228571;
 452                 break;
 453         default:
 454                 drm_err(&dev_priv->drm,
 455                         "Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n",
 456                         cdclk_config->vco, tmp);
 457                 cdclk_config->cdclk = 222222;
 458                 break;
 459         }
 460 }
 461
 462 static void hsw_get_cdclk(struct drm_i915_private *dev_priv,
 463                           struct intel_cdclk_config *cdclk_config)
 464 {
 465         u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 466         u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 467
 468         if (lcpll & LCPLL_CD_SOURCE_FCLK)
 469                 cdclk_config->cdclk = 800000;
 470         else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 471                 cdclk_config->cdclk = 450000;
 472         else if (freq == LCPLL_CLK_FREQ_450)
 473                 cdclk_config->cdclk = 450000;
 474         else if (IS_HASWELL_ULT(dev_priv))
 475                 cdclk_config->cdclk = 337500;
 476         else
 477                 cdclk_config->cdclk = 540000;
 478 }
 479
 480 static int vlv_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
 481 {
 482         int freq_320 = (dev_priv->hpll_freq <<  1) % 320000 != 0 ?
 483                 333333 : 320000;
 484
 485         /*
 486          * We seem to get an unstable or solid color picture at 200MHz.
 487          * Not sure what's wrong. For now use 200MHz only when all pipes
 488          * are off.
 489          */
 490         if (IS_VALLEYVIEW(dev_priv) && min_cdclk > freq_320)
 491                 return 400000;
 492         else if (min_cdclk > 266667)
 493                 return freq_320;
 494         else if (min_cdclk > 0)
 495                 return 266667;
 496         else
 497                 return 200000;
 498 }
 499
 500 static u8 vlv_calc_voltage_level(struct drm_i915_private *dev_priv, int cdclk)
 501 {
 502         if (IS_VALLEYVIEW(dev_priv)) {
 503                 if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
 504                         return 2;
 505                 else if (cdclk >= 266667)
 506                         return 1;
 507                 else
 508                         return 0;
 509         } else {
 510                 /*
 511                  * Specs are full of misinformation, but testing on actual
 512                  * hardware has shown that we just need to write the desired
 513                  * CCK divider into the Punit register.
 514                  */
 515                 return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
 516         }
 517 }
 518
 519 static void vlv_get_cdclk(struct drm_i915_private *dev_priv,
 520                           struct intel_cdclk_config *cdclk_config)
 521 {
 522         u32 val;
 523
 524         vlv_iosf_sb_get(dev_priv,
 525                         BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 526
 527         cdclk_config->vco = vlv_get_hpll_vco(dev_priv);
 528         cdclk_config->cdclk = vlv_get_cck_clock(dev_priv, "cdclk",
 529                                                 CCK_DISPLAY_CLOCK_CONTROL,
 530                                                 cdclk_config->vco);
 531
 532         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 533
 534         vlv_iosf_sb_put(dev_priv,
 535                         BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 536
 537         if (IS_VALLEYVIEW(dev_priv))
 538                 cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK) >>
 539                         DSPFREQGUAR_SHIFT;
 540         else
 541                 cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK_CHV) >>
 542                         DSPFREQGUAR_SHIFT_CHV;
 543 }
 544
 545 static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
 546 {
 547         unsigned int credits, default_credits;
 548
 549         if (IS_CHERRYVIEW(dev_priv))
 550                 default_credits = PFI_CREDIT(12);
 551         else
 552                 default_credits = PFI_CREDIT(8);
 553
 554         if (dev_priv->display.cdclk.hw.cdclk >= dev_priv->czclk_freq) {
 555                 /* CHV suggested value is 31 or 63 */
 556                 if (IS_CHERRYVIEW(dev_priv))
 557                         credits = PFI_CREDIT_63;
 558                 else
 559                         credits = PFI_CREDIT(15);
 560         } else {
 561                 credits = default_credits;
 562         }
 563
 564         /*
 565          * WA - write default credits before re-programming
 566          * FIXME: should we also set the resend bit here?
 567          */
 568         intel_de_write(dev_priv, GCI_CONTROL,
 569                        VGA_FAST_MODE_DISABLE | default_credits);
 570
 571         intel_de_write(dev_priv, GCI_CONTROL,
 572                        VGA_FAST_MODE_DISABLE | credits | PFI_CREDIT_RESEND);
 573
 574         /*
 575          * FIXME is this guaranteed to clear
 576          * immediately or should we poll for it?
 577          */
 578         drm_WARN_ON(&dev_priv->drm,
 579                     intel_de_read(dev_priv, GCI_CONTROL) & PFI_CREDIT_RESEND);
 580 }
 581
 582 static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
 583                           const struct intel_cdclk_config *cdclk_config,
 584                           enum pipe pipe)
 585 {
 586         int cdclk = cdclk_config->cdclk;
 587         u32 val, cmd = cdclk_config->voltage_level;
 588         intel_wakeref_t wakeref;
 589
 590         switch (cdclk) {
 591         case 400000:
 592         case 333333:
 593         case 320000:
 594         case 266667:
 595         case 200000:
 596                 break;
 597         default:
 598                 MISSING_CASE(cdclk);
 599                 return;
 600         }
 601
 602         /* There are cases where we can end up here with power domains
 603          * off and a CDCLK frequency other than the minimum, like when
 604          * issuing a modeset without actually changing any display after
 605          * a system suspend.  So grab the display core domain, which covers
 606          * the HW blocks needed for the following programming.
 607          */
 608         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 609
 610         vlv_iosf_sb_get(dev_priv,
 611                         BIT(VLV_IOSF_SB_CCK) |
 612                         BIT(VLV_IOSF_SB_BUNIT) |
 613                         BIT(VLV_IOSF_SB_PUNIT));
 614
 615         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 616         val &= ~DSPFREQGUAR_MASK;
 617         val |= (cmd << DSPFREQGUAR_SHIFT);
 618         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 619         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 620                       DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
 621                      50)) {
 622                 drm_err(&dev_priv->drm,
 623                         "timed out waiting for CDclk change\n");
 624         }
 625
 626         if (cdclk == 400000) {
 627                 u32 divider;
 628
 629                 divider = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1,
 630                                             cdclk) - 1;
 631
 632                 /* adjust cdclk divider */
 633                 val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
 634                 val &= ~CCK_FREQUENCY_VALUES;
 635                 val |= divider;
 636                 vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val);
 637
 638                 if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) &
 639                               CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT),
 640                              50))
 641                         drm_err(&dev_priv->drm,
 642                                 "timed out waiting for CDclk change\n");
 643         }
 644
 645         /* adjust self-refresh exit latency value */
 646         val = vlv_bunit_read(dev_priv, BUNIT_REG_BISOC);
 647         val &= ~0x7f;
 648
 649         /*
 650          * For high bandwidth configs, we set a higher latency in the bunit
 651          * so that the core display fetch happens in time to avoid underruns.
 652          */
 653         if (cdclk == 400000)
 654                 val |= 4500 / 250; /* 4.5 usec */
 655         else
 656                 val |= 3000 / 250; /* 3.0 usec */
 657         vlv_bunit_write(dev_priv, BUNIT_REG_BISOC, val);
 658
 659         vlv_iosf_sb_put(dev_priv,
 660                         BIT(VLV_IOSF_SB_CCK) |
 661                         BIT(VLV_IOSF_SB_BUNIT) |
 662                         BIT(VLV_IOSF_SB_PUNIT));
 663
 664         intel_update_cdclk(dev_priv);
 665
 666         vlv_program_pfi_credits(dev_priv);
 667
 668         intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 669 }
 670
 671 static void chv_set_cdclk(struct drm_i915_private *dev_priv,
 672                           const struct intel_cdclk_config *cdclk_config,
 673                           enum pipe pipe)
 674 {
 675         int cdclk = cdclk_config->cdclk;
 676         u32 val, cmd = cdclk_config->voltage_level;
 677         intel_wakeref_t wakeref;
 678
 679         switch (cdclk) {
 680         case 333333:
 681         case 320000:
 682         case 266667:
 683         case 200000:
 684                 break;
 685         default:
 686                 MISSING_CASE(cdclk);
 687                 return;
 688         }
 689
 690         /* There are cases where we can end up here with power domains
 691          * off and a CDCLK frequency other than the minimum, like when
 692          * issuing a modeset without actually changing any display after
 693          * a system suspend.  So grab the display core domain, which covers
 694          * the HW blocks needed for the following programming.
 695          */
 696         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 697
 698         vlv_punit_get(dev_priv);
 699         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 700         val &= ~DSPFREQGUAR_MASK_CHV;
 701         val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
 702         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 703         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 704                       DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
 705                      50)) {
 706                 drm_err(&dev_priv->drm,
 707                         "timed out waiting for CDclk change\n");
 708         }
 709
 710         vlv_punit_put(dev_priv);
 711
 712         intel_update_cdclk(dev_priv);
 713
 714         vlv_program_pfi_credits(dev_priv);
 715
 716         intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 717 }
 718
 719 static int bdw_calc_cdclk(int min_cdclk)
 720 {
 721         if (min_cdclk > 540000)
 722                 return 675000;
 723         else if (min_cdclk > 450000)
 724                 return 540000;
 725         else if (min_cdclk > 337500)
 726                 return 450000;
 727         else
 728                 return 337500;
 729 }
 730
 731 static u8 bdw_calc_voltage_level(int cdclk)
 732 {
 733         switch (cdclk) {
 734         default:
 735         case 337500:
 736                 return 2;
 737         case 450000:
 738                 return 0;
 739         case 540000:
 740                 return 1;
 741         case 675000:
 742                 return 3;
 743         }
 744 }
 745
 746 static void bdw_get_cdclk(struct drm_i915_private *dev_priv,
 747                           struct intel_cdclk_config *cdclk_config)
 748 {
 749         u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 750         u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 751
 752         if (lcpll & LCPLL_CD_SOURCE_FCLK)
 753                 cdclk_config->cdclk = 800000;
 754         else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 755                 cdclk_config->cdclk = 450000;
 756         else if (freq == LCPLL_CLK_FREQ_450)
 757                 cdclk_config->cdclk = 450000;
 758         else if (freq == LCPLL_CLK_FREQ_54O_BDW)
 759                 cdclk_config->cdclk = 540000;
 760         else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
 761                 cdclk_config->cdclk = 337500;
 762         else
 763                 cdclk_config->cdclk = 675000;
 764
 765         /*
 766          * Can't read this out :( Let's assume it's
 767          * at least what the CDCLK frequency requires.
 768          */
 769         cdclk_config->voltage_level =
 770                 bdw_calc_voltage_level(cdclk_config->cdclk);
 771 }
 772
 773 static u32 bdw_cdclk_freq_sel(int cdclk)
 774 {
 775         switch (cdclk) {
 776         default:
 777                 MISSING_CASE(cdclk);
 778                 fallthrough;
 779         case 337500:
 780                 return LCPLL_CLK_FREQ_337_5_BDW;
 781         case 450000:
 782                 return LCPLL_CLK_FREQ_450;
 783         case 540000:
 784                 return LCPLL_CLK_FREQ_54O_BDW;
 785         case 675000:
 786                 return LCPLL_CLK_FREQ_675_BDW;
 787         }
 788 }
 789
 790 static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
 791                           const struct intel_cdclk_config *cdclk_config,
 792                           enum pipe pipe)
 793 {
 794         int cdclk = cdclk_config->cdclk;
 795         int ret;
 796
 797         if (drm_WARN(&dev_priv->drm,
 798                      (intel_de_read(dev_priv, LCPLL_CTL) &
 799                       (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK |
 800                        LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE |
 801                        LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW |
 802                        LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK,
 803                      "trying to change cdclk frequency with cdclk not enabled\n"))
 804                 return;
 805
 806         ret = snb_pcode_write(&dev_priv->uncore, BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
 807         if (ret) {
 808                 drm_err(&dev_priv->drm,
 809                         "failed to inform pcode about cdclk change\n");
 810                 return;
 811         }
 812
 813         intel_de_rmw(dev_priv, LCPLL_CTL,
 814                      0, LCPLL_CD_SOURCE_FCLK);
 815
 816         /*
 817          * According to the spec, it should be enough to poll for this 1 us.
 818          * However, extensive testing shows that this can take longer.
 819          */
 820         if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
 821                         LCPLL_CD_SOURCE_FCLK_DONE, 100))
 822                 drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
 823
 824         intel_de_rmw(dev_priv, LCPLL_CTL,
 825                      LCPLL_CLK_FREQ_MASK, bdw_cdclk_freq_sel(cdclk));
 826
 827         intel_de_rmw(dev_priv, LCPLL_CTL,
 828                      LCPLL_CD_SOURCE_FCLK, 0);
 829
 830         if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
 831                          LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
 832                 drm_err(&dev_priv->drm, "Switching back to LCPLL failed\n");
 833
 834         snb_pcode_write(&dev_priv->uncore, HSW_PCODE_DE_WRITE_FREQ_REQ,
 835                         cdclk_config->voltage_level);
 836
 837         intel_de_write(dev_priv, CDCLK_FREQ,
 838                        DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
 839
 840         intel_update_cdclk(dev_priv);
 841 }
 842
 843 static int skl_calc_cdclk(int min_cdclk, int vco)
 844 {
 845         if (vco == 8640000) {
 846                 if (min_cdclk > 540000)
 847                         return 617143;
 848                 else if (min_cdclk > 432000)
 849                         return 540000;
 850                 else if (min_cdclk > 308571)
 851                         return 432000;
 852                 else
 853                         return 308571;
 854         } else {
 855                 if (min_cdclk > 540000)
 856                         return 675000;
 857                 else if (min_cdclk > 450000)
 858                         return 540000;
 859                 else if (min_cdclk > 337500)
 860                         return 450000;
 861                 else
 862                         return 337500;
 863         }
 864 }
 865
 866 static u8 skl_calc_voltage_level(int cdclk)
 867 {
 868         if (cdclk > 540000)
 869                 return 3;
 870         else if (cdclk > 450000)
 871                 return 2;
 872         else if (cdclk > 337500)
 873                 return 1;
 874         else
 875                 return 0;
 876 }
 877
 878 static void skl_dpll0_update(struct drm_i915_private *dev_priv,
 879                              struct intel_cdclk_config *cdclk_config)
 880 {
 881         u32 val;
 882
 883         cdclk_config->ref = 24000;
 884         cdclk_config->vco = 0;
 885
 886         val = intel_de_read(dev_priv, LCPLL1_CTL);
 887         if ((val & LCPLL_PLL_ENABLE) == 0)
 888                 return;
 889
 890         if (drm_WARN_ON(&dev_priv->drm, (val & LCPLL_PLL_LOCK) == 0))
 891                 return;
 892
 893         val = intel_de_read(dev_priv, DPLL_CTRL1);
 894
 895         if (drm_WARN_ON(&dev_priv->drm,
 896                         (val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
 897                                 DPLL_CTRL1_SSC(SKL_DPLL0) |
 898                                 DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) !=
 899                         DPLL_CTRL1_OVERRIDE(SKL_DPLL0)))
 900                 return;
 901
 902         switch (val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) {
 903         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0):
 904         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, SKL_DPLL0):
 905         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, SKL_DPLL0):
 906         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, SKL_DPLL0):
 907                 cdclk_config->vco = 8100000;
 908                 break;
 909         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0):
 910         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, SKL_DPLL0):
 911                 cdclk_config->vco = 8640000;
 912                 break;
 913         default:
 914                 MISSING_CASE(val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
 915                 break;
 916         }
 917 }
 918
 919 static void skl_get_cdclk(struct drm_i915_private *dev_priv,
 920                           struct intel_cdclk_config *cdclk_config)
 921 {
 922         u32 cdctl;
 923
 924         skl_dpll0_update(dev_priv, cdclk_config);
 925
 926         cdclk_config->cdclk = cdclk_config->bypass = cdclk_config->ref;
 927
 928         if (cdclk_config->vco == 0)
 929                 goto out;
 930
 931         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
 932
 933         if (cdclk_config->vco == 8640000) {
 934                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 935                 case CDCLK_FREQ_450_432:
 936                         cdclk_config->cdclk = 432000;
 937                         break;
 938                 case CDCLK_FREQ_337_308:
 939                         cdclk_config->cdclk = 308571;
 940                         break;
 941                 case CDCLK_FREQ_540:
 942                         cdclk_config->cdclk = 540000;
 943                         break;
 944                 case CDCLK_FREQ_675_617:
 945                         cdclk_config->cdclk = 617143;
 946                         break;
 947                 default:
 948                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
 949                         break;
 950                 }
 951         } else {
 952                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 953                 case CDCLK_FREQ_450_432:
 954                         cdclk_config->cdclk = 450000;
 955                         break;
 956                 case CDCLK_FREQ_337_308:
 957                         cdclk_config->cdclk = 337500;
 958                         break;
 959                 case CDCLK_FREQ_540:
 960                         cdclk_config->cdclk = 540000;
 961                         break;
 962                 case CDCLK_FREQ_675_617:
 963                         cdclk_config->cdclk = 675000;
 964                         break;
 965                 default:
 966                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
 967                         break;
 968                 }
 969         }
 970
 971  out:
 972         /*
 973          * Can't read this out :( Let's assume it's
 974          * at least what the CDCLK frequency requires.
 975          */
 976         cdclk_config->voltage_level =
 977                 skl_calc_voltage_level(cdclk_config->cdclk);
 978 }
 979
 980 /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
 981 static int skl_cdclk_decimal(int cdclk)
 982 {
 983         return DIV_ROUND_CLOSEST(cdclk - 1000, 500);
 984 }
 985
 986 static void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv,
 987                                         int vco)
 988 {
 989         bool changed = dev_priv->skl_preferred_vco_freq != vco;
 990
 991         dev_priv->skl_preferred_vco_freq = vco;
 992
 993         if (changed)
 994                 intel_update_max_cdclk(dev_priv);
 995 }
 996
 997 static u32 skl_dpll0_link_rate(struct drm_i915_private *dev_priv, int vco)
 998 {
 999         drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
1000
1001         /*
1002          * We always enable DPLL0 with the lowest link rate possible, but still
1003          * taking into account the VCO required to operate the eDP panel at the
1004          * desired frequency. The usual DP link rates operate with a VCO of
1005          * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640.
1006          * The modeset code is responsible for the selection of the exact link
1007          * rate later on, with the constraint of choosing a frequency that
1008          * works with vco.
1009          */
1010         if (vco == 8640000)
1011                 return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0);
1012         else
1013                 return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0);
1014 }
1015
1016 static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco)
1017 {
1018         intel_de_rmw(dev_priv, DPLL_CTRL1,
1019                      DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
1020                      DPLL_CTRL1_SSC(SKL_DPLL0) |
1021                      DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0),
1022                      DPLL_CTRL1_OVERRIDE(SKL_DPLL0) |
1023                      skl_dpll0_link_rate(dev_priv, vco));
1024         intel_de_posting_read(dev_priv, DPLL_CTRL1);
1025
1026         intel_de_rmw(dev_priv, LCPLL1_CTL,
1027                      0, LCPLL_PLL_ENABLE);
1028
1029         if (intel_de_wait_for_set(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 5))
1030                 drm_err(&dev_priv->drm, "DPLL0 not locked\n");
1031
1032         dev_priv->display.cdclk.hw.vco = vco;
1033
1034         /* We'll want to keep using the current vco from now on. */
1035         skl_set_preferred_cdclk_vco(dev_priv, vco);
1036 }
1037
1038 static void skl_dpll0_disable(struct drm_i915_private *dev_priv)
1039 {
1040         intel_de_rmw(dev_priv, LCPLL1_CTL,
1041                      LCPLL_PLL_ENABLE, 0);
1042
1043         if (intel_de_wait_for_clear(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 1))
1044                 drm_err(&dev_priv->drm, "Couldn't disable DPLL0\n");
1045
1046         dev_priv->display.cdclk.hw.vco = 0;
1047 }
1048
1049 static u32 skl_cdclk_freq_sel(struct drm_i915_private *dev_priv,
1050                               int cdclk, int vco)
1051 {
1052         switch (cdclk) {
1053         default:
1054                 drm_WARN_ON(&dev_priv->drm,
1055                             cdclk != dev_priv->display.cdclk.hw.bypass);
1056                 drm_WARN_ON(&dev_priv->drm, vco != 0);
1057                 fallthrough;
1058         case 308571:
1059         case 337500:
1060                 return CDCLK_FREQ_337_308;
1061         case 450000:
1062         case 432000:
1063                 return CDCLK_FREQ_450_432;
1064         case 540000:
1065                 return CDCLK_FREQ_540;
1066         case 617143:
1067         case 675000:
1068                 return CDCLK_FREQ_675_617;
1069         }
1070 }
1071
1072 static void skl_set_cdclk(struct drm_i915_private *dev_priv,
1073                           const struct intel_cdclk_config *cdclk_config,
1074                           enum pipe pipe)
1075 {
1076         int cdclk = cdclk_config->cdclk;
1077         int vco = cdclk_config->vco;
1078         u32 freq_select, cdclk_ctl;
1079         int ret;
1080
1081         /*
1082          * Based on WA#1183 CDCLK rates 308 and 617MHz CDCLK rates are
1083          * unsupported on SKL. In theory this should never happen since only
1084          * the eDP1.4 2.16 and 4.32Gbps rates require it, but eDP1.4 is not
1085          * supported on SKL either, see the above WA. WARN whenever trying to
1086          * use the corresponding VCO freq as that always leads to using the
1087          * minimum 308MHz CDCLK.
1088          */
1089         drm_WARN_ON_ONCE(&dev_priv->drm,
1090                          IS_SKYLAKE(dev_priv) && vco == 8640000);
1091
1092         ret = skl_pcode_request(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1093                                 SKL_CDCLK_PREPARE_FOR_CHANGE,
1094                                 SKL_CDCLK_READY_FOR_CHANGE,
1095                                 SKL_CDCLK_READY_FOR_CHANGE, 3);
1096         if (ret) {
1097                 drm_err(&dev_priv->drm,
1098                         "Failed to inform PCU about cdclk change (%d)\n", ret);
1099                 return;
1100         }
1101
1102         freq_select = skl_cdclk_freq_sel(dev_priv, cdclk, vco);
1103
1104         if (dev_priv->display.cdclk.hw.vco != 0 &&
1105             dev_priv->display.cdclk.hw.vco != vco)
1106                 skl_dpll0_disable(dev_priv);
1107
1108         cdclk_ctl = intel_de_read(dev_priv, CDCLK_CTL);
1109
1110         if (dev_priv->display.cdclk.hw.vco != vco) {
1111                 /* Wa Display #1183: skl,kbl,cfl */
1112                 cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1113                 cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1114                 intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1115         }
1116
1117         /* Wa Display #1183: skl,kbl,cfl */
1118         cdclk_ctl |= CDCLK_DIVMUX_CD_OVERRIDE;
1119         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1120         intel_de_posting_read(dev_priv, CDCLK_CTL);
1121
1122         if (dev_priv->display.cdclk.hw.vco != vco)
1123                 skl_dpll0_enable(dev_priv, vco);
1124
1125         /* Wa Display #1183: skl,kbl,cfl */
1126         cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1127         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1128
1129         cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1130         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1131
1132         /* Wa Display #1183: skl,kbl,cfl */
1133         cdclk_ctl &= ~CDCLK_DIVMUX_CD_OVERRIDE;
1134         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1135         intel_de_posting_read(dev_priv, CDCLK_CTL);
1136
1137         /* inform PCU of the change */
1138         snb_pcode_write(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1139                         cdclk_config->voltage_level);
1140
1141         intel_update_cdclk(dev_priv);
1142 }
1143
1144 static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
1145 {
1146         u32 cdctl, expected;
1147
1148         /*
1149          * check if the pre-os initialized the display
1150          * There is SWF18 scratchpad register defined which is set by the
1151          * pre-os which can be used by the OS drivers to check the status
1152          */
1153         if ((intel_de_read(dev_priv, SWF_ILK(0x18)) & 0x00FFFFFF) == 0)
1154                 goto sanitize;
1155
1156         intel_update_cdclk(dev_priv);
1157         intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
1158
1159         /* Is PLL enabled and locked ? */
1160         if (dev_priv->display.cdclk.hw.vco == 0 ||
1161             dev_priv->display.cdclk.hw.cdclk == dev_priv->display.cdclk.hw.bypass)
1162                 goto sanitize;
1163
1164         /* DPLL okay; verify the cdclock
1165          *
1166          * Noticed in some instances that the freq selection is correct but
1167          * decimal part is programmed wrong from BIOS where pre-os does not
1168          * enable display. Verify the same as well.
1169          */
1170         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1171         expected = (cdctl & CDCLK_FREQ_SEL_MASK) |
1172                 skl_cdclk_decimal(dev_priv->display.cdclk.hw.cdclk);
1173         if (cdctl == expected)
1174                 /* All well; nothing to sanitize */
1175                 return;
1176
1177 sanitize:
1178         drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1179
1180         /* force cdclk programming */
1181         dev_priv->display.cdclk.hw.cdclk = 0;
1182         /* force full PLL disable + enable */
1183         dev_priv->display.cdclk.hw.vco = -1;
1184 }
1185
1186 static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv)
1187 {
1188         struct intel_cdclk_config cdclk_config;
1189
1190         skl_sanitize_cdclk(dev_priv);
1191
1192         if (dev_priv->display.cdclk.hw.cdclk != 0 &&
1193             dev_priv->display.cdclk.hw.vco != 0) {
1194                 /*
1195                  * Use the current vco as our initial
1196                  * guess as to what the preferred vco is.
1197                  */
1198                 if (dev_priv->skl_preferred_vco_freq == 0)
1199                         skl_set_preferred_cdclk_vco(dev_priv,
1200                                                     dev_priv->display.cdclk.hw.vco);
1201                 return;
1202         }
1203
1204         cdclk_config = dev_priv->display.cdclk.hw;
1205
1206         cdclk_config.vco = dev_priv->skl_preferred_vco_freq;
1207         if (cdclk_config.vco == 0)
1208                 cdclk_config.vco = 8100000;
1209         cdclk_config.cdclk = skl_calc_cdclk(0, cdclk_config.vco);
1210         cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1211
1212         skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1213 }
1214
1215 static void skl_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
1216 {
1217         struct intel_cdclk_config cdclk_config = dev_priv->display.cdclk.hw;
1218
1219         cdclk_config.cdclk = cdclk_config.bypass;
1220         cdclk_config.vco = 0;
1221         cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1222
1223         skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1224 }
1225
1226 struct intel_cdclk_vals {
1227         u32 cdclk;
1228         u16 refclk;
1229         u16 waveform;
1230         u8 divider;     /* CD2X divider * 2 */
1231         u8 ratio;
1232 };
1233
1234 static const struct intel_cdclk_vals bxt_cdclk_table[] = {
1235         { .refclk = 19200, .cdclk = 144000, .divider = 8, .ratio = 60 },
1236         { .refclk = 19200, .cdclk = 288000, .divider = 4, .ratio = 60 },
1237         { .refclk = 19200, .cdclk = 384000, .divider = 3, .ratio = 60 },
1238         { .refclk = 19200, .cdclk = 576000, .divider = 2, .ratio = 60 },
1239         { .refclk = 19200, .cdclk = 624000, .divider = 2, .ratio = 65 },
1240         {}
1241 };
1242
1243 static const struct intel_cdclk_vals glk_cdclk_table[] = {
1244         { .refclk = 19200, .cdclk =  79200, .divider = 8, .ratio = 33 },
1245         { .refclk = 19200, .cdclk = 158400, .divider = 4, .ratio = 33 },
1246         { .refclk = 19200, .cdclk = 316800, .divider = 2, .ratio = 33 },
1247         {}
1248 };
1249
1250 static const struct intel_cdclk_vals icl_cdclk_table[] = {
1251         { .refclk = 19200, .cdclk = 172800, .divider = 2, .ratio = 18 },
1252         { .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1253         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1254         { .refclk = 19200, .cdclk = 326400, .divider = 4, .ratio = 68 },
1255         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1256         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1257
1258         { .refclk = 24000, .cdclk = 180000, .divider = 2, .ratio = 15 },
1259         { .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1260         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1261         { .refclk = 24000, .cdclk = 324000, .divider = 4, .ratio = 54 },
1262         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1263         { .refclk = 24000, .cdclk = 648000, .divider = 2, .ratio = 54 },
1264
1265         { .refclk = 38400, .cdclk = 172800, .divider = 2, .ratio =  9 },
1266         { .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1267         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1268         { .refclk = 38400, .cdclk = 326400, .divider = 4, .ratio = 34 },
1269         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1270         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1271         {}
1272 };
1273
1274 static const struct intel_cdclk_vals rkl_cdclk_table[] = {
1275         { .refclk = 19200, .cdclk = 172800, .divider = 4, .ratio =  36 },
1276         { .refclk = 19200, .cdclk = 192000, .divider = 4, .ratio =  40 },
1277         { .refclk = 19200, .cdclk = 307200, .divider = 4, .ratio =  64 },
1278         { .refclk = 19200, .cdclk = 326400, .divider = 8, .ratio = 136 },
1279         { .refclk = 19200, .cdclk = 556800, .divider = 4, .ratio = 116 },
1280         { .refclk = 19200, .cdclk = 652800, .divider = 4, .ratio = 136 },
1281
1282         { .refclk = 24000, .cdclk = 180000, .divider = 4, .ratio =  30 },
1283         { .refclk = 24000, .cdclk = 192000, .divider = 4, .ratio =  32 },
1284         { .refclk = 24000, .cdclk = 312000, .divider = 4, .ratio =  52 },
1285         { .refclk = 24000, .cdclk = 324000, .divider = 8, .ratio = 108 },
1286         { .refclk = 24000, .cdclk = 552000, .divider = 4, .ratio =  92 },
1287         { .refclk = 24000, .cdclk = 648000, .divider = 4, .ratio = 108 },
1288
1289         { .refclk = 38400, .cdclk = 172800, .divider = 4, .ratio = 18 },
1290         { .refclk = 38400, .cdclk = 192000, .divider = 4, .ratio = 20 },
1291         { .refclk = 38400, .cdclk = 307200, .divider = 4, .ratio = 32 },
1292         { .refclk = 38400, .cdclk = 326400, .divider = 8, .ratio = 68 },
1293         { .refclk = 38400, .cdclk = 556800, .divider = 4, .ratio = 58 },
1294         { .refclk = 38400, .cdclk = 652800, .divider = 4, .ratio = 68 },
1295         {}
1296 };
1297
1298 static const struct intel_cdclk_vals adlp_a_step_cdclk_table[] = {
1299         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1300         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1301         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1302
1303         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1304         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1305         { .refclk = 24400, .cdclk = 648000, .divider = 2, .ratio = 54 },
1306
1307         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1308         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1309         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1310         {}
1311 };
1312
1313 static const struct intel_cdclk_vals adlp_cdclk_table[] = {
1314         { .refclk = 19200, .cdclk = 172800, .divider = 3, .ratio = 27 },
1315         { .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1316         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1317         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1318         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1319
1320         { .refclk = 24000, .cdclk = 176000, .divider = 3, .ratio = 22 },
1321         { .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1322         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1323         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1324         { .refclk = 24000, .cdclk = 648000, .divider = 2, .ratio = 54 },
1325
1326         { .refclk = 38400, .cdclk = 179200, .divider = 3, .ratio = 14 },
1327         { .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1328         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1329         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1330         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1331         {}
1332 };
1333
1334 static const struct intel_cdclk_vals rplu_cdclk_table[] = {
1335         { .refclk = 19200, .cdclk = 172800, .divider = 3, .ratio = 27 },
1336         { .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1337         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1338         { .refclk = 19200, .cdclk = 480000, .divider = 2, .ratio = 50 },
1339         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1340         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1341
1342         { .refclk = 24000, .cdclk = 176000, .divider = 3, .ratio = 22 },
1343         { .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1344         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1345         { .refclk = 24000, .cdclk = 480000, .divider = 2, .ratio = 40 },
1346         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1347         { .refclk = 24000, .cdclk = 648000, .divider = 2, .ratio = 54 },
1348
1349         { .refclk = 38400, .cdclk = 179200, .divider = 3, .ratio = 14 },
1350         { .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1351         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1352         { .refclk = 38400, .cdclk = 480000, .divider = 2, .ratio = 25 },
1353         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1354         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1355         {}
1356 };
1357
1358 static const struct intel_cdclk_vals dg2_cdclk_table[] = {
1359         { .refclk = 38400, .cdclk = 163200, .divider = 2, .ratio = 34, .waveform = 0x8888 },
1360         { .refclk = 38400, .cdclk = 204000, .divider = 2, .ratio = 34, .waveform = 0x9248 },
1361         { .refclk = 38400, .cdclk = 244800, .divider = 2, .ratio = 34, .waveform = 0xa4a4 },
1362         { .refclk = 38400, .cdclk = 285600, .divider = 2, .ratio = 34, .waveform = 0xa54a },
1363         { .refclk = 38400, .cdclk = 326400, .divider = 2, .ratio = 34, .waveform = 0xaaaa },
1364         { .refclk = 38400, .cdclk = 367200, .divider = 2, .ratio = 34, .waveform = 0xad5a },
1365         { .refclk = 38400, .cdclk = 408000, .divider = 2, .ratio = 34, .waveform = 0xb6b6 },
1366         { .refclk = 38400, .cdclk = 448800, .divider = 2, .ratio = 34, .waveform = 0xdbb6 },
1367         { .refclk = 38400, .cdclk = 489600, .divider = 2, .ratio = 34, .waveform = 0xeeee },
1368         { .refclk = 38400, .cdclk = 530400, .divider = 2, .ratio = 34, .waveform = 0xf7de },
1369         { .refclk = 38400, .cdclk = 571200, .divider = 2, .ratio = 34, .waveform = 0xfefe },
1370         { .refclk = 38400, .cdclk = 612000, .divider = 2, .ratio = 34, .waveform = 0xfffe },
1371         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34, .waveform = 0xffff },
1372         {}
1373 };
1374
1375 static const struct intel_cdclk_vals mtl_cdclk_table[] = {
1376         { .refclk = 38400, .cdclk = 172800, .divider = 2, .ratio = 16, .waveform = 0xad5a },
1377         { .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 16, .waveform = 0xb6b6 },
1378         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16, .waveform = 0x0000 },
1379         { .refclk = 38400, .cdclk = 480000, .divider = 2, .ratio = 25, .waveform = 0x0000 },
1380         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29, .waveform = 0x0000 },
1381         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34, .waveform = 0x0000 },
1382         {}
1383 };
1384
1385 static const struct intel_cdclk_vals lnl_cdclk_table[] = {
1386         { .refclk = 38400, .cdclk = 153600, .divider = 2, .ratio = 16, .waveform = 0xaaaa },
1387         { .refclk = 38400, .cdclk = 172800, .divider = 2, .ratio = 16, .waveform = 0xad5a },
1388         { .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 16, .waveform = 0xb6b6 },
1389         { .refclk = 38400, .cdclk = 211200, .divider = 2, .ratio = 16, .waveform = 0xdbb6 },
1390         { .refclk = 38400, .cdclk = 230400, .divider = 2, .ratio = 16, .waveform = 0xeeee },
1391         { .refclk = 38400, .cdclk = 249600, .divider = 2, .ratio = 16, .waveform = 0xf7de },
1392         { .refclk = 38400, .cdclk = 268800, .divider = 2, .ratio = 16, .waveform = 0xfefe },
1393         { .refclk = 38400, .cdclk = 288000, .divider = 2, .ratio = 16, .waveform = 0xfffe },
1394         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16, .waveform = 0xffff },
1395         { .refclk = 38400, .cdclk = 330000, .divider = 2, .ratio = 25, .waveform = 0xdbb6 },
1396         { .refclk = 38400, .cdclk = 360000, .divider = 2, .ratio = 25, .waveform = 0xeeee },
1397         { .refclk = 38400, .cdclk = 390000, .divider = 2, .ratio = 25, .waveform = 0xf7de },
1398         { .refclk = 38400, .cdclk = 420000, .divider = 2, .ratio = 25, .waveform = 0xfefe },
1399         { .refclk = 38400, .cdclk = 450000, .divider = 2, .ratio = 25, .waveform = 0xfffe },
1400         { .refclk = 38400, .cdclk = 480000, .divider = 2, .ratio = 25, .waveform = 0xffff },
1401         { .refclk = 38400, .cdclk = 487200, .divider = 2, .ratio = 29, .waveform = 0xfefe },
1402         { .refclk = 38400, .cdclk = 522000, .divider = 2, .ratio = 29, .waveform = 0xfffe },
1403         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29, .waveform = 0xffff },
1404         { .refclk = 38400, .cdclk = 571200, .divider = 2, .ratio = 34, .waveform = 0xfefe },
1405         { .refclk = 38400, .cdclk = 612000, .divider = 2, .ratio = 34, .waveform = 0xfffe },
1406         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34, .waveform = 0xffff },
1407         {}
1408 };
1409
1410 static int bxt_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
1411 {
1412         const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1413         int i;
1414
1415         for (i = 0; table[i].refclk; i++)
1416                 if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1417                     table[i].cdclk >= min_cdclk)
1418                         return table[i].cdclk;
1419
1420         drm_WARN(&dev_priv->drm, 1,
1421                  "Cannot satisfy minimum cdclk %d with refclk %u\n",
1422                  min_cdclk, dev_priv->display.cdclk.hw.ref);
1423         return 0;
1424 }
1425
1426 static int bxt_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
1427 {
1428         const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1429         int i;
1430
1431         if (cdclk == dev_priv->display.cdclk.hw.bypass)
1432                 return 0;
1433
1434         for (i = 0; table[i].refclk; i++)
1435                 if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1436                     table[i].cdclk == cdclk)
1437                         return dev_priv->display.cdclk.hw.ref * table[i].ratio;
1438
1439         drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1440                  cdclk, dev_priv->display.cdclk.hw.ref);
1441         return 0;
1442 }
1443
1444 static u8 bxt_calc_voltage_level(int cdclk)
1445 {
1446         return DIV_ROUND_UP(cdclk, 25000);
1447 }
1448
1449 static u8 icl_calc_voltage_level(int cdclk)
1450 {
1451         if (cdclk > 556800)
1452                 return 2;
1453         else if (cdclk > 312000)
1454                 return 1;
1455         else
1456                 return 0;
1457 }
1458
1459 static u8 ehl_calc_voltage_level(int cdclk)
1460 {
1461         if (cdclk > 326400)
1462                 return 3;
1463         else if (cdclk > 312000)
1464                 return 2;
1465         else if (cdclk > 180000)
1466                 return 1;
1467         else
1468                 return 0;
1469 }
1470
1471 static u8 tgl_calc_voltage_level(int cdclk)
1472 {
1473         if (cdclk > 556800)
1474                 return 3;
1475         else if (cdclk > 326400)
1476                 return 2;
1477         else if (cdclk > 312000)
1478                 return 1;
1479         else
1480                 return 0;
1481 }
1482
1483 static u8 rplu_calc_voltage_level(int cdclk)
1484 {
1485         if (cdclk > 556800)
1486                 return 3;
1487         else if (cdclk > 480000)
1488                 return 2;
1489         else if (cdclk > 312000)
1490                 return 1;
1491         else
1492                 return 0;
1493 }
1494
1495 static void icl_readout_refclk(struct drm_i915_private *dev_priv,
1496                                struct intel_cdclk_config *cdclk_config)
1497 {
1498         u32 dssm = intel_de_read(dev_priv, SKL_DSSM) & ICL_DSSM_CDCLK_PLL_REFCLK_MASK;
1499
1500         switch (dssm) {
1501         default:
1502                 MISSING_CASE(dssm);
1503                 fallthrough;
1504         case ICL_DSSM_CDCLK_PLL_REFCLK_24MHz:
1505                 cdclk_config->ref = 24000;
1506                 break;
1507         case ICL_DSSM_CDCLK_PLL_REFCLK_19_2MHz:
1508                 cdclk_config->ref = 19200;
1509                 break;
1510         case ICL_DSSM_CDCLK_PLL_REFCLK_38_4MHz:
1511                 cdclk_config->ref = 38400;
1512                 break;
1513         }
1514 }
1515
1516 static void bxt_de_pll_readout(struct drm_i915_private *dev_priv,
1517                                struct intel_cdclk_config *cdclk_config)
1518 {
1519         u32 val, ratio;
1520
1521         if (IS_DG2(dev_priv))
1522                 cdclk_config->ref = 38400;
1523         else if (DISPLAY_VER(dev_priv) >= 11)
1524                 icl_readout_refclk(dev_priv, cdclk_config);
1525         else
1526                 cdclk_config->ref = 19200;
1527
1528         val = intel_de_read(dev_priv, BXT_DE_PLL_ENABLE);
1529         if ((val & BXT_DE_PLL_PLL_ENABLE) == 0 ||
1530             (val & BXT_DE_PLL_LOCK) == 0) {
1531                 /*
1532                  * CDCLK PLL is disabled, the VCO/ratio doesn't matter, but
1533                  * setting it to zero is a way to signal that.
1534                  */
1535                 cdclk_config->vco = 0;
1536                 return;
1537         }
1538
1539         /*
1540          * DISPLAY_VER >= 11 have the ratio directly in the PLL enable register,
1541          * gen9lp had it in a separate PLL control register.
1542          */
1543         if (DISPLAY_VER(dev_priv) >= 11)
1544                 ratio = val & ICL_CDCLK_PLL_RATIO_MASK;
1545         else
1546                 ratio = intel_de_read(dev_priv, BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK;
1547
1548         cdclk_config->vco = ratio * cdclk_config->ref;
1549 }
1550
1551 static void bxt_get_cdclk(struct drm_i915_private *dev_priv,
1552                           struct intel_cdclk_config *cdclk_config)
1553 {
1554         u32 squash_ctl = 0;
1555         u32 divider;
1556         int div;
1557
1558         bxt_de_pll_readout(dev_priv, cdclk_config);
1559
1560         if (DISPLAY_VER(dev_priv) >= 12)
1561                 cdclk_config->bypass = cdclk_config->ref / 2;
1562         else if (DISPLAY_VER(dev_priv) >= 11)
1563                 cdclk_config->bypass = 50000;
1564         else
1565                 cdclk_config->bypass = cdclk_config->ref;
1566
1567         if (cdclk_config->vco == 0) {
1568                 cdclk_config->cdclk = cdclk_config->bypass;
1569                 goto out;
1570         }
1571
1572         divider = intel_de_read(dev_priv, CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
1573
1574         switch (divider) {
1575         case BXT_CDCLK_CD2X_DIV_SEL_1:
1576                 div = 2;
1577                 break;
1578         case BXT_CDCLK_CD2X_DIV_SEL_1_5:
1579                 div = 3;
1580                 break;
1581         case BXT_CDCLK_CD2X_DIV_SEL_2:
1582                 div = 4;
1583                 break;
1584         case BXT_CDCLK_CD2X_DIV_SEL_4:
1585                 div = 8;
1586                 break;
1587         default:
1588                 MISSING_CASE(divider);
1589                 return;
1590         }
1591
1592         if (HAS_CDCLK_SQUASH(dev_priv))
1593                 squash_ctl = intel_de_read(dev_priv, CDCLK_SQUASH_CTL);
1594
1595         if (squash_ctl & CDCLK_SQUASH_ENABLE) {
1596                 u16 waveform;
1597                 int size;
1598
1599                 size = REG_FIELD_GET(CDCLK_SQUASH_WINDOW_SIZE_MASK, squash_ctl) + 1;
1600                 waveform = REG_FIELD_GET(CDCLK_SQUASH_WAVEFORM_MASK, squash_ctl) >> (16 - size);
1601
1602                 cdclk_config->cdclk = DIV_ROUND_CLOSEST(hweight16(waveform) *
1603                                                         cdclk_config->vco, size * div);
1604         } else {
1605                 cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco, div);
1606         }
1607
1608  out:
1609         /*
1610          * Can't read this out :( Let's assume it's
1611          * at least what the CDCLK frequency requires.
1612          */
1613         cdclk_config->voltage_level =
1614                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config->cdclk);
1615 }
1616
1617 static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
1618 {
1619         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, 0);
1620
1621         /* Timeout 200us */
1622         if (intel_de_wait_for_clear(dev_priv,
1623                                     BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1624                 drm_err(&dev_priv->drm, "timeout waiting for DE PLL unlock\n");
1625
1626         dev_priv->display.cdclk.hw.vco = 0;
1627 }
1628
1629 static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
1630 {
1631         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1632
1633         intel_de_rmw(dev_priv, BXT_DE_PLL_CTL,
1634                      BXT_DE_PLL_RATIO_MASK, BXT_DE_PLL_RATIO(ratio));
1635
1636         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
1637
1638         /* Timeout 200us */
1639         if (intel_de_wait_for_set(dev_priv,
1640                                   BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1641                 drm_err(&dev_priv->drm, "timeout waiting for DE PLL lock\n");
1642
1643         dev_priv->display.cdclk.hw.vco = vco;
1644 }
1645
1646 static void icl_cdclk_pll_disable(struct drm_i915_private *dev_priv)
1647 {
1648         intel_de_rmw(dev_priv, BXT_DE_PLL_ENABLE,
1649                      BXT_DE_PLL_PLL_ENABLE, 0);
1650
1651         /* Timeout 200us */
1652         if (intel_de_wait_for_clear(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1653                 drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL unlock\n");
1654
1655         dev_priv->display.cdclk.hw.vco = 0;
1656 }
1657
1658 static void icl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco)
1659 {
1660         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1661         u32 val;
1662
1663         val = ICL_CDCLK_PLL_RATIO(ratio);
1664         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1665
1666         val |= BXT_DE_PLL_PLL_ENABLE;
1667         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1668
1669         /* Timeout 200us */
1670         if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1671                 drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL lock\n");
1672
1673         dev_priv->display.cdclk.hw.vco = vco;
1674 }
1675
1676 static void adlp_cdclk_pll_crawl(struct drm_i915_private *dev_priv, int vco)
1677 {
1678         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1679         u32 val;
1680
1681         /* Write PLL ratio without disabling */
1682         val = ICL_CDCLK_PLL_RATIO(ratio) | BXT_DE_PLL_PLL_ENABLE;
1683         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1684
1685         /* Submit freq change request */
1686         val |= BXT_DE_PLL_FREQ_REQ;
1687         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1688
1689         /* Timeout 200us */
1690         if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE,
1691                                   BXT_DE_PLL_LOCK | BXT_DE_PLL_FREQ_REQ_ACK, 1))
1692                 drm_err(&dev_priv->drm, "timeout waiting for FREQ change request ack\n");
1693
1694         val &= ~BXT_DE_PLL_FREQ_REQ;
1695         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1696
1697         dev_priv->display.cdclk.hw.vco = vco;
1698 }
1699
1700 static u32 bxt_cdclk_cd2x_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
1701 {
1702         if (DISPLAY_VER(dev_priv) >= 12) {
1703                 if (pipe == INVALID_PIPE)
1704                         return TGL_CDCLK_CD2X_PIPE_NONE;
1705                 else
1706                         return TGL_CDCLK_CD2X_PIPE(pipe);
1707         } else if (DISPLAY_VER(dev_priv) >= 11) {
1708                 if (pipe == INVALID_PIPE)
1709                         return ICL_CDCLK_CD2X_PIPE_NONE;
1710                 else
1711                         return ICL_CDCLK_CD2X_PIPE(pipe);
1712         } else {
1713                 if (pipe == INVALID_PIPE)
1714                         return BXT_CDCLK_CD2X_PIPE_NONE;
1715                 else
1716                         return BXT_CDCLK_CD2X_PIPE(pipe);
1717         }
1718 }
1719
1720 static u32 bxt_cdclk_cd2x_div_sel(struct drm_i915_private *dev_priv,
1721                                   int cdclk, int vco)
1722 {
1723         /* cdclk = vco / 2 / div{1,1.5,2,4} */
1724         switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
1725         default:
1726                 drm_WARN_ON(&dev_priv->drm,
1727                             cdclk != dev_priv->display.cdclk.hw.bypass);
1728                 drm_WARN_ON(&dev_priv->drm, vco != 0);
1729                 fallthrough;
1730         case 2:
1731                 return BXT_CDCLK_CD2X_DIV_SEL_1;
1732         case 3:
1733                 return BXT_CDCLK_CD2X_DIV_SEL_1_5;
1734         case 4:
1735                 return BXT_CDCLK_CD2X_DIV_SEL_2;
1736         case 8:
1737                 return BXT_CDCLK_CD2X_DIV_SEL_4;
1738         }
1739 }
1740
1741 static u32 cdclk_squash_waveform(struct drm_i915_private *dev_priv,
1742                                  int cdclk)
1743 {
1744         const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1745         int i;
1746
1747         if (cdclk == dev_priv->display.cdclk.hw.bypass)
1748                 return 0;
1749
1750         for (i = 0; table[i].refclk; i++)
1751                 if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1752                     table[i].cdclk == cdclk)
1753                         return table[i].waveform;
1754
1755         drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1756                  cdclk, dev_priv->display.cdclk.hw.ref);
1757
1758         return 0xffff;
1759 }
1760
1761 static void icl_cdclk_pll_update(struct drm_i915_private *i915, int vco)
1762 {
1763         if (i915->display.cdclk.hw.vco != 0 &&
1764             i915->display.cdclk.hw.vco != vco)
1765                 icl_cdclk_pll_disable(i915);
1766
1767         if (i915->display.cdclk.hw.vco != vco)
1768                 icl_cdclk_pll_enable(i915, vco);
1769 }
1770
1771 static void bxt_cdclk_pll_update(struct drm_i915_private *i915, int vco)
1772 {
1773         if (i915->display.cdclk.hw.vco != 0 &&
1774             i915->display.cdclk.hw.vco != vco)
1775                 bxt_de_pll_disable(i915);
1776
1777         if (i915->display.cdclk.hw.vco != vco)
1778                 bxt_de_pll_enable(i915, vco);
1779 }
1780
1781 static void dg2_cdclk_squash_program(struct drm_i915_private *i915,
1782                                      u16 waveform)
1783 {
1784         u32 squash_ctl = 0;
1785
1786         if (waveform)
1787                 squash_ctl = CDCLK_SQUASH_ENABLE |
1788                              CDCLK_SQUASH_WINDOW_SIZE(0xf) | waveform;
1789
1790         intel_de_write(i915, CDCLK_SQUASH_CTL, squash_ctl);
1791 }
1792
1793 static bool cdclk_pll_is_unknown(unsigned int vco)
1794 {
1795         /*
1796          * Ensure driver does not take the crawl path for the
1797          * case when the vco is set to ~0 in the
1798          * sanitize path.
1799          */
1800         return vco == ~0;
1801 }
1802
1803 static int cdclk_squash_divider(u16 waveform)
1804 {
1805         return hweight16(waveform ?: 0xffff);
1806 }
1807
1808 static bool cdclk_compute_crawl_and_squash_midpoint(struct drm_i915_private *i915,
1809                                                     const struct intel_cdclk_config *old_cdclk_config,
1810                                                     const struct intel_cdclk_config *new_cdclk_config,
1811                                                     struct intel_cdclk_config *mid_cdclk_config)
1812 {
1813         u16 old_waveform, new_waveform, mid_waveform;
1814         int size = 16;
1815         int div = 2;
1816
1817         /* Return if PLL is in an unknown state, force a complete disable and re-enable. */
1818         if (cdclk_pll_is_unknown(old_cdclk_config->vco))
1819                 return false;
1820
1821         /* Return if both Squash and Crawl are not present */
1822         if (!HAS_CDCLK_CRAWL(i915) || !HAS_CDCLK_SQUASH(i915))
1823                 return false;
1824
1825         old_waveform = cdclk_squash_waveform(i915, old_cdclk_config->cdclk);
1826         new_waveform = cdclk_squash_waveform(i915, new_cdclk_config->cdclk);
1827
1828         /* Return if Squash only or Crawl only is the desired action */
1829         if (old_cdclk_config->vco == 0 || new_cdclk_config->vco == 0 ||
1830             old_cdclk_config->vco == new_cdclk_config->vco ||
1831             old_waveform == new_waveform)
1832                 return false;
1833
1834         *mid_cdclk_config = *new_cdclk_config;
1835
1836         /*
1837          * Populate the mid_cdclk_config accordingly.
1838          * - If moving to a higher cdclk, the desired action is squashing.
1839          * The mid cdclk config should have the new (squash) waveform.
1840          * - If moving to a lower cdclk, the desired action is crawling.
1841          * The mid cdclk config should have the new vco.
1842          */
1843
1844         if (cdclk_squash_divider(new_waveform) > cdclk_squash_divider(old_waveform)) {
1845                 mid_cdclk_config->vco = old_cdclk_config->vco;
1846                 mid_waveform = new_waveform;
1847         } else {
1848                 mid_cdclk_config->vco = new_cdclk_config->vco;
1849                 mid_waveform = old_waveform;
1850         }
1851
1852         mid_cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_squash_divider(mid_waveform) *
1853                                                     mid_cdclk_config->vco, size * div);
1854
1855         /* make sure the mid clock came out sane */
1856
1857         drm_WARN_ON(&i915->drm, mid_cdclk_config->cdclk <
1858                     min(old_cdclk_config->cdclk, new_cdclk_config->cdclk));
1859         drm_WARN_ON(&i915->drm, mid_cdclk_config->cdclk >
1860                     i915->display.cdclk.max_cdclk_freq);
1861         drm_WARN_ON(&i915->drm, cdclk_squash_waveform(i915, mid_cdclk_config->cdclk) !=
1862                     mid_waveform);
1863
1864         return true;
1865 }
1866
1867 static bool pll_enable_wa_needed(struct drm_i915_private *dev_priv)
1868 {
1869         return (DISPLAY_VER_FULL(dev_priv) == IP_VER(20, 0) ||
1870                 DISPLAY_VER_FULL(dev_priv) == IP_VER(14, 0) ||
1871                 IS_DG2(dev_priv)) &&
1872                 dev_priv->display.cdclk.hw.vco > 0;
1873 }
1874
1875 static void _bxt_set_cdclk(struct drm_i915_private *dev_priv,
1876                            const struct intel_cdclk_config *cdclk_config,
1877                            enum pipe pipe)
1878 {
1879         int cdclk = cdclk_config->cdclk;
1880         int vco = cdclk_config->vco;
1881         u32 val;
1882         u16 waveform;
1883         int clock;
1884
1885         if (HAS_CDCLK_CRAWL(dev_priv) && dev_priv->display.cdclk.hw.vco > 0 && vco > 0 &&
1886             !cdclk_pll_is_unknown(dev_priv->display.cdclk.hw.vco)) {
1887                 if (dev_priv->display.cdclk.hw.vco != vco)
1888                         adlp_cdclk_pll_crawl(dev_priv, vco);
1889         } else if (DISPLAY_VER(dev_priv) >= 11) {
1890                 /* wa_15010685871: dg2, mtl */
1891                 if (pll_enable_wa_needed(dev_priv))
1892                         dg2_cdclk_squash_program(dev_priv, 0);
1893
1894                 icl_cdclk_pll_update(dev_priv, vco);
1895         } else
1896                 bxt_cdclk_pll_update(dev_priv, vco);
1897
1898         waveform = cdclk_squash_waveform(dev_priv, cdclk);
1899
1900         if (waveform)
1901                 clock = vco / 2;
1902         else
1903                 clock = cdclk;
1904
1905         if (HAS_CDCLK_SQUASH(dev_priv))
1906                 dg2_cdclk_squash_program(dev_priv, waveform);
1907
1908         val = bxt_cdclk_cd2x_div_sel(dev_priv, clock, vco) |
1909                 bxt_cdclk_cd2x_pipe(dev_priv, pipe);
1910
1911         /*
1912          * Disable SSA Precharge when CD clock frequency < 500 MHz,
1913          * enable otherwise.
1914          */
1915         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
1916             cdclk >= 500000)
1917                 val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
1918
1919         if (DISPLAY_VER(dev_priv) >= 20)
1920                 val |= MDCLK_SOURCE_SEL_CDCLK_PLL;
1921         else
1922                 val |= skl_cdclk_decimal(cdclk);
1923
1924         intel_de_write(dev_priv, CDCLK_CTL, val);
1925
1926         if (pipe != INVALID_PIPE)
1927                 intel_crtc_wait_for_next_vblank(intel_crtc_for_pipe(dev_priv, pipe));
1928 }
1929
1930 static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
1931                           const struct intel_cdclk_config *cdclk_config,
1932                           enum pipe pipe)
1933 {
1934         struct intel_cdclk_config mid_cdclk_config;
1935         int cdclk = cdclk_config->cdclk;
1936         int ret = 0;
1937
1938         /*
1939          * Inform power controller of upcoming frequency change.
1940          * Display versions 14 and beyond do not follow the PUnit
1941          * mailbox communication, skip
1942          * this step.
1943          */
1944         if (DISPLAY_VER(dev_priv) >= 14 || IS_DG2(dev_priv))
1945                 /* NOOP */;
1946         else if (DISPLAY_VER(dev_priv) >= 11)
1947                 ret = skl_pcode_request(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1948                                         SKL_CDCLK_PREPARE_FOR_CHANGE,
1949                                         SKL_CDCLK_READY_FOR_CHANGE,
1950                                         SKL_CDCLK_READY_FOR_CHANGE, 3);
1951         else
1952                 /*
1953                  * BSpec requires us to wait up to 150usec, but that leads to
1954                  * timeouts; the 2ms used here is based on experiment.
1955                  */
1956                 ret = snb_pcode_write_timeout(&dev_priv->uncore,
1957                                               HSW_PCODE_DE_WRITE_FREQ_REQ,
1958                                               0x80000000, 150, 2);
1959
1960         if (ret) {
1961                 drm_err(&dev_priv->drm,
1962                         "Failed to inform PCU about cdclk change (err %d, freq %d)\n",
1963                         ret, cdclk);
1964                 return;
1965         }
1966
1967         if (cdclk_compute_crawl_and_squash_midpoint(dev_priv, &dev_priv->display.cdclk.hw,
1968                                                     cdclk_config, &mid_cdclk_config)) {
1969                 _bxt_set_cdclk(dev_priv, &mid_cdclk_config, pipe);
1970                 _bxt_set_cdclk(dev_priv, cdclk_config, pipe);
1971         } else {
1972                 _bxt_set_cdclk(dev_priv, cdclk_config, pipe);
1973         }
1974
1975         if (DISPLAY_VER(dev_priv) >= 14)
1976                 /*
1977                  * NOOP - No Pcode communication needed for
1978                  * Display versions 14 and beyond
1979                  */;
1980         else if (DISPLAY_VER(dev_priv) >= 11 && !IS_DG2(dev_priv))
1981                 ret = snb_pcode_write(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1982                                       cdclk_config->voltage_level);
1983         if (DISPLAY_VER(dev_priv) < 11) {
1984                 /*
1985                  * The timeout isn't specified, the 2ms used here is based on
1986                  * experiment.
1987                  * FIXME: Waiting for the request completion could be delayed
1988                  * until the next PCODE request based on BSpec.
1989                  */
1990                 ret = snb_pcode_write_timeout(&dev_priv->uncore,
1991                                               HSW_PCODE_DE_WRITE_FREQ_REQ,
1992                                               cdclk_config->voltage_level,
1993                                               150, 2);
1994         }
1995         if (ret) {
1996                 drm_err(&dev_priv->drm,
1997                         "PCode CDCLK freq set failed, (err %d, freq %d)\n",
1998                         ret, cdclk);
1999                 return;
2000         }
2001
2002         intel_update_cdclk(dev_priv);
2003
2004         if (DISPLAY_VER(dev_priv) >= 11)
2005                 /*
2006                  * Can't read out the voltage level :(
2007                  * Let's just assume everything is as expected.
2008                  */
2009                 dev_priv->display.cdclk.hw.voltage_level = cdclk_config->voltage_level;
2010 }
2011
2012 static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
2013 {
2014         u32 cdctl, expected;
2015         int cdclk, clock, vco;
2016
2017         intel_update_cdclk(dev_priv);
2018         intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
2019
2020         if (dev_priv->display.cdclk.hw.vco == 0 ||
2021             dev_priv->display.cdclk.hw.cdclk == dev_priv->display.cdclk.hw.bypass)
2022                 goto sanitize;
2023
2024         /* DPLL okay; verify the cdclock
2025          *
2026          * Some BIOS versions leave an incorrect decimal frequency value and
2027          * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4,
2028          * so sanitize this register.
2029          */
2030         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
2031         /*
2032          * Let's ignore the pipe field, since BIOS could have configured the
2033          * dividers both synching to an active pipe, or asynchronously
2034          * (PIPE_NONE).
2035          */
2036         cdctl &= ~bxt_cdclk_cd2x_pipe(dev_priv, INVALID_PIPE);
2037
2038         /* Make sure this is a legal cdclk value for the platform */
2039         cdclk = bxt_calc_cdclk(dev_priv, dev_priv->display.cdclk.hw.cdclk);
2040         if (cdclk != dev_priv->display.cdclk.hw.cdclk)
2041                 goto sanitize;
2042
2043         /* Make sure the VCO is correct for the cdclk */
2044         vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2045         if (vco != dev_priv->display.cdclk.hw.vco)
2046                 goto sanitize;
2047
2048         expected = skl_cdclk_decimal(cdclk);
2049
2050         /* Figure out what CD2X divider we should be using for this cdclk */
2051         if (HAS_CDCLK_SQUASH(dev_priv))
2052                 clock = dev_priv->display.cdclk.hw.vco / 2;
2053         else
2054                 clock = dev_priv->display.cdclk.hw.cdclk;
2055
2056         expected |= bxt_cdclk_cd2x_div_sel(dev_priv, clock,
2057                                            dev_priv->display.cdclk.hw.vco);
2058
2059         /*
2060          * Disable SSA Precharge when CD clock frequency < 500 MHz,
2061          * enable otherwise.
2062          */
2063         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
2064             dev_priv->display.cdclk.hw.cdclk >= 500000)
2065                 expected |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
2066
2067         if (cdctl == expected)
2068                 /* All well; nothing to sanitize */
2069                 return;
2070
2071 sanitize:
2072         drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
2073
2074         /* force cdclk programming */
2075         dev_priv->display.cdclk.hw.cdclk = 0;
2076
2077         /* force full PLL disable + enable */
2078         dev_priv->display.cdclk.hw.vco = -1;
2079 }
2080
2081 static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv)
2082 {
2083         struct intel_cdclk_config cdclk_config;
2084
2085         bxt_sanitize_cdclk(dev_priv);
2086
2087         if (dev_priv->display.cdclk.hw.cdclk != 0 &&
2088             dev_priv->display.cdclk.hw.vco != 0)
2089                 return;
2090
2091         cdclk_config = dev_priv->display.cdclk.hw;
2092
2093         /*
2094          * FIXME:
2095          * - The initial CDCLK needs to be read from VBT.
2096          *   Need to make this change after VBT has changes for BXT.
2097          */
2098         cdclk_config.cdclk = bxt_calc_cdclk(dev_priv, 0);
2099         cdclk_config.vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk_config.cdclk);
2100         cdclk_config.voltage_level =
2101                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk);
2102
2103         bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
2104 }
2105
2106 static void bxt_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
2107 {
2108         struct intel_cdclk_config cdclk_config = dev_priv->display.cdclk.hw;
2109
2110         cdclk_config.cdclk = cdclk_config.bypass;
2111         cdclk_config.vco = 0;
2112         cdclk_config.voltage_level =
2113                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk);
2114
2115         bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
2116 }
2117
2118 /**
2119  * intel_cdclk_init_hw - Initialize CDCLK hardware
2120  * @i915: i915 device
2121  *
2122  * Initialize CDCLK. This consists mainly of initializing dev_priv->display.cdclk.hw and
2123  * sanitizing the state of the hardware if needed. This is generally done only
2124  * during the display core initialization sequence, after which the DMC will
2125  * take care of turning CDCLK off/on as needed.
2126  */
2127 void intel_cdclk_init_hw(struct drm_i915_private *i915)
2128 {
2129         if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
2130                 bxt_cdclk_init_hw(i915);
2131         else if (DISPLAY_VER(i915) == 9)
2132                 skl_cdclk_init_hw(i915);
2133 }
2134
2135 /**
2136  * intel_cdclk_uninit_hw - Uninitialize CDCLK hardware
2137  * @i915: i915 device
2138  *
2139  * Uninitialize CDCLK. This is done only during the display core
2140  * uninitialization sequence.
2141  */
2142 void intel_cdclk_uninit_hw(struct drm_i915_private *i915)
2143 {
2144         if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
2145                 bxt_cdclk_uninit_hw(i915);
2146         else if (DISPLAY_VER(i915) == 9)
2147                 skl_cdclk_uninit_hw(i915);
2148 }
2149
2150 static bool intel_cdclk_can_crawl_and_squash(struct drm_i915_private *i915,
2151                                              const struct intel_cdclk_config *a,
2152                                              const struct intel_cdclk_config *b)
2153 {
2154         u16 old_waveform;
2155         u16 new_waveform;
2156
2157         drm_WARN_ON(&i915->drm, cdclk_pll_is_unknown(a->vco));
2158
2159         if (a->vco == 0 || b->vco == 0)
2160                 return false;
2161
2162         if (!HAS_CDCLK_CRAWL(i915) || !HAS_CDCLK_SQUASH(i915))
2163                 return false;
2164
2165         old_waveform = cdclk_squash_waveform(i915, a->cdclk);
2166         new_waveform = cdclk_squash_waveform(i915, b->cdclk);
2167
2168         return a->vco != b->vco &&
2169                old_waveform != new_waveform;
2170 }
2171
2172 static bool intel_cdclk_can_crawl(struct drm_i915_private *dev_priv,
2173                                   const struct intel_cdclk_config *a,
2174                                   const struct intel_cdclk_config *b)
2175 {
2176         int a_div, b_div;
2177
2178         if (!HAS_CDCLK_CRAWL(dev_priv))
2179                 return false;
2180
2181         /*
2182          * The vco and cd2x divider will change independently
2183          * from each, so we disallow cd2x change when crawling.
2184          */
2185         a_div = DIV_ROUND_CLOSEST(a->vco, a->cdclk);
2186         b_div = DIV_ROUND_CLOSEST(b->vco, b->cdclk);
2187
2188         return a->vco != 0 && b->vco != 0 &&
2189                 a->vco != b->vco &&
2190                 a_div == b_div &&
2191                 a->ref == b->ref;
2192 }
2193
2194 static bool intel_cdclk_can_squash(struct drm_i915_private *dev_priv,
2195                                    const struct intel_cdclk_config *a,
2196                                    const struct intel_cdclk_config *b)
2197 {
2198         /*
2199          * FIXME should store a bit more state in intel_cdclk_config
2200          * to differentiate squasher vs. cd2x divider properly. For
2201          * the moment all platforms with squasher use a fixed cd2x
2202          * divider.
2203          */
2204         if (!HAS_CDCLK_SQUASH(dev_priv))
2205                 return false;
2206
2207         return a->cdclk != b->cdclk &&
2208                 a->vco != 0 &&
2209                 a->vco == b->vco &&
2210                 a->ref == b->ref;
2211 }
2212
2213 /**
2214  * intel_cdclk_needs_modeset - Determine if changong between the CDCLK
2215  *                             configurations requires a modeset on all pipes
2216  * @a: first CDCLK configuration
2217  * @b: second CDCLK configuration
2218  *
2219  * Returns:
2220  * True if changing between the two CDCLK configurations
2221  * requires all pipes to be off, false if not.
2222  */
2223 bool intel_cdclk_needs_modeset(const struct intel_cdclk_config *a,
2224                                const struct intel_cdclk_config *b)
2225 {
2226         return a->cdclk != b->cdclk ||
2227                 a->vco != b->vco ||
2228                 a->ref != b->ref;
2229 }
2230
2231 /**
2232  * intel_cdclk_can_cd2x_update - Determine if changing between the two CDCLK
2233  *                               configurations requires only a cd2x divider update
2234  * @dev_priv: i915 device
2235  * @a: first CDCLK configuration
2236  * @b: second CDCLK configuration
2237  *
2238  * Returns:
2239  * True if changing between the two CDCLK configurations
2240  * can be done with just a cd2x divider update, false if not.
2241  */
2242 static bool intel_cdclk_can_cd2x_update(struct drm_i915_private *dev_priv,
2243                                         const struct intel_cdclk_config *a,
2244                                         const struct intel_cdclk_config *b)
2245 {
2246         /* Older hw doesn't have the capability */
2247         if (DISPLAY_VER(dev_priv) < 10 && !IS_BROXTON(dev_priv))
2248                 return false;
2249
2250         /*
2251          * FIXME should store a bit more state in intel_cdclk_config
2252          * to differentiate squasher vs. cd2x divider properly. For
2253          * the moment all platforms with squasher use a fixed cd2x
2254          * divider.
2255          */
2256         if (HAS_CDCLK_SQUASH(dev_priv))
2257                 return false;
2258
2259         return a->cdclk != b->cdclk &&
2260                 a->vco != 0 &&
2261                 a->vco == b->vco &&
2262                 a->ref == b->ref;
2263 }
2264
2265 /**
2266  * intel_cdclk_changed - Determine if two CDCLK configurations are different
2267  * @a: first CDCLK configuration
2268  * @b: second CDCLK configuration
2269  *
2270  * Returns:
2271  * True if the CDCLK configurations don't match, false if they do.
2272  */
2273 static bool intel_cdclk_changed(const struct intel_cdclk_config *a,
2274                                 const struct intel_cdclk_config *b)
2275 {
2276         return intel_cdclk_needs_modeset(a, b) ||
2277                 a->voltage_level != b->voltage_level;
2278 }
2279
2280 void intel_cdclk_dump_config(struct drm_i915_private *i915,
2281                              const struct intel_cdclk_config *cdclk_config,
2282                              const char *context)
2283 {
2284         drm_dbg_kms(&i915->drm, "%s %d kHz, VCO %d kHz, ref %d kHz, bypass %d kHz, voltage level %d\n",
2285                     context, cdclk_config->cdclk, cdclk_config->vco,
2286                     cdclk_config->ref, cdclk_config->bypass,
2287                     cdclk_config->voltage_level);
2288 }
2289
2290 static void intel_pcode_notify(struct drm_i915_private *i915,
2291                                u8 voltage_level,
2292                                u8 active_pipe_count,
2293                                u16 cdclk,
2294                                bool cdclk_update_valid,
2295                                bool pipe_count_update_valid)
2296 {
2297         int ret;
2298         u32 update_mask = 0;
2299
2300         if (!IS_DG2(i915))
2301                 return;
2302
2303         update_mask = DISPLAY_TO_PCODE_UPDATE_MASK(cdclk, active_pipe_count, voltage_level);
2304
2305         if (cdclk_update_valid)
2306                 update_mask |= DISPLAY_TO_PCODE_CDCLK_VALID;
2307
2308         if (pipe_count_update_valid)
2309                 update_mask |= DISPLAY_TO_PCODE_PIPE_COUNT_VALID;
2310
2311         ret = skl_pcode_request(&i915->uncore, SKL_PCODE_CDCLK_CONTROL,
2312                                 SKL_CDCLK_PREPARE_FOR_CHANGE |
2313                                 update_mask,
2314                                 SKL_CDCLK_READY_FOR_CHANGE,
2315                                 SKL_CDCLK_READY_FOR_CHANGE, 3);
2316         if (ret)
2317                 drm_err(&i915->drm,
2318                         "Failed to inform PCU about display config (err %d)\n",
2319                         ret);
2320 }
2321
2322 /**
2323  * intel_set_cdclk - Push the CDCLK configuration to the hardware
2324  * @dev_priv: i915 device
2325  * @cdclk_config: new CDCLK configuration
2326  * @pipe: pipe with which to synchronize the update
2327  *
2328  * Program the hardware based on the passed in CDCLK state,
2329  * if necessary.
2330  */
2331 static void intel_set_cdclk(struct drm_i915_private *dev_priv,
2332                             const struct intel_cdclk_config *cdclk_config,
2333                             enum pipe pipe)
2334 {
2335         struct intel_encoder *encoder;
2336
2337         if (!intel_cdclk_changed(&dev_priv->display.cdclk.hw, cdclk_config))
2338                 return;
2339
2340         if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->display.funcs.cdclk->set_cdclk))
2341                 return;
2342
2343         intel_cdclk_dump_config(dev_priv, cdclk_config, "Changing CDCLK to");
2344
2345         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2346                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2347
2348                 intel_psr_pause(intel_dp);
2349         }
2350
2351         intel_audio_cdclk_change_pre(dev_priv);
2352
2353         /*
2354          * Lock aux/gmbus while we change cdclk in case those
2355          * functions use cdclk. Not all platforms/ports do,
2356          * but we'll lock them all for simplicity.
2357          */
2358         mutex_lock(&dev_priv->display.gmbus.mutex);
2359         for_each_intel_dp(&dev_priv->drm, encoder) {
2360                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2361
2362                 mutex_lock_nest_lock(&intel_dp->aux.hw_mutex,
2363                                      &dev_priv->display.gmbus.mutex);
2364         }
2365
2366         intel_cdclk_set_cdclk(dev_priv, cdclk_config, pipe);
2367
2368         for_each_intel_dp(&dev_priv->drm, encoder) {
2369                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2370
2371                 mutex_unlock(&intel_dp->aux.hw_mutex);
2372         }
2373         mutex_unlock(&dev_priv->display.gmbus.mutex);
2374
2375         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2376                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2377
2378                 intel_psr_resume(intel_dp);
2379         }
2380
2381         intel_audio_cdclk_change_post(dev_priv);
2382
2383         if (drm_WARN(&dev_priv->drm,
2384                      intel_cdclk_changed(&dev_priv->display.cdclk.hw, cdclk_config),
2385                      "cdclk state doesn't match!\n")) {
2386                 intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "[hw state]");
2387                 intel_cdclk_dump_config(dev_priv, cdclk_config, "[sw state]");
2388         }
2389 }
2390
2391 static void intel_cdclk_pcode_pre_notify(struct intel_atomic_state *state)
2392 {
2393         struct drm_i915_private *i915 = to_i915(state->base.dev);
2394         const struct intel_cdclk_state *old_cdclk_state =
2395                 intel_atomic_get_old_cdclk_state(state);
2396         const struct intel_cdclk_state *new_cdclk_state =
2397                 intel_atomic_get_new_cdclk_state(state);
2398         unsigned int cdclk = 0; u8 voltage_level, num_active_pipes = 0;
2399         bool change_cdclk, update_pipe_count;
2400
2401         if (!intel_cdclk_changed(&old_cdclk_state->actual,
2402                                  &new_cdclk_state->actual) &&
2403                                  new_cdclk_state->active_pipes ==
2404                                  old_cdclk_state->active_pipes)
2405                 return;
2406
2407         /* According to "Sequence Before Frequency Change", voltage level set to 0x3 */
2408         voltage_level = DISPLAY_TO_PCODE_VOLTAGE_MAX;
2409
2410         change_cdclk = new_cdclk_state->actual.cdclk != old_cdclk_state->actual.cdclk;
2411         update_pipe_count = hweight8(new_cdclk_state->active_pipes) >
2412                             hweight8(old_cdclk_state->active_pipes);
2413
2414         /*
2415          * According to "Sequence Before Frequency Change",
2416          * if CDCLK is increasing, set bits 25:16 to upcoming CDCLK,
2417          * if CDCLK is decreasing or not changing, set bits 25:16 to current CDCLK,
2418          * which basically means we choose the maximum of old and new CDCLK, if we know both
2419          */
2420         if (change_cdclk)
2421                 cdclk = max(new_cdclk_state->actual.cdclk, old_cdclk_state->actual.cdclk);
2422
2423         /*
2424          * According to "Sequence For Pipe Count Change",
2425          * if pipe count is increasing, set bits 25:16 to upcoming pipe count
2426          * (power well is enabled)
2427          * no action if it is decreasing, before the change
2428          */
2429         if (update_pipe_count)
2430                 num_active_pipes = hweight8(new_cdclk_state->active_pipes);
2431
2432         intel_pcode_notify(i915, voltage_level, num_active_pipes, cdclk,
2433                            change_cdclk, update_pipe_count);
2434 }
2435
2436 static void intel_cdclk_pcode_post_notify(struct intel_atomic_state *state)
2437 {
2438         struct drm_i915_private *i915 = to_i915(state->base.dev);
2439         const struct intel_cdclk_state *new_cdclk_state =
2440                 intel_atomic_get_new_cdclk_state(state);
2441         const struct intel_cdclk_state *old_cdclk_state =
2442                 intel_atomic_get_old_cdclk_state(state);
2443         unsigned int cdclk = 0; u8 voltage_level, num_active_pipes = 0;
2444         bool update_cdclk, update_pipe_count;
2445
2446         /* According to "Sequence After Frequency Change", set voltage to used level */
2447         voltage_level = new_cdclk_state->actual.voltage_level;
2448
2449         update_cdclk = new_cdclk_state->actual.cdclk != old_cdclk_state->actual.cdclk;
2450         update_pipe_count = hweight8(new_cdclk_state->active_pipes) <
2451                             hweight8(old_cdclk_state->active_pipes);
2452
2453         /*
2454          * According to "Sequence After Frequency Change",
2455          * set bits 25:16 to current CDCLK
2456          */
2457         if (update_cdclk)
2458                 cdclk = new_cdclk_state->actual.cdclk;
2459
2460         /*
2461          * According to "Sequence For Pipe Count Change",
2462          * if pipe count is decreasing, set bits 25:16 to current pipe count,
2463          * after the change(power well is disabled)
2464          * no action if it is increasing, after the change
2465          */
2466         if (update_pipe_count)
2467                 num_active_pipes = hweight8(new_cdclk_state->active_pipes);
2468
2469         intel_pcode_notify(i915, voltage_level, num_active_pipes, cdclk,
2470                            update_cdclk, update_pipe_count);
2471 }
2472
2473 /**
2474  * intel_set_cdclk_pre_plane_update - Push the CDCLK state to the hardware
2475  * @state: intel atomic state
2476  *
2477  * Program the hardware before updating the HW plane state based on the
2478  * new CDCLK state, if necessary.
2479  */
2480 void
2481 intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state)
2482 {
2483         struct drm_i915_private *i915 = to_i915(state->base.dev);
2484         const struct intel_cdclk_state *old_cdclk_state =
2485                 intel_atomic_get_old_cdclk_state(state);
2486         const struct intel_cdclk_state *new_cdclk_state =
2487                 intel_atomic_get_new_cdclk_state(state);
2488         enum pipe pipe = new_cdclk_state->pipe;
2489
2490         if (!intel_cdclk_changed(&old_cdclk_state->actual,
2491                                  &new_cdclk_state->actual))
2492                 return;
2493
2494         if (IS_DG2(i915))
2495                 intel_cdclk_pcode_pre_notify(state);
2496
2497         if (pipe == INVALID_PIPE ||
2498             old_cdclk_state->actual.cdclk <= new_cdclk_state->actual.cdclk) {
2499                 drm_WARN_ON(&i915->drm, !new_cdclk_state->base.changed);
2500
2501                 intel_set_cdclk(i915, &new_cdclk_state->actual, pipe);
2502         }
2503 }
2504
2505 /**
2506  * intel_set_cdclk_post_plane_update - Push the CDCLK state to the hardware
2507  * @state: intel atomic state
2508  *
2509  * Program the hardware after updating the HW plane state based on the
2510  * new CDCLK state, if necessary.
2511  */
2512 void
2513 intel_set_cdclk_post_plane_update(struct intel_atomic_state *state)
2514 {
2515         struct drm_i915_private *i915 = to_i915(state->base.dev);
2516         const struct intel_cdclk_state *old_cdclk_state =
2517                 intel_atomic_get_old_cdclk_state(state);
2518         const struct intel_cdclk_state *new_cdclk_state =
2519                 intel_atomic_get_new_cdclk_state(state);
2520         enum pipe pipe = new_cdclk_state->pipe;
2521
2522         if (!intel_cdclk_changed(&old_cdclk_state->actual,
2523                                  &new_cdclk_state->actual))
2524                 return;
2525
2526         if (IS_DG2(i915))
2527                 intel_cdclk_pcode_post_notify(state);
2528
2529         if (pipe != INVALID_PIPE &&
2530             old_cdclk_state->actual.cdclk > new_cdclk_state->actual.cdclk) {
2531                 drm_WARN_ON(&i915->drm, !new_cdclk_state->base.changed);
2532
2533                 intel_set_cdclk(i915, &new_cdclk_state->actual, pipe);
2534         }
2535 }
2536
2537 static int intel_pixel_rate_to_cdclk(const struct intel_crtc_state *crtc_state)
2538 {
2539         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2540         int pixel_rate = crtc_state->pixel_rate;
2541
2542         if (DISPLAY_VER(dev_priv) >= 10)
2543                 return DIV_ROUND_UP(pixel_rate, 2);
2544         else if (DISPLAY_VER(dev_priv) == 9 ||
2545                  IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2546                 return pixel_rate;
2547         else if (IS_CHERRYVIEW(dev_priv))
2548                 return DIV_ROUND_UP(pixel_rate * 100, 95);
2549         else if (crtc_state->double_wide)
2550                 return DIV_ROUND_UP(pixel_rate * 100, 90 * 2);
2551         else
2552                 return DIV_ROUND_UP(pixel_rate * 100, 90);
2553 }
2554
2555 static int intel_planes_min_cdclk(const struct intel_crtc_state *crtc_state)
2556 {
2557         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2558         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2559         struct intel_plane *plane;
2560         int min_cdclk = 0;
2561
2562         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
2563                 min_cdclk = max(crtc_state->min_cdclk[plane->id], min_cdclk);
2564
2565         return min_cdclk;
2566 }
2567
2568 static int intel_vdsc_min_cdclk(const struct intel_crtc_state *crtc_state)
2569 {
2570         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2571         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2572         int num_vdsc_instances = intel_dsc_get_num_vdsc_instances(crtc_state);
2573         int min_cdclk = 0;
2574
2575         /*
2576          * When we decide to use only one VDSC engine, since
2577          * each VDSC operates with 1 ppc throughput, pixel clock
2578          * cannot be higher than the VDSC clock (cdclk)
2579          * If there 2 VDSC engines, then pixel clock can't be higher than
2580          * VDSC clock(cdclk) * 2 and so on.
2581          */
2582         min_cdclk = max_t(int, min_cdclk,
2583                           DIV_ROUND_UP(crtc_state->pixel_rate, num_vdsc_instances));
2584
2585         if (crtc_state->bigjoiner_pipes) {
2586                 int pixel_clock = intel_dp_mode_to_fec_clock(crtc_state->hw.adjusted_mode.clock);
2587
2588                 /*
2589                  * According to Bigjoiner bw check:
2590                  * compressed_bpp <= PPC * CDCLK * Big joiner Interface bits / Pixel clock
2591                  *
2592                  * We have already computed compressed_bpp, so now compute the min CDCLK that
2593                  * is required to support this compressed_bpp.
2594                  *
2595                  * => CDCLK >= compressed_bpp * Pixel clock / (PPC * Bigjoiner Interface bits)
2596                  *
2597                  * Since PPC = 2 with bigjoiner
2598                  * => CDCLK >= compressed_bpp * Pixel clock  / 2 * Bigjoiner Interface bits
2599                  */
2600                 int bigjoiner_interface_bits = DISPLAY_VER(i915) > 13 ? 36 : 24;
2601                 int min_cdclk_bj = (crtc_state->dsc.compressed_bpp * pixel_clock) /
2602                                    (2 * bigjoiner_interface_bits);
2603
2604                 min_cdclk = max(min_cdclk, min_cdclk_bj);
2605         }
2606
2607         return min_cdclk;
2608 }
2609
2610 int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
2611 {
2612         struct drm_i915_private *dev_priv =
2613                 to_i915(crtc_state->uapi.crtc->dev);
2614         int min_cdclk;
2615
2616         if (!crtc_state->hw.enable)
2617                 return 0;
2618
2619         min_cdclk = intel_pixel_rate_to_cdclk(crtc_state);
2620
2621         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
2622         if (IS_BROADWELL(dev_priv) && hsw_crtc_state_ips_capable(crtc_state))
2623                 min_cdclk = DIV_ROUND_UP(min_cdclk * 100, 95);
2624
2625         /* BSpec says "Do not use DisplayPort with CDCLK less than 432 MHz,
2626          * audio enabled, port width x4, and link rate HBR2 (5.4 GHz), or else
2627          * there may be audio corruption or screen corruption." This cdclk
2628          * restriction for GLK is 316.8 MHz.
2629          */
2630         if (intel_crtc_has_dp_encoder(crtc_state) &&
2631             crtc_state->has_audio &&
2632             crtc_state->port_clock >= 540000 &&
2633             crtc_state->lane_count == 4) {
2634                 if (DISPLAY_VER(dev_priv) == 10) {
2635                         /* Display WA #1145: glk */
2636                         min_cdclk = max(316800, min_cdclk);
2637                 } else if (DISPLAY_VER(dev_priv) == 9 || IS_BROADWELL(dev_priv)) {
2638                         /* Display WA #1144: skl,bxt */
2639                         min_cdclk = max(432000, min_cdclk);
2640                 }
2641         }
2642
2643         /*
2644          * According to BSpec, "The CD clock frequency must be at least twice
2645          * the frequency of the Azalia BCLK." and BCLK is 96 MHz by default.
2646          */
2647         if (crtc_state->has_audio && DISPLAY_VER(dev_priv) >= 9)
2648                 min_cdclk = max(2 * 96000, min_cdclk);
2649
2650         /*
2651          * "For DP audio configuration, cdclk frequency shall be set to
2652          *  meet the following requirements:
2653          *  DP Link Frequency(MHz) | Cdclk frequency(MHz)
2654          *  270                    | 320 or higher
2655          *  162                    | 200 or higher"
2656          */
2657         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
2658             intel_crtc_has_dp_encoder(crtc_state) && crtc_state->has_audio)
2659                 min_cdclk = max(crtc_state->port_clock, min_cdclk);
2660
2661         /*
2662          * On Valleyview some DSI panels lose (v|h)sync when the clock is lower
2663          * than 320000KHz.
2664          */
2665         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2666             IS_VALLEYVIEW(dev_priv))
2667                 min_cdclk = max(320000, min_cdclk);
2668
2669         /*
2670          * On Geminilake once the CDCLK gets as low as 79200
2671          * picture gets unstable, despite that values are
2672          * correct for DSI PLL and DE PLL.
2673          */
2674         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2675             IS_GEMINILAKE(dev_priv))
2676                 min_cdclk = max(158400, min_cdclk);
2677
2678         /* Account for additional needs from the planes */
2679         min_cdclk = max(intel_planes_min_cdclk(crtc_state), min_cdclk);
2680
2681         if (crtc_state->dsc.compression_enable)
2682                 min_cdclk = max(min_cdclk, intel_vdsc_min_cdclk(crtc_state));
2683
2684         /*
2685          * HACK. Currently for TGL/DG2 platforms we calculate
2686          * min_cdclk initially based on pixel_rate divided
2687          * by 2, accounting for also plane requirements,
2688          * however in some cases the lowest possible CDCLK
2689          * doesn't work and causing the underruns.
2690          * Explicitly stating here that this seems to be currently
2691          * rather a Hack, than final solution.
2692          */
2693         if (IS_TIGERLAKE(dev_priv) || IS_DG2(dev_priv)) {
2694                 /*
2695                  * Clamp to max_cdclk_freq in case pixel rate is higher,
2696                  * in order not to break an 8K, but still leave W/A at place.
2697                  */
2698                 min_cdclk = max_t(int, min_cdclk,
2699                                   min_t(int, crtc_state->pixel_rate,
2700                                         dev_priv->display.cdclk.max_cdclk_freq));
2701         }
2702
2703         return min_cdclk;
2704 }
2705
2706 static int intel_compute_min_cdclk(struct intel_cdclk_state *cdclk_state)
2707 {
2708         struct intel_atomic_state *state = cdclk_state->base.state;
2709         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2710         const struct intel_bw_state *bw_state;
2711         struct intel_crtc *crtc;
2712         struct intel_crtc_state *crtc_state;
2713         int min_cdclk, i;
2714         enum pipe pipe;
2715
2716         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2717                 int ret;
2718
2719                 min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
2720                 if (min_cdclk < 0)
2721                         return min_cdclk;
2722
2723                 if (cdclk_state->min_cdclk[crtc->pipe] == min_cdclk)
2724                         continue;
2725
2726                 cdclk_state->min_cdclk[crtc->pipe] = min_cdclk;
2727
2728                 ret = intel_atomic_lock_global_state(&cdclk_state->base);
2729                 if (ret)
2730                         return ret;
2731         }
2732
2733         bw_state = intel_atomic_get_new_bw_state(state);
2734         if (bw_state) {
2735                 min_cdclk = intel_bw_min_cdclk(dev_priv, bw_state);
2736
2737                 if (cdclk_state->bw_min_cdclk != min_cdclk) {
2738                         int ret;
2739
2740                         cdclk_state->bw_min_cdclk = min_cdclk;
2741
2742                         ret = intel_atomic_lock_global_state(&cdclk_state->base);
2743                         if (ret)
2744                                 return ret;
2745                 }
2746         }
2747
2748         min_cdclk = max(cdclk_state->force_min_cdclk,
2749                         cdclk_state->bw_min_cdclk);
2750         for_each_pipe(dev_priv, pipe)
2751                 min_cdclk = max(cdclk_state->min_cdclk[pipe], min_cdclk);
2752
2753         /*
2754          * Avoid glk_force_audio_cdclk() causing excessive screen
2755          * blinking when multiple pipes are active by making sure
2756          * CDCLK frequency is always high enough for audio. With a
2757          * single active pipe we can always change CDCLK frequency
2758          * by changing the cd2x divider (see glk_cdclk_table[]) and
2759          * thus a full modeset won't be needed then.
2760          */
2761         if (IS_GEMINILAKE(dev_priv) && cdclk_state->active_pipes &&
2762             !is_power_of_2(cdclk_state->active_pipes))
2763                 min_cdclk = max(2 * 96000, min_cdclk);
2764
2765         if (min_cdclk > dev_priv->display.cdclk.max_cdclk_freq) {
2766                 drm_dbg_kms(&dev_priv->drm,
2767                             "required cdclk (%d kHz) exceeds max (%d kHz)\n",
2768                             min_cdclk, dev_priv->display.cdclk.max_cdclk_freq);
2769                 return -EINVAL;
2770         }
2771
2772         return min_cdclk;
2773 }
2774
2775 /*
2776  * Account for port clock min voltage level requirements.
2777  * This only really does something on DISPLA_VER >= 11 but can be
2778  * called on earlier platforms as well.
2779  *
2780  * Note that this functions assumes that 0 is
2781  * the lowest voltage value, and higher values
2782  * correspond to increasingly higher voltages.
2783  *
2784  * Should that relationship no longer hold on
2785  * future platforms this code will need to be
2786  * adjusted.
2787  */
2788 static int bxt_compute_min_voltage_level(struct intel_cdclk_state *cdclk_state)
2789 {
2790         struct intel_atomic_state *state = cdclk_state->base.state;
2791         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2792         struct intel_crtc *crtc;
2793         struct intel_crtc_state *crtc_state;
2794         u8 min_voltage_level;
2795         int i;
2796         enum pipe pipe;
2797
2798         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2799                 int ret;
2800
2801                 if (crtc_state->hw.enable)
2802                         min_voltage_level = crtc_state->min_voltage_level;
2803                 else
2804                         min_voltage_level = 0;
2805
2806                 if (cdclk_state->min_voltage_level[crtc->pipe] == min_voltage_level)
2807                         continue;
2808
2809                 cdclk_state->min_voltage_level[crtc->pipe] = min_voltage_level;
2810
2811                 ret = intel_atomic_lock_global_state(&cdclk_state->base);
2812                 if (ret)
2813                         return ret;
2814         }
2815
2816         min_voltage_level = 0;
2817         for_each_pipe(dev_priv, pipe)
2818                 min_voltage_level = max(cdclk_state->min_voltage_level[pipe],
2819                                         min_voltage_level);
2820
2821         return min_voltage_level;
2822 }
2823
2824 static int vlv_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2825 {
2826         struct intel_atomic_state *state = cdclk_state->base.state;
2827         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2828         int min_cdclk, cdclk;
2829
2830         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2831         if (min_cdclk < 0)
2832                 return min_cdclk;
2833
2834         cdclk = vlv_calc_cdclk(dev_priv, min_cdclk);
2835
2836         cdclk_state->logical.cdclk = cdclk;
2837         cdclk_state->logical.voltage_level =
2838                 vlv_calc_voltage_level(dev_priv, cdclk);
2839
2840         if (!cdclk_state->active_pipes) {
2841                 cdclk = vlv_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2842
2843                 cdclk_state->actual.cdclk = cdclk;
2844                 cdclk_state->actual.voltage_level =
2845                         vlv_calc_voltage_level(dev_priv, cdclk);
2846         } else {
2847                 cdclk_state->actual = cdclk_state->logical;
2848         }
2849
2850         return 0;
2851 }
2852
2853 static int bdw_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2854 {
2855         int min_cdclk, cdclk;
2856
2857         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2858         if (min_cdclk < 0)
2859                 return min_cdclk;
2860
2861         cdclk = bdw_calc_cdclk(min_cdclk);
2862
2863         cdclk_state->logical.cdclk = cdclk;
2864         cdclk_state->logical.voltage_level =
2865                 bdw_calc_voltage_level(cdclk);
2866
2867         if (!cdclk_state->active_pipes) {
2868                 cdclk = bdw_calc_cdclk(cdclk_state->force_min_cdclk);
2869
2870                 cdclk_state->actual.cdclk = cdclk;
2871                 cdclk_state->actual.voltage_level =
2872                         bdw_calc_voltage_level(cdclk);
2873         } else {
2874                 cdclk_state->actual = cdclk_state->logical;
2875         }
2876
2877         return 0;
2878 }
2879
2880 static int skl_dpll0_vco(struct intel_cdclk_state *cdclk_state)
2881 {
2882         struct intel_atomic_state *state = cdclk_state->base.state;
2883         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2884         struct intel_crtc *crtc;
2885         struct intel_crtc_state *crtc_state;
2886         int vco, i;
2887
2888         vco = cdclk_state->logical.vco;
2889         if (!vco)
2890                 vco = dev_priv->skl_preferred_vco_freq;
2891
2892         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2893                 if (!crtc_state->hw.enable)
2894                         continue;
2895
2896                 if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
2897                         continue;
2898
2899                 /*
2900                  * DPLL0 VCO may need to be adjusted to get the correct
2901                  * clock for eDP. This will affect cdclk as well.
2902                  */
2903                 switch (crtc_state->port_clock / 2) {
2904                 case 108000:
2905                 case 216000:
2906                         vco = 8640000;
2907                         break;
2908                 default:
2909                         vco = 8100000;
2910                         break;
2911                 }
2912         }
2913
2914         return vco;
2915 }
2916
2917 static int skl_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2918 {
2919         int min_cdclk, cdclk, vco;
2920
2921         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2922         if (min_cdclk < 0)
2923                 return min_cdclk;
2924
2925         vco = skl_dpll0_vco(cdclk_state);
2926
2927         cdclk = skl_calc_cdclk(min_cdclk, vco);
2928
2929         cdclk_state->logical.vco = vco;
2930         cdclk_state->logical.cdclk = cdclk;
2931         cdclk_state->logical.voltage_level =
2932                 skl_calc_voltage_level(cdclk);
2933
2934         if (!cdclk_state->active_pipes) {
2935                 cdclk = skl_calc_cdclk(cdclk_state->force_min_cdclk, vco);
2936
2937                 cdclk_state->actual.vco = vco;
2938                 cdclk_state->actual.cdclk = cdclk;
2939                 cdclk_state->actual.voltage_level =
2940                         skl_calc_voltage_level(cdclk);
2941         } else {
2942                 cdclk_state->actual = cdclk_state->logical;
2943         }
2944
2945         return 0;
2946 }
2947
2948 static int bxt_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2949 {
2950         struct intel_atomic_state *state = cdclk_state->base.state;
2951         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2952         int min_cdclk, min_voltage_level, cdclk, vco;
2953
2954         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2955         if (min_cdclk < 0)
2956                 return min_cdclk;
2957
2958         min_voltage_level = bxt_compute_min_voltage_level(cdclk_state);
2959         if (min_voltage_level < 0)
2960                 return min_voltage_level;
2961
2962         cdclk = bxt_calc_cdclk(dev_priv, min_cdclk);
2963         vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2964
2965         cdclk_state->logical.vco = vco;
2966         cdclk_state->logical.cdclk = cdclk;
2967         cdclk_state->logical.voltage_level =
2968                 max_t(int, min_voltage_level,
2969                       intel_cdclk_calc_voltage_level(dev_priv, cdclk));
2970
2971         if (!cdclk_state->active_pipes) {
2972                 cdclk = bxt_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2973                 vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2974
2975                 cdclk_state->actual.vco = vco;
2976                 cdclk_state->actual.cdclk = cdclk;
2977                 cdclk_state->actual.voltage_level =
2978                         intel_cdclk_calc_voltage_level(dev_priv, cdclk);
2979         } else {
2980                 cdclk_state->actual = cdclk_state->logical;
2981         }
2982
2983         return 0;
2984 }
2985
2986 static int fixed_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2987 {
2988         int min_cdclk;
2989
2990         /*
2991          * We can't change the cdclk frequency, but we still want to
2992          * check that the required minimum frequency doesn't exceed
2993          * the actual cdclk frequency.
2994          */
2995         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2996         if (min_cdclk < 0)
2997                 return min_cdclk;
2998
2999         return 0;
3000 }
3001
3002 static struct intel_global_state *intel_cdclk_duplicate_state(struct intel_global_obj *obj)
3003 {
3004         struct intel_cdclk_state *cdclk_state;
3005
3006         cdclk_state = kmemdup(obj->state, sizeof(*cdclk_state), GFP_KERNEL);
3007         if (!cdclk_state)
3008                 return NULL;
3009
3010         cdclk_state->pipe = INVALID_PIPE;
3011
3012         return &cdclk_state->base;
3013 }
3014
3015 static void intel_cdclk_destroy_state(struct intel_global_obj *obj,
3016                                       struct intel_global_state *state)
3017 {
3018         kfree(state);
3019 }
3020
3021 static const struct intel_global_state_funcs intel_cdclk_funcs = {
3022         .atomic_duplicate_state = intel_cdclk_duplicate_state,
3023         .atomic_destroy_state = intel_cdclk_destroy_state,
3024 };
3025
3026 struct intel_cdclk_state *
3027 intel_atomic_get_cdclk_state(struct intel_atomic_state *state)
3028 {
3029         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3030         struct intel_global_state *cdclk_state;
3031
3032         cdclk_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.cdclk.obj);
3033         if (IS_ERR(cdclk_state))
3034                 return ERR_CAST(cdclk_state);
3035
3036         return to_intel_cdclk_state(cdclk_state);
3037 }
3038
3039 int intel_cdclk_atomic_check(struct intel_atomic_state *state,
3040                              bool *need_cdclk_calc)
3041 {
3042         const struct intel_cdclk_state *old_cdclk_state;
3043         const struct intel_cdclk_state *new_cdclk_state;
3044         struct intel_plane_state __maybe_unused *plane_state;
3045         struct intel_plane *plane;
3046         int ret;
3047         int i;
3048
3049         /*
3050          * active_planes bitmask has been updated, and potentially affected
3051          * planes are part of the state. We can now compute the minimum cdclk
3052          * for each plane.
3053          */
3054         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
3055                 ret = intel_plane_calc_min_cdclk(state, plane, need_cdclk_calc);
3056                 if (ret)
3057                         return ret;
3058         }
3059
3060         ret = intel_bw_calc_min_cdclk(state, need_cdclk_calc);
3061         if (ret)
3062                 return ret;
3063
3064         old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
3065         new_cdclk_state = intel_atomic_get_new_cdclk_state(state);
3066
3067         if (new_cdclk_state &&
3068             old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk)
3069                 *need_cdclk_calc = true;
3070
3071         return 0;
3072 }
3073
3074 int intel_cdclk_init(struct drm_i915_private *dev_priv)
3075 {
3076         struct intel_cdclk_state *cdclk_state;
3077
3078         cdclk_state = kzalloc(sizeof(*cdclk_state), GFP_KERNEL);
3079         if (!cdclk_state)
3080                 return -ENOMEM;
3081
3082         intel_atomic_global_obj_init(dev_priv, &dev_priv->display.cdclk.obj,
3083                                      &cdclk_state->base, &intel_cdclk_funcs);
3084
3085         return 0;
3086 }
3087
3088 static bool intel_cdclk_need_serialize(struct drm_i915_private *i915,
3089                                        const struct intel_cdclk_state *old_cdclk_state,
3090                                        const struct intel_cdclk_state *new_cdclk_state)
3091 {
3092         bool power_well_cnt_changed = hweight8(old_cdclk_state->active_pipes) !=
3093                                       hweight8(new_cdclk_state->active_pipes);
3094         bool cdclk_changed = intel_cdclk_changed(&old_cdclk_state->actual,
3095                                                  &new_cdclk_state->actual);
3096         /*
3097          * We need to poke hw for gen >= 12, because we notify PCode if
3098          * pipe power well count changes.
3099          */
3100         return cdclk_changed || (IS_DG2(i915) && power_well_cnt_changed);
3101 }
3102
3103 int intel_modeset_calc_cdclk(struct intel_atomic_state *state)
3104 {
3105         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3106         const struct intel_cdclk_state *old_cdclk_state;
3107         struct intel_cdclk_state *new_cdclk_state;
3108         enum pipe pipe = INVALID_PIPE;
3109         int ret;
3110
3111         new_cdclk_state = intel_atomic_get_cdclk_state(state);
3112         if (IS_ERR(new_cdclk_state))
3113                 return PTR_ERR(new_cdclk_state);
3114
3115         old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
3116
3117         new_cdclk_state->active_pipes =
3118                 intel_calc_active_pipes(state, old_cdclk_state->active_pipes);
3119
3120         ret = intel_cdclk_modeset_calc_cdclk(dev_priv, new_cdclk_state);
3121         if (ret)
3122                 return ret;
3123
3124         if (intel_cdclk_need_serialize(dev_priv, old_cdclk_state, new_cdclk_state)) {
3125                 /*
3126                  * Also serialize commits across all crtcs
3127                  * if the actual hw needs to be poked.
3128                  */
3129                 ret = intel_atomic_serialize_global_state(&new_cdclk_state->base);
3130                 if (ret)
3131                         return ret;
3132         } else if (old_cdclk_state->active_pipes != new_cdclk_state->active_pipes ||
3133                    old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk ||
3134                    intel_cdclk_changed(&old_cdclk_state->logical,
3135                                        &new_cdclk_state->logical)) {
3136                 ret = intel_atomic_lock_global_state(&new_cdclk_state->base);
3137                 if (ret)
3138                         return ret;
3139         } else {
3140                 return 0;
3141         }
3142
3143         if (is_power_of_2(new_cdclk_state->active_pipes) &&
3144             intel_cdclk_can_cd2x_update(dev_priv,
3145                                         &old_cdclk_state->actual,
3146                                         &new_cdclk_state->actual)) {
3147                 struct intel_crtc *crtc;
3148                 struct intel_crtc_state *crtc_state;
3149
3150                 pipe = ilog2(new_cdclk_state->active_pipes);
3151                 crtc = intel_crtc_for_pipe(dev_priv, pipe);
3152
3153                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
3154                 if (IS_ERR(crtc_state))
3155                         return PTR_ERR(crtc_state);
3156
3157                 if (intel_crtc_needs_modeset(crtc_state))
3158                         pipe = INVALID_PIPE;
3159         }
3160
3161         if (intel_cdclk_can_crawl_and_squash(dev_priv,
3162                                              &old_cdclk_state->actual,
3163                                              &new_cdclk_state->actual)) {
3164                 drm_dbg_kms(&dev_priv->drm,
3165                             "Can change cdclk via crawling and squashing\n");
3166         } else if (intel_cdclk_can_squash(dev_priv,
3167                                         &old_cdclk_state->actual,
3168                                         &new_cdclk_state->actual)) {
3169                 drm_dbg_kms(&dev_priv->drm,
3170                             "Can change cdclk via squashing\n");
3171         } else if (intel_cdclk_can_crawl(dev_priv,
3172                                          &old_cdclk_state->actual,
3173                                          &new_cdclk_state->actual)) {
3174                 drm_dbg_kms(&dev_priv->drm,
3175                             "Can change cdclk via crawling\n");
3176         } else if (pipe != INVALID_PIPE) {
3177                 new_cdclk_state->pipe = pipe;
3178
3179                 drm_dbg_kms(&dev_priv->drm,
3180                             "Can change cdclk cd2x divider with pipe %c active\n",
3181                             pipe_name(pipe));
3182         } else if (intel_cdclk_needs_modeset(&old_cdclk_state->actual,
3183                                              &new_cdclk_state->actual)) {
3184                 /* All pipes must be switched off while we change the cdclk. */
3185                 ret = intel_modeset_all_pipes_late(state, "CDCLK change");
3186                 if (ret)
3187                         return ret;
3188
3189                 drm_dbg_kms(&dev_priv->drm,
3190                             "Modeset required for cdclk change\n");
3191         }
3192
3193         drm_dbg_kms(&dev_priv->drm,
3194                     "New cdclk calculated to be logical %u kHz, actual %u kHz\n",
3195                     new_cdclk_state->logical.cdclk,
3196                     new_cdclk_state->actual.cdclk);
3197         drm_dbg_kms(&dev_priv->drm,
3198                     "New voltage level calculated to be logical %u, actual %u\n",
3199                     new_cdclk_state->logical.voltage_level,
3200                     new_cdclk_state->actual.voltage_level);
3201
3202         return 0;
3203 }
3204
3205 static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
3206 {
3207         int max_cdclk_freq = dev_priv->display.cdclk.max_cdclk_freq;
3208
3209         if (DISPLAY_VER(dev_priv) >= 10)
3210                 return 2 * max_cdclk_freq;
3211         else if (DISPLAY_VER(dev_priv) == 9 ||
3212                  IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
3213                 return max_cdclk_freq;
3214         else if (IS_CHERRYVIEW(dev_priv))
3215                 return max_cdclk_freq*95/100;
3216         else if (DISPLAY_VER(dev_priv) < 4)
3217                 return 2*max_cdclk_freq*90/100;
3218         else
3219                 return max_cdclk_freq*90/100;
3220 }
3221
3222 /**
3223  * intel_update_max_cdclk - Determine the maximum support CDCLK frequency
3224  * @dev_priv: i915 device
3225  *
3226  * Determine the maximum CDCLK frequency the platform supports, and also
3227  * derive the maximum dot clock frequency the maximum CDCLK frequency
3228  * allows.
3229  */
3230 void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
3231 {
3232         if (IS_JASPERLAKE(dev_priv) || IS_ELKHARTLAKE(dev_priv)) {
3233                 if (dev_priv->display.cdclk.hw.ref == 24000)
3234                         dev_priv->display.cdclk.max_cdclk_freq = 552000;
3235                 else
3236                         dev_priv->display.cdclk.max_cdclk_freq = 556800;
3237         } else if (DISPLAY_VER(dev_priv) >= 11) {
3238                 if (dev_priv->display.cdclk.hw.ref == 24000)
3239                         dev_priv->display.cdclk.max_cdclk_freq = 648000;
3240                 else
3241                         dev_priv->display.cdclk.max_cdclk_freq = 652800;
3242         } else if (IS_GEMINILAKE(dev_priv)) {
3243                 dev_priv->display.cdclk.max_cdclk_freq = 316800;
3244         } else if (IS_BROXTON(dev_priv)) {
3245                 dev_priv->display.cdclk.max_cdclk_freq = 624000;
3246         } else if (DISPLAY_VER(dev_priv) == 9) {
3247                 u32 limit = intel_de_read(dev_priv, SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
3248                 int max_cdclk, vco;
3249
3250                 vco = dev_priv->skl_preferred_vco_freq;
3251                 drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
3252
3253                 /*
3254                  * Use the lower (vco 8640) cdclk values as a
3255                  * first guess. skl_calc_cdclk() will correct it
3256                  * if the preferred vco is 8100 instead.
3257                  */
3258                 if (limit == SKL_DFSM_CDCLK_LIMIT_675)
3259                         max_cdclk = 617143;
3260                 else if (limit == SKL_DFSM_CDCLK_LIMIT_540)
3261                         max_cdclk = 540000;
3262                 else if (limit == SKL_DFSM_CDCLK_LIMIT_450)
3263                         max_cdclk = 432000;
3264                 else
3265                         max_cdclk = 308571;
3266
3267                 dev_priv->display.cdclk.max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
3268         } else if (IS_BROADWELL(dev_priv))  {
3269                 /*
3270                  * FIXME with extra cooling we can allow
3271                  * 540 MHz for ULX and 675 Mhz for ULT.
3272                  * How can we know if extra cooling is
3273                  * available? PCI ID, VTB, something else?
3274                  */
3275                 if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
3276                         dev_priv->display.cdclk.max_cdclk_freq = 450000;
3277                 else if (IS_BROADWELL_ULX(dev_priv))
3278                         dev_priv->display.cdclk.max_cdclk_freq = 450000;
3279                 else if (IS_BROADWELL_ULT(dev_priv))
3280                         dev_priv->display.cdclk.max_cdclk_freq = 540000;
3281                 else
3282                         dev_priv->display.cdclk.max_cdclk_freq = 675000;
3283         } else if (IS_CHERRYVIEW(dev_priv)) {
3284                 dev_priv->display.cdclk.max_cdclk_freq = 320000;
3285         } else if (IS_VALLEYVIEW(dev_priv)) {
3286                 dev_priv->display.cdclk.max_cdclk_freq = 400000;
3287         } else {
3288                 /* otherwise assume cdclk is fixed */
3289                 dev_priv->display.cdclk.max_cdclk_freq = dev_priv->display.cdclk.hw.cdclk;
3290         }
3291
3292         dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
3293
3294         drm_dbg(&dev_priv->drm, "Max CD clock rate: %d kHz\n",
3295                 dev_priv->display.cdclk.max_cdclk_freq);
3296
3297         drm_dbg(&dev_priv->drm, "Max dotclock rate: %d kHz\n",
3298                 dev_priv->max_dotclk_freq);
3299 }
3300
3301 /**
3302  * intel_update_cdclk - Determine the current CDCLK frequency
3303  * @dev_priv: i915 device
3304  *
3305  * Determine the current CDCLK frequency.
3306  */
3307 void intel_update_cdclk(struct drm_i915_private *dev_priv)
3308 {
3309         intel_cdclk_get_cdclk(dev_priv, &dev_priv->display.cdclk.hw);
3310
3311         /*
3312          * 9:0 CMBUS [sic] CDCLK frequency (cdfreq):
3313          * Programmng [sic] note: bit[9:2] should be programmed to the number
3314          * of cdclk that generates 4MHz reference clock freq which is used to
3315          * generate GMBus clock. This will vary with the cdclk freq.
3316          */
3317         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
3318                 intel_de_write(dev_priv, GMBUSFREQ_VLV,
3319                                DIV_ROUND_UP(dev_priv->display.cdclk.hw.cdclk, 1000));
3320 }
3321
3322 static int dg1_rawclk(struct drm_i915_private *dev_priv)
3323 {
3324         /*
3325          * DG1 always uses a 38.4 MHz rawclk.  The bspec tells us
3326          * "Program Numerator=2, Denominator=4, Divider=37 decimal."
3327          */
3328         intel_de_write(dev_priv, PCH_RAWCLK_FREQ,
3329                        CNP_RAWCLK_DEN(4) | CNP_RAWCLK_DIV(37) | ICP_RAWCLK_NUM(2));
3330
3331         return 38400;
3332 }
3333
3334 static int cnp_rawclk(struct drm_i915_private *dev_priv)
3335 {
3336         u32 rawclk;
3337         int divider, fraction;
3338
3339         if (intel_de_read(dev_priv, SFUSE_STRAP) & SFUSE_STRAP_RAW_FREQUENCY) {
3340                 /* 24 MHz */
3341                 divider = 24000;
3342                 fraction = 0;
3343         } else {
3344                 /* 19.2 MHz */
3345                 divider = 19000;
3346                 fraction = 200;
3347         }
3348
3349         rawclk = CNP_RAWCLK_DIV(divider / 1000);
3350         if (fraction) {
3351                 int numerator = 1;
3352
3353                 rawclk |= CNP_RAWCLK_DEN(DIV_ROUND_CLOSEST(numerator * 1000,
3354                                                            fraction) - 1);
3355                 if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3356                         rawclk |= ICP_RAWCLK_NUM(numerator);
3357         }
3358
3359         intel_de_write(dev_priv, PCH_RAWCLK_FREQ, rawclk);
3360         return divider + fraction;
3361 }
3362
3363 static int pch_rawclk(struct drm_i915_private *dev_priv)
3364 {
3365         return (intel_de_read(dev_priv, PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK) * 1000;
3366 }
3367
3368 static int vlv_hrawclk(struct drm_i915_private *dev_priv)
3369 {
3370         /* RAWCLK_FREQ_VLV register updated from power well code */
3371         return vlv_get_cck_clock_hpll(dev_priv, "hrawclk",
3372                                       CCK_DISPLAY_REF_CLOCK_CONTROL);
3373 }
3374
3375 static int i9xx_hrawclk(struct drm_i915_private *dev_priv)
3376 {
3377         u32 clkcfg;
3378
3379         /*
3380          * hrawclock is 1/4 the FSB frequency
3381          *
3382          * Note that this only reads the state of the FSB
3383          * straps, not the actual FSB frequency. Some BIOSen
3384          * let you configure each independently. Ideally we'd
3385          * read out the actual FSB frequency but sadly we
3386          * don't know which registers have that information,
3387          * and all the relevant docs have gone to bit heaven :(
3388          */
3389         clkcfg = intel_de_read(dev_priv, CLKCFG) & CLKCFG_FSB_MASK;
3390
3391         if (IS_MOBILE(dev_priv)) {
3392                 switch (clkcfg) {
3393                 case CLKCFG_FSB_400:
3394                         return 100000;
3395                 case CLKCFG_FSB_533:
3396                         return 133333;
3397                 case CLKCFG_FSB_667:
3398                         return 166667;
3399                 case CLKCFG_FSB_800:
3400                         return 200000;
3401                 case CLKCFG_FSB_1067:
3402                         return 266667;
3403                 case CLKCFG_FSB_1333:
3404                         return 333333;
3405                 default:
3406                         MISSING_CASE(clkcfg);
3407                         return 133333;
3408                 }
3409         } else {
3410                 switch (clkcfg) {
3411                 case CLKCFG_FSB_400_ALT:
3412                         return 100000;
3413                 case CLKCFG_FSB_533:
3414                         return 133333;
3415                 case CLKCFG_FSB_667:
3416                         return 166667;
3417                 case CLKCFG_FSB_800:
3418                         return 200000;
3419                 case CLKCFG_FSB_1067_ALT:
3420                         return 266667;
3421                 case CLKCFG_FSB_1333_ALT:
3422                         return 333333;
3423                 case CLKCFG_FSB_1600_ALT:
3424                         return 400000;
3425                 default:
3426                         return 133333;
3427                 }
3428         }
3429 }
3430
3431 /**
3432  * intel_read_rawclk - Determine the current RAWCLK frequency
3433  * @dev_priv: i915 device
3434  *
3435  * Determine the current RAWCLK frequency. RAWCLK is a fixed
3436  * frequency clock so this needs to done only once.
3437  */
3438 u32 intel_read_rawclk(struct drm_i915_private *dev_priv)
3439 {
3440         u32 freq;
3441
3442         if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1)
3443                 freq = dg1_rawclk(dev_priv);
3444         else if (INTEL_PCH_TYPE(dev_priv) >= PCH_MTP)
3445                 /*
3446                  * MTL always uses a 38.4 MHz rawclk.  The bspec tells us
3447                  * "RAWCLK_FREQ defaults to the values for 38.4 and does
3448                  * not need to be programmed."
3449                  */
3450                 freq = 38400;
3451         else if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
3452                 freq = cnp_rawclk(dev_priv);
3453         else if (HAS_PCH_SPLIT(dev_priv))
3454                 freq = pch_rawclk(dev_priv);
3455         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
3456                 freq = vlv_hrawclk(dev_priv);
3457         else if (DISPLAY_VER(dev_priv) >= 3)
3458                 freq = i9xx_hrawclk(dev_priv);
3459         else
3460                 /* no rawclk on other platforms, or no need to know it */
3461                 return 0;
3462
3463         return freq;
3464 }
3465
3466 static int i915_cdclk_info_show(struct seq_file *m, void *unused)
3467 {
3468         struct drm_i915_private *i915 = m->private;
3469
3470         seq_printf(m, "Current CD clock frequency: %d kHz\n", i915->display.cdclk.hw.cdclk);
3471         seq_printf(m, "Max CD clock frequency: %d kHz\n", i915->display.cdclk.max_cdclk_freq);
3472         seq_printf(m, "Max pixel clock frequency: %d kHz\n", i915->max_dotclk_freq);
3473
3474         return 0;
3475 }
3476
3477 DEFINE_SHOW_ATTRIBUTE(i915_cdclk_info);
3478
3479 void intel_cdclk_debugfs_register(struct drm_i915_private *i915)
3480 {
3481         struct drm_minor *minor = i915->drm.primary;
3482
3483         debugfs_create_file("i915_cdclk_info", 0444, minor->debugfs_root,
3484                             i915, &i915_cdclk_info_fops);
3485 }
3486
3487 static const struct intel_cdclk_funcs mtl_cdclk_funcs = {
3488         .get_cdclk = bxt_get_cdclk,
3489         .set_cdclk = bxt_set_cdclk,
3490         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3491         .calc_voltage_level = tgl_calc_voltage_level,
3492 };
3493
3494 static const struct intel_cdclk_funcs rplu_cdclk_funcs = {
3495         .get_cdclk = bxt_get_cdclk,
3496         .set_cdclk = bxt_set_cdclk,
3497         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3498         .calc_voltage_level = rplu_calc_voltage_level,
3499 };
3500
3501 static const struct intel_cdclk_funcs tgl_cdclk_funcs = {
3502         .get_cdclk = bxt_get_cdclk,
3503         .set_cdclk = bxt_set_cdclk,
3504         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3505         .calc_voltage_level = tgl_calc_voltage_level,
3506 };
3507
3508 static const struct intel_cdclk_funcs ehl_cdclk_funcs = {
3509         .get_cdclk = bxt_get_cdclk,
3510         .set_cdclk = bxt_set_cdclk,
3511         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3512         .calc_voltage_level = ehl_calc_voltage_level,
3513 };
3514
3515 static const struct intel_cdclk_funcs icl_cdclk_funcs = {
3516         .get_cdclk = bxt_get_cdclk,
3517         .set_cdclk = bxt_set_cdclk,
3518         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3519         .calc_voltage_level = icl_calc_voltage_level,
3520 };
3521
3522 static const struct intel_cdclk_funcs bxt_cdclk_funcs = {
3523         .get_cdclk = bxt_get_cdclk,
3524         .set_cdclk = bxt_set_cdclk,
3525         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3526         .calc_voltage_level = bxt_calc_voltage_level,
3527 };
3528
3529 static const struct intel_cdclk_funcs skl_cdclk_funcs = {
3530         .get_cdclk = skl_get_cdclk,
3531         .set_cdclk = skl_set_cdclk,
3532         .modeset_calc_cdclk = skl_modeset_calc_cdclk,
3533 };
3534
3535 static const struct intel_cdclk_funcs bdw_cdclk_funcs = {
3536         .get_cdclk = bdw_get_cdclk,
3537         .set_cdclk = bdw_set_cdclk,
3538         .modeset_calc_cdclk = bdw_modeset_calc_cdclk,
3539 };
3540
3541 static const struct intel_cdclk_funcs chv_cdclk_funcs = {
3542         .get_cdclk = vlv_get_cdclk,
3543         .set_cdclk = chv_set_cdclk,
3544         .modeset_calc_cdclk = vlv_modeset_calc_cdclk,
3545 };
3546
3547 static const struct intel_cdclk_funcs vlv_cdclk_funcs = {
3548         .get_cdclk = vlv_get_cdclk,
3549         .set_cdclk = vlv_set_cdclk,
3550         .modeset_calc_cdclk = vlv_modeset_calc_cdclk,
3551 };
3552
3553 static const struct intel_cdclk_funcs hsw_cdclk_funcs = {
3554         .get_cdclk = hsw_get_cdclk,
3555         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3556 };
3557
3558 /* SNB, IVB, 965G, 945G */
3559 static const struct intel_cdclk_funcs fixed_400mhz_cdclk_funcs = {
3560         .get_cdclk = fixed_400mhz_get_cdclk,
3561         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3562 };
3563
3564 static const struct intel_cdclk_funcs ilk_cdclk_funcs = {
3565         .get_cdclk = fixed_450mhz_get_cdclk,
3566         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3567 };
3568
3569 static const struct intel_cdclk_funcs gm45_cdclk_funcs = {
3570         .get_cdclk = gm45_get_cdclk,
3571         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3572 };
3573
3574 /* G45 uses G33 */
3575
3576 static const struct intel_cdclk_funcs i965gm_cdclk_funcs = {
3577         .get_cdclk = i965gm_get_cdclk,
3578         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3579 };
3580
3581 /* i965G uses fixed 400 */
3582
3583 static const struct intel_cdclk_funcs pnv_cdclk_funcs = {
3584         .get_cdclk = pnv_get_cdclk,
3585         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3586 };
3587
3588 static const struct intel_cdclk_funcs g33_cdclk_funcs = {
3589         .get_cdclk = g33_get_cdclk,
3590         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3591 };
3592
3593 static const struct intel_cdclk_funcs i945gm_cdclk_funcs = {
3594         .get_cdclk = i945gm_get_cdclk,
3595         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3596 };
3597
3598 /* i945G uses fixed 400 */
3599
3600 static const struct intel_cdclk_funcs i915gm_cdclk_funcs = {
3601         .get_cdclk = i915gm_get_cdclk,
3602         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3603 };
3604
3605 static const struct intel_cdclk_funcs i915g_cdclk_funcs = {
3606         .get_cdclk = fixed_333mhz_get_cdclk,
3607         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3608 };
3609
3610 static const struct intel_cdclk_funcs i865g_cdclk_funcs = {
3611         .get_cdclk = fixed_266mhz_get_cdclk,
3612         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3613 };
3614
3615 static const struct intel_cdclk_funcs i85x_cdclk_funcs = {
3616         .get_cdclk = i85x_get_cdclk,
3617         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3618 };
3619
3620 static const struct intel_cdclk_funcs i845g_cdclk_funcs = {
3621         .get_cdclk = fixed_200mhz_get_cdclk,
3622         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3623 };
3624
3625 static const struct intel_cdclk_funcs i830_cdclk_funcs = {
3626         .get_cdclk = fixed_133mhz_get_cdclk,
3627         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3628 };
3629
3630 /**
3631  * intel_init_cdclk_hooks - Initialize CDCLK related modesetting hooks
3632  * @dev_priv: i915 device
3633  */
3634 void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
3635 {
3636         if (DISPLAY_VER(dev_priv) >= 20) {
3637                 dev_priv->display.funcs.cdclk = &mtl_cdclk_funcs;
3638                 dev_priv->display.cdclk.table = lnl_cdclk_table;
3639         } else if (DISPLAY_VER(dev_priv) >= 14) {
3640                 dev_priv->display.funcs.cdclk = &mtl_cdclk_funcs;
3641                 dev_priv->display.cdclk.table = mtl_cdclk_table;
3642         } else if (IS_DG2(dev_priv)) {
3643                 dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3644                 dev_priv->display.cdclk.table = dg2_cdclk_table;
3645         } else if (IS_ALDERLAKE_P(dev_priv)) {
3646                 /* Wa_22011320316:adl-p[a0] */
3647                 if (IS_ALDERLAKE_P(dev_priv) && IS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) {
3648                         dev_priv->display.cdclk.table = adlp_a_step_cdclk_table;
3649                         dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3650                 } else if (IS_RAPTORLAKE_U(dev_priv)) {
3651                         dev_priv->display.cdclk.table = rplu_cdclk_table;
3652                         dev_priv->display.funcs.cdclk = &rplu_cdclk_funcs;
3653                 } else {
3654                         dev_priv->display.cdclk.table = adlp_cdclk_table;
3655                         dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3656                 }
3657         } else if (IS_ROCKETLAKE(dev_priv)) {
3658                 dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3659                 dev_priv->display.cdclk.table = rkl_cdclk_table;
3660         } else if (DISPLAY_VER(dev_priv) >= 12) {
3661                 dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3662                 dev_priv->display.cdclk.table = icl_cdclk_table;
3663         } else if (IS_JASPERLAKE(dev_priv) || IS_ELKHARTLAKE(dev_priv)) {
3664                 dev_priv->display.funcs.cdclk = &ehl_cdclk_funcs;
3665                 dev_priv->display.cdclk.table = icl_cdclk_table;
3666         } else if (DISPLAY_VER(dev_priv) >= 11) {
3667                 dev_priv->display.funcs.cdclk = &icl_cdclk_funcs;
3668                 dev_priv->display.cdclk.table = icl_cdclk_table;
3669         } else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
3670                 dev_priv->display.funcs.cdclk = &bxt_cdclk_funcs;
3671                 if (IS_GEMINILAKE(dev_priv))
3672                         dev_priv->display.cdclk.table = glk_cdclk_table;
3673                 else
3674                         dev_priv->display.cdclk.table = bxt_cdclk_table;
3675         } else if (DISPLAY_VER(dev_priv) == 9) {
3676                 dev_priv->display.funcs.cdclk = &skl_cdclk_funcs;
3677         } else if (IS_BROADWELL(dev_priv)) {
3678                 dev_priv->display.funcs.cdclk = &bdw_cdclk_funcs;
3679         } else if (IS_HASWELL(dev_priv)) {
3680                 dev_priv->display.funcs.cdclk = &hsw_cdclk_funcs;
3681         } else if (IS_CHERRYVIEW(dev_priv)) {
3682                 dev_priv->display.funcs.cdclk = &chv_cdclk_funcs;
3683         } else if (IS_VALLEYVIEW(dev_priv)) {
3684                 dev_priv->display.funcs.cdclk = &vlv_cdclk_funcs;
3685         } else if (IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
3686                 dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3687         } else if (IS_IRONLAKE(dev_priv)) {
3688                 dev_priv->display.funcs.cdclk = &ilk_cdclk_funcs;
3689         } else if (IS_GM45(dev_priv)) {
3690                 dev_priv->display.funcs.cdclk = &gm45_cdclk_funcs;
3691         } else if (IS_G45(dev_priv)) {
3692                 dev_priv->display.funcs.cdclk = &g33_cdclk_funcs;
3693         } else if (IS_I965GM(dev_priv)) {
3694                 dev_priv->display.funcs.cdclk = &i965gm_cdclk_funcs;
3695         } else if (IS_I965G(dev_priv)) {
3696                 dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3697         } else if (IS_PINEVIEW(dev_priv)) {
3698                 dev_priv->display.funcs.cdclk = &pnv_cdclk_funcs;
3699         } else if (IS_G33(dev_priv)) {
3700                 dev_priv->display.funcs.cdclk = &g33_cdclk_funcs;
3701         } else if (IS_I945GM(dev_priv)) {
3702                 dev_priv->display.funcs.cdclk = &i945gm_cdclk_funcs;
3703         } else if (IS_I945G(dev_priv)) {
3704                 dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3705         } else if (IS_I915GM(dev_priv)) {
3706                 dev_priv->display.funcs.cdclk = &i915gm_cdclk_funcs;
3707         } else if (IS_I915G(dev_priv)) {
3708                 dev_priv->display.funcs.cdclk = &i915g_cdclk_funcs;
3709         } else if (IS_I865G(dev_priv)) {
3710                 dev_priv->display.funcs.cdclk = &i865g_cdclk_funcs;
3711         } else if (IS_I85X(dev_priv)) {
3712                 dev_priv->display.funcs.cdclk = &i85x_cdclk_funcs;
3713         } else if (IS_I845G(dev_priv)) {
3714                 dev_priv->display.funcs.cdclk = &i845g_cdclk_funcs;
3715         } else if (IS_I830(dev_priv)) {
3716                 dev_priv->display.funcs.cdclk = &i830_cdclk_funcs;
3717         }
3718
3719         if (drm_WARN(&dev_priv->drm, !dev_priv->display.funcs.cdclk,
3720                      "Unknown platform. Assuming i830\n"))
3721                 dev_priv->display.funcs.cdclk = &i830_cdclk_funcs;
3722 }