drivers/gpu/drm/i915/display/intel_bw.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <drm/drm_atomic_state_helper.h>
   7
   8 #include "i915_drv.h"
   9 #include "i915_reg.h"
  10 #include "i915_utils.h"
  11 #include "intel_atomic.h"
  12 #include "intel_bw.h"
  13 #include "intel_cdclk.h"
  14 #include "intel_display_core.h"
  15 #include "intel_display_types.h"
  16 #include "skl_watermark.h"
  17 #include "intel_mchbar_regs.h"
  18 #include "intel_pcode.h"
  19
  20 /* Parameters for Qclk Geyserville (QGV) */
  21 struct intel_qgv_point {
  22         u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd;
  23 };
  24
  25 struct intel_psf_gv_point {
  26         u8 clk; /* clock in multiples of 16.6666 MHz */
  27 };
  28
  29 struct intel_qgv_info {
  30         struct intel_qgv_point points[I915_NUM_QGV_POINTS];
  31         struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS];
  32         u8 num_points;
  33         u8 num_psf_points;
  34         u8 t_bl;
  35         u8 max_numchannels;
  36         u8 channel_width;
  37         u8 deinterleave;
  38 };
  39
  40 static int dg1_mchbar_read_qgv_point_info(struct drm_i915_private *dev_priv,
  41                                           struct intel_qgv_point *sp,
  42                                           int point)
  43 {
  44         u32 dclk_ratio, dclk_reference;
  45         u32 val;
  46
  47         val = intel_uncore_read(&dev_priv->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC);
  48         dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val);
  49         if (val & DG1_QCLK_REFERENCE)
  50                 dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */
  51         else
  52                 dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */
  53         sp->dclk = DIV_ROUND_UP((16667 * dclk_ratio * dclk_reference) + 500, 1000);
  54
  55         val = intel_uncore_read(&dev_priv->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
  56         if (val & DG1_GEAR_TYPE)
  57                 sp->dclk *= 2;
  58
  59         if (sp->dclk == 0)
  60                 return -EINVAL;
  61
  62         val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR);
  63         sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val);
  64         sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val);
  65
  66         val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH);
  67         sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val);
  68         sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val);
  69
  70         sp->t_rc = sp->t_rp + sp->t_ras;
  71
  72         return 0;
  73 }
  74
  75 static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv,
  76                                          struct intel_qgv_point *sp,
  77                                          int point)
  78 {
  79         u32 val = 0, val2 = 0;
  80         u16 dclk;
  81         int ret;
  82
  83         ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
  84                              ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point),
  85                              &val, &val2);
  86         if (ret)
  87                 return ret;
  88
  89         dclk = val & 0xffff;
  90         sp->dclk = DIV_ROUND_UP((16667 * dclk) + (DISPLAY_VER(dev_priv) >= 12 ? 500 : 0),
  91                                 1000);
  92         sp->t_rp = (val & 0xff0000) >> 16;
  93         sp->t_rcd = (val & 0xff000000) >> 24;
  94
  95         sp->t_rdpre = val2 & 0xff;
  96         sp->t_ras = (val2 & 0xff00) >> 8;
  97
  98         sp->t_rc = sp->t_rp + sp->t_ras;
  99
 100         return 0;
 101 }
 102
 103 static int adls_pcode_read_psf_gv_point_info(struct drm_i915_private *dev_priv,
 104                                             struct intel_psf_gv_point *points)
 105 {
 106         u32 val = 0;
 107         int ret;
 108         int i;
 109
 110         ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
 111                              ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL);
 112         if (ret)
 113                 return ret;
 114
 115         for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) {
 116                 points[i].clk = val & 0xff;
 117                 val >>= 8;
 118         }
 119
 120         return 0;
 121 }
 122
 123 static u16 icl_qgv_points_mask(struct drm_i915_private *i915)
 124 {
 125         unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
 126         unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
 127         u16 qgv_points = 0, psf_points = 0;
 128
 129         /*
 130          * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects
 131          * it with failure if we try masking any unadvertised points.
 132          * So need to operate only with those returned from PCode.
 133          */
 134         if (num_qgv_points > 0)
 135                 qgv_points = GENMASK(num_qgv_points - 1, 0);
 136
 137         if (num_psf_gv_points > 0)
 138                 psf_points = GENMASK(num_psf_gv_points - 1, 0);
 139
 140         return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points);
 141 }
 142
 143 static bool is_sagv_enabled(struct drm_i915_private *i915, u16 points_mask)
 144 {
 145         return !is_power_of_2(~points_mask & icl_qgv_points_mask(i915) &
 146                               ICL_PCODE_REQ_QGV_PT_MASK);
 147 }
 148
 149 int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv,
 150                                   u32 points_mask)
 151 {
 152         int ret;
 153
 154         if (DISPLAY_VER(dev_priv) >= 14)
 155                 return 0;
 156
 157         /* bspec says to keep retrying for at least 1 ms */
 158         ret = skl_pcode_request(&dev_priv->uncore, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG,
 159                                 points_mask,
 160                                 ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK,
 161                                 ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE,
 162                                 1);
 163
 164         if (ret < 0) {
 165                 drm_err(&dev_priv->drm,
 166                         "Failed to disable qgv points (0x%x) points: 0x%x\n",
 167                         ret, points_mask);
 168                 return ret;
 169         }
 170
 171         dev_priv->display.sagv.status = is_sagv_enabled(dev_priv, points_mask) ?
 172                 I915_SAGV_ENABLED : I915_SAGV_DISABLED;
 173
 174         return 0;
 175 }
 176
 177 static int mtl_read_qgv_point_info(struct drm_i915_private *dev_priv,
 178                                    struct intel_qgv_point *sp, int point)
 179 {
 180         u32 val, val2;
 181         u16 dclk;
 182
 183         val = intel_uncore_read(&dev_priv->uncore,
 184                                 MTL_MEM_SS_INFO_QGV_POINT_LOW(point));
 185         val2 = intel_uncore_read(&dev_priv->uncore,
 186                                  MTL_MEM_SS_INFO_QGV_POINT_HIGH(point));
 187         dclk = REG_FIELD_GET(MTL_DCLK_MASK, val);
 188         sp->dclk = DIV_ROUND_CLOSEST(16667 * dclk, 1000);
 189         sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val);
 190         sp->t_rcd = REG_FIELD_GET(MTL_TRCD_MASK, val);
 191
 192         sp->t_rdpre = REG_FIELD_GET(MTL_TRDPRE_MASK, val2);
 193         sp->t_ras = REG_FIELD_GET(MTL_TRAS_MASK, val2);
 194
 195         sp->t_rc = sp->t_rp + sp->t_ras;
 196
 197         return 0;
 198 }
 199
 200 static int
 201 intel_read_qgv_point_info(struct drm_i915_private *dev_priv,
 202                           struct intel_qgv_point *sp,
 203                           int point)
 204 {
 205         if (DISPLAY_VER(dev_priv) >= 14)
 206                 return mtl_read_qgv_point_info(dev_priv, sp, point);
 207         else if (IS_DG1(dev_priv))
 208                 return dg1_mchbar_read_qgv_point_info(dev_priv, sp, point);
 209         else
 210                 return icl_pcode_read_qgv_point_info(dev_priv, sp, point);
 211 }
 212
 213 static int icl_get_qgv_points(struct drm_i915_private *dev_priv,
 214                               struct intel_qgv_info *qi,
 215                               bool is_y_tile)
 216 {
 217         const struct dram_info *dram_info = &dev_priv->dram_info;
 218         int i, ret;
 219
 220         qi->num_points = dram_info->num_qgv_points;
 221         qi->num_psf_points = dram_info->num_psf_gv_points;
 222
 223         if (DISPLAY_VER(dev_priv) >= 14) {
 224                 switch (dram_info->type) {
 225                 case INTEL_DRAM_DDR4:
 226                         qi->t_bl = 4;
 227                         qi->max_numchannels = 2;
 228                         qi->channel_width = 64;
 229                         qi->deinterleave = 2;
 230                         break;
 231                 case INTEL_DRAM_DDR5:
 232                         qi->t_bl = 8;
 233                         qi->max_numchannels = 4;
 234                         qi->channel_width = 32;
 235                         qi->deinterleave = 2;
 236                         break;
 237                 case INTEL_DRAM_LPDDR4:
 238                 case INTEL_DRAM_LPDDR5:
 239                         qi->t_bl = 16;
 240                         qi->max_numchannels = 8;
 241                         qi->channel_width = 16;
 242                         qi->deinterleave = 4;
 243                         break;
 244                 default:
 245                         MISSING_CASE(dram_info->type);
 246                         return -EINVAL;
 247                 }
 248         } else if (DISPLAY_VER(dev_priv) >= 12) {
 249                 switch (dram_info->type) {
 250                 case INTEL_DRAM_DDR4:
 251                         qi->t_bl = is_y_tile ? 8 : 4;
 252                         qi->max_numchannels = 2;
 253                         qi->channel_width = 64;
 254                         qi->deinterleave = is_y_tile ? 1 : 2;
 255                         break;
 256                 case INTEL_DRAM_DDR5:
 257                         qi->t_bl = is_y_tile ? 16 : 8;
 258                         qi->max_numchannels = 4;
 259                         qi->channel_width = 32;
 260                         qi->deinterleave = is_y_tile ? 1 : 2;
 261                         break;
 262                 case INTEL_DRAM_LPDDR4:
 263                         if (IS_ROCKETLAKE(dev_priv)) {
 264                                 qi->t_bl = 8;
 265                                 qi->max_numchannels = 4;
 266                                 qi->channel_width = 32;
 267                                 qi->deinterleave = 2;
 268                                 break;
 269                         }
 270                         fallthrough;
 271                 case INTEL_DRAM_LPDDR5:
 272                         qi->t_bl = 16;
 273                         qi->max_numchannels = 8;
 274                         qi->channel_width = 16;
 275                         qi->deinterleave = is_y_tile ? 2 : 4;
 276                         break;
 277                 default:
 278                         qi->t_bl = 16;
 279                         qi->max_numchannels = 1;
 280                         break;
 281                 }
 282         } else if (DISPLAY_VER(dev_priv) == 11) {
 283                 qi->t_bl = dev_priv->dram_info.type == INTEL_DRAM_DDR4 ? 4 : 8;
 284                 qi->max_numchannels = 1;
 285         }
 286
 287         if (drm_WARN_ON(&dev_priv->drm,
 288                         qi->num_points > ARRAY_SIZE(qi->points)))
 289                 qi->num_points = ARRAY_SIZE(qi->points);
 290
 291         for (i = 0; i < qi->num_points; i++) {
 292                 struct intel_qgv_point *sp = &qi->points[i];
 293
 294                 ret = intel_read_qgv_point_info(dev_priv, sp, i);
 295                 if (ret) {
 296                         drm_dbg_kms(&dev_priv->drm, "Could not read QGV %d info\n", i);
 297                         return ret;
 298                 }
 299
 300                 drm_dbg_kms(&dev_priv->drm,
 301                             "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n",
 302                             i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras,
 303                             sp->t_rcd, sp->t_rc);
 304         }
 305
 306         if (qi->num_psf_points > 0) {
 307                 ret = adls_pcode_read_psf_gv_point_info(dev_priv, qi->psf_points);
 308                 if (ret) {
 309                         drm_err(&dev_priv->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n");
 310                         qi->num_psf_points = 0;
 311                 }
 312
 313                 for (i = 0; i < qi->num_psf_points; i++)
 314                         drm_dbg_kms(&dev_priv->drm,
 315                                     "PSF GV %d: CLK=%d \n",
 316                                     i, qi->psf_points[i].clk);
 317         }
 318
 319         return 0;
 320 }
 321
 322 static int adl_calc_psf_bw(int clk)
 323 {
 324         /*
 325          * clk is multiples of 16.666MHz (100/6)
 326          * According to BSpec PSF GV bandwidth is
 327          * calculated as BW = 64 * clk * 16.666Mhz
 328          */
 329         return DIV_ROUND_CLOSEST(64 * clk * 100, 6);
 330 }
 331
 332 static int icl_sagv_max_dclk(const struct intel_qgv_info *qi)
 333 {
 334         u16 dclk = 0;
 335         int i;
 336
 337         for (i = 0; i < qi->num_points; i++)
 338                 dclk = max(dclk, qi->points[i].dclk);
 339
 340         return dclk;
 341 }
 342
 343 struct intel_sa_info {
 344         u16 displayrtids;
 345         u8 deburst, deprogbwlimit, derating;
 346 };
 347
 348 static const struct intel_sa_info icl_sa_info = {
 349         .deburst = 8,
 350         .deprogbwlimit = 25, /* GB/s */
 351         .displayrtids = 128,
 352         .derating = 10,
 353 };
 354
 355 static const struct intel_sa_info tgl_sa_info = {
 356         .deburst = 16,
 357         .deprogbwlimit = 34, /* GB/s */
 358         .displayrtids = 256,
 359         .derating = 10,
 360 };
 361
 362 static const struct intel_sa_info rkl_sa_info = {
 363         .deburst = 8,
 364         .deprogbwlimit = 20, /* GB/s */
 365         .displayrtids = 128,
 366         .derating = 10,
 367 };
 368
 369 static const struct intel_sa_info adls_sa_info = {
 370         .deburst = 16,
 371         .deprogbwlimit = 38, /* GB/s */
 372         .displayrtids = 256,
 373         .derating = 10,
 374 };
 375
 376 static const struct intel_sa_info adlp_sa_info = {
 377         .deburst = 16,
 378         .deprogbwlimit = 38, /* GB/s */
 379         .displayrtids = 256,
 380         .derating = 20,
 381 };
 382
 383 static const struct intel_sa_info mtl_sa_info = {
 384         .deburst = 32,
 385         .deprogbwlimit = 38, /* GB/s */
 386         .displayrtids = 256,
 387         .derating = 10,
 388 };
 389
 390 static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
 391 {
 392         struct intel_qgv_info qi = {};
 393         bool is_y_tile = true; /* assume y tile may be used */
 394         int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
 395         int ipqdepth, ipqdepthpch = 16;
 396         int dclk_max;
 397         int maxdebw;
 398         int num_groups = ARRAY_SIZE(dev_priv->display.bw.max);
 399         int i, ret;
 400
 401         ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
 402         if (ret) {
 403                 drm_dbg_kms(&dev_priv->drm,
 404                             "Failed to get memory subsystem information, ignoring bandwidth limits");
 405                 return ret;
 406         }
 407
 408         dclk_max = icl_sagv_max_dclk(&qi);
 409         maxdebw = min(sa->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10);
 410         ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
 411         qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
 412
 413         for (i = 0; i < num_groups; i++) {
 414                 struct intel_bw_info *bi = &dev_priv->display.bw.max[i];
 415                 int clpchgroup;
 416                 int j;
 417
 418                 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
 419                 bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;
 420
 421                 bi->num_qgv_points = qi.num_points;
 422                 bi->num_psf_gv_points = qi.num_psf_points;
 423
 424                 for (j = 0; j < qi.num_points; j++) {
 425                         const struct intel_qgv_point *sp = &qi.points[j];
 426                         int ct, bw;
 427
 428                         /*
 429                          * Max row cycle time
 430                          *
 431                          * FIXME what is the logic behind the
 432                          * assumed burst length?
 433                          */
 434                         ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
 435                                    (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
 436                         bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
 437
 438                         bi->deratedbw[j] = min(maxdebw,
 439                                                bw * (100 - sa->derating) / 100);
 440
 441                         drm_dbg_kms(&dev_priv->drm,
 442                                     "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
 443                                     i, j, bi->num_planes, bi->deratedbw[j]);
 444                 }
 445         }
 446         /*
 447          * In case if SAGV is disabled in BIOS, we always get 1
 448          * SAGV point, but we can't send PCode commands to restrict it
 449          * as it will fail and pointless anyway.
 450          */
 451         if (qi.num_points == 1)
 452                 dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 453         else
 454                 dev_priv->display.sagv.status = I915_SAGV_ENABLED;
 455
 456         return 0;
 457 }
 458
 459 static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
 460 {
 461         struct intel_qgv_info qi = {};
 462         const struct dram_info *dram_info = &dev_priv->dram_info;
 463         bool is_y_tile = true; /* assume y tile may be used */
 464         int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
 465         int ipqdepth, ipqdepthpch = 16;
 466         int dclk_max;
 467         int maxdebw, peakbw;
 468         int clperchgroup;
 469         int num_groups = ARRAY_SIZE(dev_priv->display.bw.max);
 470         int i, ret;
 471
 472         ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
 473         if (ret) {
 474                 drm_dbg_kms(&dev_priv->drm,
 475                             "Failed to get memory subsystem information, ignoring bandwidth limits");
 476                 return ret;
 477         }
 478
 479         if (DISPLAY_VER(dev_priv) < 14 &&
 480             (dram_info->type == INTEL_DRAM_LPDDR4 || dram_info->type == INTEL_DRAM_LPDDR5))
 481                 num_channels *= 2;
 482
 483         qi.deinterleave = qi.deinterleave ? : DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
 484
 485         if (num_channels < qi.max_numchannels && DISPLAY_VER(dev_priv) >= 12)
 486                 qi.deinterleave = max(DIV_ROUND_UP(qi.deinterleave, 2), 1);
 487
 488         if (DISPLAY_VER(dev_priv) >= 12 && num_channels > qi.max_numchannels)
 489                 drm_warn(&dev_priv->drm, "Number of channels exceeds max number of channels.");
 490         if (qi.max_numchannels != 0)
 491                 num_channels = min_t(u8, num_channels, qi.max_numchannels);
 492
 493         dclk_max = icl_sagv_max_dclk(&qi);
 494
 495         peakbw = num_channels * DIV_ROUND_UP(qi.channel_width, 8) * dclk_max;
 496         maxdebw = min(sa->deprogbwlimit * 1000, peakbw * 6 / 10); /* 60% */
 497
 498         ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
 499         /*
 500          * clperchgroup = 4kpagespermempage * clperchperblock,
 501          * clperchperblock = 8 / num_channels * interleave
 502          */
 503         clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave;
 504
 505         for (i = 0; i < num_groups; i++) {
 506                 struct intel_bw_info *bi = &dev_priv->display.bw.max[i];
 507                 struct intel_bw_info *bi_next;
 508                 int clpchgroup;
 509                 int j;
 510
 511                 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
 512
 513                 if (i < num_groups - 1) {
 514                         bi_next = &dev_priv->display.bw.max[i + 1];
 515
 516                         if (clpchgroup < clperchgroup)
 517                                 bi_next->num_planes = (ipqdepth - clpchgroup) /
 518                                                        clpchgroup + 1;
 519                         else
 520                                 bi_next->num_planes = 0;
 521                 }
 522
 523                 bi->num_qgv_points = qi.num_points;
 524                 bi->num_psf_gv_points = qi.num_psf_points;
 525
 526                 for (j = 0; j < qi.num_points; j++) {
 527                         const struct intel_qgv_point *sp = &qi.points[j];
 528                         int ct, bw;
 529
 530                         /*
 531                          * Max row cycle time
 532                          *
 533                          * FIXME what is the logic behind the
 534                          * assumed burst length?
 535                          */
 536                         ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
 537                                    (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
 538                         bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
 539
 540                         bi->deratedbw[j] = min(maxdebw,
 541                                                bw * (100 - sa->derating) / 100);
 542                         bi->peakbw[j] = DIV_ROUND_CLOSEST(sp->dclk *
 543                                                           num_channels *
 544                                                           qi.channel_width, 8);
 545
 546                         drm_dbg_kms(&dev_priv->drm,
 547                                     "BW%d / QGV %d: num_planes=%d deratedbw=%u peakbw: %u\n",
 548                                     i, j, bi->num_planes, bi->deratedbw[j],
 549                                     bi->peakbw[j]);
 550                 }
 551
 552                 for (j = 0; j < qi.num_psf_points; j++) {
 553                         const struct intel_psf_gv_point *sp = &qi.psf_points[j];
 554
 555                         bi->psf_bw[j] = adl_calc_psf_bw(sp->clk);
 556
 557                         drm_dbg_kms(&dev_priv->drm,
 558                                     "BW%d / PSF GV %d: num_planes=%d bw=%u\n",
 559                                     i, j, bi->num_planes, bi->psf_bw[j]);
 560                 }
 561         }
 562
 563         /*
 564          * In case if SAGV is disabled in BIOS, we always get 1
 565          * SAGV point, but we can't send PCode commands to restrict it
 566          * as it will fail and pointless anyway.
 567          */
 568         if (qi.num_points == 1)
 569                 dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 570         else
 571                 dev_priv->display.sagv.status = I915_SAGV_ENABLED;
 572
 573         return 0;
 574 }
 575
 576 static void dg2_get_bw_info(struct drm_i915_private *i915)
 577 {
 578         unsigned int deratedbw = IS_DG2_G11(i915) ? 38000 : 50000;
 579         int num_groups = ARRAY_SIZE(i915->display.bw.max);
 580         int i;
 581
 582         /*
 583          * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth
 584          * that doesn't depend on the number of planes enabled. So fill all the
 585          * plane group with constant bw information for uniformity with other
 586          * platforms. DG2-G10 platforms have a constant 50 GB/s bandwidth,
 587          * whereas DG2-G11 platforms have 38 GB/s.
 588          */
 589         for (i = 0; i < num_groups; i++) {
 590                 struct intel_bw_info *bi = &i915->display.bw.max[i];
 591
 592                 bi->num_planes = 1;
 593                 /* Need only one dummy QGV point per group */
 594                 bi->num_qgv_points = 1;
 595                 bi->deratedbw[0] = deratedbw;
 596         }
 597
 598         i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 599 }
 600
 601 static unsigned int icl_max_bw_index(struct drm_i915_private *dev_priv,
 602                                      int num_planes, int qgv_point)
 603 {
 604         int i;
 605
 606         /*
 607          * Let's return max bw for 0 planes
 608          */
 609         num_planes = max(1, num_planes);
 610
 611         for (i = 0; i < ARRAY_SIZE(dev_priv->display.bw.max); i++) {
 612                 const struct intel_bw_info *bi =
 613                         &dev_priv->display.bw.max[i];
 614
 615                 /*
 616                  * Pcode will not expose all QGV points when
 617                  * SAGV is forced to off/min/med/max.
 618                  */
 619                 if (qgv_point >= bi->num_qgv_points)
 620                         return UINT_MAX;
 621
 622                 if (num_planes >= bi->num_planes)
 623                         return i;
 624         }
 625
 626         return UINT_MAX;
 627 }
 628
 629 static unsigned int tgl_max_bw_index(struct drm_i915_private *dev_priv,
 630                                      int num_planes, int qgv_point)
 631 {
 632         int i;
 633
 634         /*
 635          * Let's return max bw for 0 planes
 636          */
 637         num_planes = max(1, num_planes);
 638
 639         for (i = ARRAY_SIZE(dev_priv->display.bw.max) - 1; i >= 0; i--) {
 640                 const struct intel_bw_info *bi =
 641                         &dev_priv->display.bw.max[i];
 642
 643                 /*
 644                  * Pcode will not expose all QGV points when
 645                  * SAGV is forced to off/min/med/max.
 646                  */
 647                 if (qgv_point >= bi->num_qgv_points)
 648                         return UINT_MAX;
 649
 650                 if (num_planes <= bi->num_planes)
 651                         return i;
 652         }
 653
 654         return 0;
 655 }
 656
 657 static unsigned int adl_psf_bw(struct drm_i915_private *dev_priv,
 658                                int psf_gv_point)
 659 {
 660         const struct intel_bw_info *bi =
 661                         &dev_priv->display.bw.max[0];
 662
 663         return bi->psf_bw[psf_gv_point];
 664 }
 665
 666 static unsigned int icl_qgv_bw(struct drm_i915_private *i915,
 667                                int num_active_planes, int qgv_point)
 668 {
 669         unsigned int idx;
 670
 671         if (DISPLAY_VER(i915) >= 12)
 672                 idx = tgl_max_bw_index(i915, num_active_planes, qgv_point);
 673         else
 674                 idx = icl_max_bw_index(i915, num_active_planes, qgv_point);
 675
 676         if (idx >= ARRAY_SIZE(i915->display.bw.max))
 677                 return 0;
 678
 679         return i915->display.bw.max[idx].deratedbw[qgv_point];
 680 }
 681
 682 void intel_bw_init_hw(struct drm_i915_private *dev_priv)
 683 {
 684         if (!HAS_DISPLAY(dev_priv))
 685                 return;
 686
 687         if (DISPLAY_VER(dev_priv) >= 14)
 688                 tgl_get_bw_info(dev_priv, &mtl_sa_info);
 689         else if (IS_DG2(dev_priv))
 690                 dg2_get_bw_info(dev_priv);
 691         else if (IS_ALDERLAKE_P(dev_priv))
 692                 tgl_get_bw_info(dev_priv, &adlp_sa_info);
 693         else if (IS_ALDERLAKE_S(dev_priv))
 694                 tgl_get_bw_info(dev_priv, &adls_sa_info);
 695         else if (IS_ROCKETLAKE(dev_priv))
 696                 tgl_get_bw_info(dev_priv, &rkl_sa_info);
 697         else if (DISPLAY_VER(dev_priv) == 12)
 698                 tgl_get_bw_info(dev_priv, &tgl_sa_info);
 699         else if (DISPLAY_VER(dev_priv) == 11)
 700                 icl_get_bw_info(dev_priv, &icl_sa_info);
 701 }
 702
 703 static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state)
 704 {
 705         /*
 706          * We assume cursors are small enough
 707          * to not not cause bandwidth problems.
 708          */
 709         return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR));
 710 }
 711
 712 static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state)
 713 {
 714         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 715         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 716         unsigned int data_rate = 0;
 717         enum plane_id plane_id;
 718
 719         for_each_plane_id_on_crtc(crtc, plane_id) {
 720                 /*
 721                  * We assume cursors are small enough
 722                  * to not not cause bandwidth problems.
 723                  */
 724                 if (plane_id == PLANE_CURSOR)
 725                         continue;
 726
 727                 data_rate += crtc_state->data_rate[plane_id];
 728
 729                 if (DISPLAY_VER(i915) < 11)
 730                         data_rate += crtc_state->data_rate_y[plane_id];
 731         }
 732
 733         return data_rate;
 734 }
 735
 736 /* "Maximum Pipe Read Bandwidth" */
 737 static int intel_bw_crtc_min_cdclk(const struct intel_crtc_state *crtc_state)
 738 {
 739         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 740         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 741
 742         if (DISPLAY_VER(i915) < 12)
 743                 return 0;
 744
 745         return DIV_ROUND_UP_ULL(mul_u32_u32(intel_bw_crtc_data_rate(crtc_state), 10), 512);
 746 }
 747
 748 void intel_bw_crtc_update(struct intel_bw_state *bw_state,
 749                           const struct intel_crtc_state *crtc_state)
 750 {
 751         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 752         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 753
 754         bw_state->data_rate[crtc->pipe] =
 755                 intel_bw_crtc_data_rate(crtc_state);
 756         bw_state->num_active_planes[crtc->pipe] =
 757                 intel_bw_crtc_num_active_planes(crtc_state);
 758         bw_state->force_check_qgv = true;
 759
 760         drm_dbg_kms(&i915->drm, "pipe %c data rate %u num active planes %u\n",
 761                     pipe_name(crtc->pipe),
 762                     bw_state->data_rate[crtc->pipe],
 763                     bw_state->num_active_planes[crtc->pipe]);
 764 }
 765
 766 static unsigned int intel_bw_num_active_planes(struct drm_i915_private *dev_priv,
 767                                                const struct intel_bw_state *bw_state)
 768 {
 769         unsigned int num_active_planes = 0;
 770         enum pipe pipe;
 771
 772         for_each_pipe(dev_priv, pipe)
 773                 num_active_planes += bw_state->num_active_planes[pipe];
 774
 775         return num_active_planes;
 776 }
 777
 778 static unsigned int intel_bw_data_rate(struct drm_i915_private *dev_priv,
 779                                        const struct intel_bw_state *bw_state)
 780 {
 781         unsigned int data_rate = 0;
 782         enum pipe pipe;
 783
 784         for_each_pipe(dev_priv, pipe)
 785                 data_rate += bw_state->data_rate[pipe];
 786
 787         if (DISPLAY_VER(dev_priv) >= 13 && i915_vtd_active(dev_priv))
 788                 data_rate = DIV_ROUND_UP(data_rate * 105, 100);
 789
 790         return data_rate;
 791 }
 792
 793 struct intel_bw_state *
 794 intel_atomic_get_old_bw_state(struct intel_atomic_state *state)
 795 {
 796         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 797         struct intel_global_state *bw_state;
 798
 799         bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->display.bw.obj);
 800
 801         return to_intel_bw_state(bw_state);
 802 }
 803
 804 struct intel_bw_state *
 805 intel_atomic_get_new_bw_state(struct intel_atomic_state *state)
 806 {
 807         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 808         struct intel_global_state *bw_state;
 809
 810         bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->display.bw.obj);
 811
 812         return to_intel_bw_state(bw_state);
 813 }
 814
 815 struct intel_bw_state *
 816 intel_atomic_get_bw_state(struct intel_atomic_state *state)
 817 {
 818         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 819         struct intel_global_state *bw_state;
 820
 821         bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.bw.obj);
 822         if (IS_ERR(bw_state))
 823                 return ERR_CAST(bw_state);
 824
 825         return to_intel_bw_state(bw_state);
 826 }
 827
 828 static unsigned int icl_max_bw_qgv_point_mask(struct drm_i915_private *i915,
 829                                               int num_active_planes)
 830 {
 831         unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
 832         unsigned int max_bw_point = 0;
 833         unsigned int max_bw = 0;
 834         int i;
 835
 836         for (i = 0; i < num_qgv_points; i++) {
 837                 unsigned int max_data_rate =
 838                         icl_qgv_bw(i915, num_active_planes, i);
 839
 840                 /*
 841                  * We need to know which qgv point gives us
 842                  * maximum bandwidth in order to disable SAGV
 843                  * if we find that we exceed SAGV block time
 844                  * with watermarks. By that moment we already
 845                  * have those, as it is calculated earlier in
 846                  * intel_atomic_check,
 847                  */
 848                 if (max_data_rate > max_bw) {
 849                         max_bw_point = BIT(i);
 850                         max_bw = max_data_rate;
 851                 }
 852         }
 853
 854         return max_bw_point;
 855 }
 856
 857 static u16 icl_prepare_qgv_points_mask(struct drm_i915_private *i915,
 858                                        unsigned int qgv_points,
 859                                        unsigned int psf_points)
 860 {
 861         return ~(ICL_PCODE_REQ_QGV_PT(qgv_points) |
 862                  ADLS_PCODE_REQ_PSF_PT(psf_points)) & icl_qgv_points_mask(i915);
 863 }
 864
 865 static unsigned int icl_max_bw_psf_gv_point_mask(struct drm_i915_private *i915)
 866 {
 867         unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
 868         unsigned int max_bw_point_mask = 0;
 869         unsigned int max_bw = 0;
 870         int i;
 871
 872         for (i = 0; i < num_psf_gv_points; i++) {
 873                 unsigned int max_data_rate = adl_psf_bw(i915, i);
 874
 875                 if (max_data_rate > max_bw) {
 876                         max_bw_point_mask = BIT(i);
 877                         max_bw = max_data_rate;
 878                 } else if (max_data_rate == max_bw) {
 879                         max_bw_point_mask |= BIT(i);
 880                 }
 881         }
 882
 883         return max_bw_point_mask;
 884 }
 885
 886 static void icl_force_disable_sagv(struct drm_i915_private *i915,
 887                                    struct intel_bw_state *bw_state)
 888 {
 889         unsigned int qgv_points = icl_max_bw_qgv_point_mask(i915, 0);
 890         unsigned int psf_points = icl_max_bw_psf_gv_point_mask(i915);
 891
 892         bw_state->qgv_points_mask = icl_prepare_qgv_points_mask(i915,
 893                                                                 qgv_points,
 894                                                                 psf_points);
 895
 896         drm_dbg_kms(&i915->drm, "Forcing SAGV disable: mask 0x%x\n",
 897                     bw_state->qgv_points_mask);
 898
 899         icl_pcode_restrict_qgv_points(i915, bw_state->qgv_points_mask);
 900 }
 901
 902 static int mtl_find_qgv_points(struct drm_i915_private *i915,
 903                                unsigned int data_rate,
 904                                unsigned int num_active_planes,
 905                                struct intel_bw_state *new_bw_state)
 906 {
 907         unsigned int best_rate = UINT_MAX;
 908         unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
 909         unsigned int qgv_peak_bw  = 0;
 910         int i;
 911         int ret;
 912
 913         ret = intel_atomic_lock_global_state(&new_bw_state->base);
 914         if (ret)
 915                 return ret;
 916
 917         /*
 918          * If SAGV cannot be enabled, disable the pcode SAGV by passing all 1's
 919          * for qgv peak bw in PM Demand request. So assign UINT_MAX if SAGV is
 920          * not enabled. PM Demand code will clamp the value for the register
 921          */
 922         if (!intel_can_enable_sagv(i915, new_bw_state)) {
 923                 new_bw_state->qgv_point_peakbw = U16_MAX;
 924                 drm_dbg_kms(&i915->drm, "No SAGV, use UINT_MAX as peak bw.");
 925                 return 0;
 926         }
 927
 928         /*
 929          * Find the best QGV point by comparing the data_rate with max data rate
 930          * offered per plane group
 931          */
 932         for (i = 0; i < num_qgv_points; i++) {
 933                 unsigned int bw_index =
 934                         tgl_max_bw_index(i915, num_active_planes, i);
 935                 unsigned int max_data_rate;
 936
 937                 if (bw_index >= ARRAY_SIZE(i915->display.bw.max))
 938                         continue;
 939
 940                 max_data_rate = i915->display.bw.max[bw_index].deratedbw[i];
 941
 942                 if (max_data_rate < data_rate)
 943                         continue;
 944
 945                 if (max_data_rate - data_rate < best_rate) {
 946                         best_rate = max_data_rate - data_rate;
 947                         qgv_peak_bw = i915->display.bw.max[bw_index].peakbw[i];
 948                 }
 949
 950                 drm_dbg_kms(&i915->drm, "QGV point %d: max bw %d required %d qgv_peak_bw: %d\n",
 951                             i, max_data_rate, data_rate, qgv_peak_bw);
 952         }
 953
 954         drm_dbg_kms(&i915->drm, "Matching peaks QGV bw: %d for required data rate: %d\n",
 955                     qgv_peak_bw, data_rate);
 956
 957         /*
 958          * The display configuration cannot be supported if no QGV point
 959          * satisfying the required data rate is found
 960          */
 961         if (qgv_peak_bw == 0) {
 962                 drm_dbg_kms(&i915->drm, "No QGV points for bw %d for display configuration(%d active planes).\n",
 963                             data_rate, num_active_planes);
 964                 return -EINVAL;
 965         }
 966
 967         /* MTL PM DEMAND expects QGV BW parameter in multiples of 100 mbps */
 968         new_bw_state->qgv_point_peakbw = DIV_ROUND_CLOSEST(qgv_peak_bw, 100);
 969
 970         return 0;
 971 }
 972
 973 static int icl_find_qgv_points(struct drm_i915_private *i915,
 974                                unsigned int data_rate,
 975                                unsigned int num_active_planes,
 976                                const struct intel_bw_state *old_bw_state,
 977                                struct intel_bw_state *new_bw_state)
 978 {
 979         unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
 980         unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
 981         u16 psf_points = 0;
 982         u16 qgv_points = 0;
 983         int i;
 984         int ret;
 985
 986         ret = intel_atomic_lock_global_state(&new_bw_state->base);
 987         if (ret)
 988                 return ret;
 989
 990         for (i = 0; i < num_qgv_points; i++) {
 991                 unsigned int max_data_rate = icl_qgv_bw(i915,
 992                                                         num_active_planes, i);
 993                 if (max_data_rate >= data_rate)
 994                         qgv_points |= BIT(i);
 995
 996                 drm_dbg_kms(&i915->drm, "QGV point %d: max bw %d required %d\n",
 997                             i, max_data_rate, data_rate);
 998         }
 999
1000         for (i = 0; i < num_psf_gv_points; i++) {
1001                 unsigned int max_data_rate = adl_psf_bw(i915, i);
1002
1003                 if (max_data_rate >= data_rate)
1004                         psf_points |= BIT(i);
1005
1006                 drm_dbg_kms(&i915->drm, "PSF GV point %d: max bw %d"
1007                             " required %d\n",
1008                             i, max_data_rate, data_rate);
1009         }
1010
1011         /*
1012          * BSpec states that we always should have at least one allowed point
1013          * left, so if we couldn't - simply reject the configuration for obvious
1014          * reasons.
1015          */
1016         if (qgv_points == 0) {
1017                 drm_dbg_kms(&i915->drm, "No QGV points provide sufficient memory"
1018                             " bandwidth %d for display configuration(%d active planes).\n",
1019                             data_rate, num_active_planes);
1020                 return -EINVAL;
1021         }
1022
1023         if (num_psf_gv_points > 0 && psf_points == 0) {
1024                 drm_dbg_kms(&i915->drm, "No PSF GV points provide sufficient memory"
1025                             " bandwidth %d for display configuration(%d active planes).\n",
1026                             data_rate, num_active_planes);
1027                 return -EINVAL;
1028         }
1029
1030         /*
1031          * Leave only single point with highest bandwidth, if
1032          * we can't enable SAGV due to the increased memory latency it may
1033          * cause.
1034          */
1035         if (!intel_can_enable_sagv(i915, new_bw_state)) {
1036                 qgv_points = icl_max_bw_qgv_point_mask(i915, num_active_planes);
1037                 drm_dbg_kms(&i915->drm, "No SAGV, using single QGV point mask 0x%x\n",
1038                             qgv_points);
1039         }
1040
1041         /*
1042          * We store the ones which need to be masked as that is what PCode
1043          * actually accepts as a parameter.
1044          */
1045         new_bw_state->qgv_points_mask = icl_prepare_qgv_points_mask(i915,
1046                                                                     qgv_points,
1047                                                                     psf_points);
1048         /*
1049          * If the actual mask had changed we need to make sure that
1050          * the commits are serialized(in case this is a nomodeset, nonblocking)
1051          */
1052         if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) {
1053                 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
1054                 if (ret)
1055                         return ret;
1056         }
1057
1058         return 0;
1059 }
1060
1061 static int intel_bw_check_qgv_points(struct drm_i915_private *i915,
1062                                      const struct intel_bw_state *old_bw_state,
1063                                      struct intel_bw_state *new_bw_state)
1064 {
1065         unsigned int data_rate = intel_bw_data_rate(i915, new_bw_state);
1066         unsigned int num_active_planes =
1067                         intel_bw_num_active_planes(i915, new_bw_state);
1068
1069         data_rate = DIV_ROUND_UP(data_rate, 1000);
1070
1071         if (DISPLAY_VER(i915) >= 14)
1072                 return mtl_find_qgv_points(i915, data_rate, num_active_planes,
1073                                            new_bw_state);
1074         else
1075                 return icl_find_qgv_points(i915, data_rate, num_active_planes,
1076                                            old_bw_state, new_bw_state);
1077 }
1078
1079 static bool intel_bw_state_changed(struct drm_i915_private *i915,
1080                                    const struct intel_bw_state *old_bw_state,
1081                                    const struct intel_bw_state *new_bw_state)
1082 {
1083         enum pipe pipe;
1084
1085         for_each_pipe(i915, pipe) {
1086                 const struct intel_dbuf_bw *old_crtc_bw =
1087                         &old_bw_state->dbuf_bw[pipe];
1088                 const struct intel_dbuf_bw *new_crtc_bw =
1089                         &new_bw_state->dbuf_bw[pipe];
1090                 enum dbuf_slice slice;
1091
1092                 for_each_dbuf_slice(i915, slice) {
1093                         if (old_crtc_bw->max_bw[slice] != new_crtc_bw->max_bw[slice] ||
1094                             old_crtc_bw->active_planes[slice] != new_crtc_bw->active_planes[slice])
1095                                 return true;
1096                 }
1097
1098                 if (old_bw_state->min_cdclk[pipe] != new_bw_state->min_cdclk[pipe])
1099                         return true;
1100         }
1101
1102         return false;
1103 }
1104
1105 static void skl_plane_calc_dbuf_bw(struct intel_bw_state *bw_state,
1106                                    struct intel_crtc *crtc,
1107                                    enum plane_id plane_id,
1108                                    const struct skl_ddb_entry *ddb,
1109                                    unsigned int data_rate)
1110 {
1111         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1112         struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
1113         unsigned int dbuf_mask = skl_ddb_dbuf_slice_mask(i915, ddb);
1114         enum dbuf_slice slice;
1115
1116         /*
1117          * The arbiter can only really guarantee an
1118          * equal share of the total bw to each plane.
1119          */
1120         for_each_dbuf_slice_in_mask(i915, slice, dbuf_mask) {
1121                 crtc_bw->max_bw[slice] = max(crtc_bw->max_bw[slice], data_rate);
1122                 crtc_bw->active_planes[slice] |= BIT(plane_id);
1123         }
1124 }
1125
1126 static void skl_crtc_calc_dbuf_bw(struct intel_bw_state *bw_state,
1127                                   const struct intel_crtc_state *crtc_state)
1128 {
1129         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1130         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1131         struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
1132         enum plane_id plane_id;
1133
1134         memset(crtc_bw, 0, sizeof(*crtc_bw));
1135
1136         if (!crtc_state->hw.active)
1137                 return;
1138
1139         for_each_plane_id_on_crtc(crtc, plane_id) {
1140                 /*
1141                  * We assume cursors are small enough
1142                  * to not cause bandwidth problems.
1143                  */
1144                 if (plane_id == PLANE_CURSOR)
1145                         continue;
1146
1147                 skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
1148                                        &crtc_state->wm.skl.plane_ddb[plane_id],
1149                                        crtc_state->data_rate[plane_id]);
1150
1151                 if (DISPLAY_VER(i915) < 11)
1152                         skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
1153                                                &crtc_state->wm.skl.plane_ddb_y[plane_id],
1154                                                crtc_state->data_rate[plane_id]);
1155         }
1156 }
1157
1158 /* "Maximum Data Buffer Bandwidth" */
1159 static int
1160 intel_bw_dbuf_min_cdclk(struct drm_i915_private *i915,
1161                         const struct intel_bw_state *bw_state)
1162 {
1163         unsigned int total_max_bw = 0;
1164         enum dbuf_slice slice;
1165
1166         for_each_dbuf_slice(i915, slice) {
1167                 int num_active_planes = 0;
1168                 unsigned int max_bw = 0;
1169                 enum pipe pipe;
1170
1171                 /*
1172                  * The arbiter can only really guarantee an
1173                  * equal share of the total bw to each plane.
1174                  */
1175                 for_each_pipe(i915, pipe) {
1176                         const struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[pipe];
1177
1178                         max_bw = max(crtc_bw->max_bw[slice], max_bw);
1179                         num_active_planes += hweight8(crtc_bw->active_planes[slice]);
1180                 }
1181                 max_bw *= num_active_planes;
1182
1183                 total_max_bw = max(total_max_bw, max_bw);
1184         }
1185
1186         return DIV_ROUND_UP(total_max_bw, 64);
1187 }
1188
1189 int intel_bw_min_cdclk(struct drm_i915_private *i915,
1190                        const struct intel_bw_state *bw_state)
1191 {
1192         enum pipe pipe;
1193         int min_cdclk;
1194
1195         min_cdclk = intel_bw_dbuf_min_cdclk(i915, bw_state);
1196
1197         for_each_pipe(i915, pipe)
1198                 min_cdclk = max(bw_state->min_cdclk[pipe], min_cdclk);
1199
1200         return min_cdclk;
1201 }
1202
1203 int intel_bw_calc_min_cdclk(struct intel_atomic_state *state,
1204                             bool *need_cdclk_calc)
1205 {
1206         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
1207         struct intel_bw_state *new_bw_state = NULL;
1208         const struct intel_bw_state *old_bw_state = NULL;
1209         const struct intel_cdclk_state *cdclk_state;
1210         const struct intel_crtc_state *crtc_state;
1211         int old_min_cdclk, new_min_cdclk;
1212         struct intel_crtc *crtc;
1213         int i;
1214
1215         if (DISPLAY_VER(dev_priv) < 9)
1216                 return 0;
1217
1218         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
1219                 new_bw_state = intel_atomic_get_bw_state(state);
1220                 if (IS_ERR(new_bw_state))
1221                         return PTR_ERR(new_bw_state);
1222
1223                 old_bw_state = intel_atomic_get_old_bw_state(state);
1224
1225                 skl_crtc_calc_dbuf_bw(new_bw_state, crtc_state);
1226
1227                 new_bw_state->min_cdclk[crtc->pipe] =
1228                         intel_bw_crtc_min_cdclk(crtc_state);
1229         }
1230
1231         if (!old_bw_state)
1232                 return 0;
1233
1234         if (intel_bw_state_changed(dev_priv, old_bw_state, new_bw_state)) {
1235                 int ret = intel_atomic_lock_global_state(&new_bw_state->base);
1236                 if (ret)
1237                         return ret;
1238         }
1239
1240         old_min_cdclk = intel_bw_min_cdclk(dev_priv, old_bw_state);
1241         new_min_cdclk = intel_bw_min_cdclk(dev_priv, new_bw_state);
1242
1243         /*
1244          * No need to check against the cdclk state if
1245          * the min cdclk doesn't increase.
1246          *
1247          * Ie. we only ever increase the cdclk due to bandwidth
1248          * requirements. This can reduce back and forth
1249          * display blinking due to constant cdclk changes.
1250          */
1251         if (new_min_cdclk <= old_min_cdclk)
1252                 return 0;
1253
1254         cdclk_state = intel_atomic_get_cdclk_state(state);
1255         if (IS_ERR(cdclk_state))
1256                 return PTR_ERR(cdclk_state);
1257
1258         /*
1259          * No need to recalculate the cdclk state if
1260          * the min cdclk doesn't increase.
1261          *
1262          * Ie. we only ever increase the cdclk due to bandwidth
1263          * requirements. This can reduce back and forth
1264          * display blinking due to constant cdclk changes.
1265          */
1266         if (new_min_cdclk <= cdclk_state->bw_min_cdclk)
1267                 return 0;
1268
1269         drm_dbg_kms(&dev_priv->drm,
1270                     "new bandwidth min cdclk (%d kHz) > old min cdclk (%d kHz)\n",
1271                     new_min_cdclk, cdclk_state->bw_min_cdclk);
1272         *need_cdclk_calc = true;
1273
1274         return 0;
1275 }
1276
1277 static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed)
1278 {
1279         struct drm_i915_private *i915 = to_i915(state->base.dev);
1280         const struct intel_crtc_state *new_crtc_state, *old_crtc_state;
1281         struct intel_crtc *crtc;
1282         int i;
1283
1284         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
1285                                             new_crtc_state, i) {
1286                 unsigned int old_data_rate =
1287                         intel_bw_crtc_data_rate(old_crtc_state);
1288                 unsigned int new_data_rate =
1289                         intel_bw_crtc_data_rate(new_crtc_state);
1290                 unsigned int old_active_planes =
1291                         intel_bw_crtc_num_active_planes(old_crtc_state);
1292                 unsigned int new_active_planes =
1293                         intel_bw_crtc_num_active_planes(new_crtc_state);
1294                 struct intel_bw_state *new_bw_state;
1295
1296                 /*
1297                  * Avoid locking the bw state when
1298                  * nothing significant has changed.
1299                  */
1300                 if (old_data_rate == new_data_rate &&
1301                     old_active_planes == new_active_planes)
1302                         continue;
1303
1304                 new_bw_state = intel_atomic_get_bw_state(state);
1305                 if (IS_ERR(new_bw_state))
1306                         return PTR_ERR(new_bw_state);
1307
1308                 new_bw_state->data_rate[crtc->pipe] = new_data_rate;
1309                 new_bw_state->num_active_planes[crtc->pipe] = new_active_planes;
1310
1311                 *changed = true;
1312
1313                 drm_dbg_kms(&i915->drm,
1314                             "[CRTC:%d:%s] data rate %u num active planes %u\n",
1315                             crtc->base.base.id, crtc->base.name,
1316                             new_bw_state->data_rate[crtc->pipe],
1317                             new_bw_state->num_active_planes[crtc->pipe]);
1318         }
1319
1320         return 0;
1321 }
1322
1323 int intel_bw_atomic_check(struct intel_atomic_state *state)
1324 {
1325         bool changed = false;
1326         struct drm_i915_private *i915 = to_i915(state->base.dev);
1327         struct intel_bw_state *new_bw_state;
1328         const struct intel_bw_state *old_bw_state;
1329         int ret;
1330
1331         /* FIXME earlier gens need some checks too */
1332         if (DISPLAY_VER(i915) < 11)
1333                 return 0;
1334
1335         ret = intel_bw_check_data_rate(state, &changed);
1336         if (ret)
1337                 return ret;
1338
1339         old_bw_state = intel_atomic_get_old_bw_state(state);
1340         new_bw_state = intel_atomic_get_new_bw_state(state);
1341
1342         if (new_bw_state &&
1343             (intel_can_enable_sagv(i915, old_bw_state) !=
1344              intel_can_enable_sagv(i915, new_bw_state) ||
1345              new_bw_state->force_check_qgv))
1346                 changed = true;
1347
1348         /*
1349          * If none of our inputs (data rates, number of active
1350          * planes, SAGV yes/no) changed then nothing to do here.
1351          */
1352         if (!changed)
1353                 return 0;
1354
1355         ret = intel_bw_check_qgv_points(i915, old_bw_state, new_bw_state);
1356         if (ret)
1357                 return ret;
1358
1359         new_bw_state->force_check_qgv = false;
1360
1361         return 0;
1362 }
1363
1364 static struct intel_global_state *
1365 intel_bw_duplicate_state(struct intel_global_obj *obj)
1366 {
1367         struct intel_bw_state *state;
1368
1369         state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
1370         if (!state)
1371                 return NULL;
1372
1373         return &state->base;
1374 }
1375
1376 static void intel_bw_destroy_state(struct intel_global_obj *obj,
1377                                    struct intel_global_state *state)
1378 {
1379         kfree(state);
1380 }
1381
1382 static const struct intel_global_state_funcs intel_bw_funcs = {
1383         .atomic_duplicate_state = intel_bw_duplicate_state,
1384         .atomic_destroy_state = intel_bw_destroy_state,
1385 };
1386
1387 int intel_bw_init(struct drm_i915_private *i915)
1388 {
1389         struct intel_bw_state *state;
1390
1391         state = kzalloc(sizeof(*state), GFP_KERNEL);
1392         if (!state)
1393                 return -ENOMEM;
1394
1395         intel_atomic_global_obj_init(i915, &i915->display.bw.obj,
1396                                      &state->base, &intel_bw_funcs);
1397
1398         /*
1399          * Limit this only if we have SAGV. And for Display version 14 onwards
1400          * sagv is handled though pmdemand requests
1401          */
1402         if (intel_has_sagv(i915) && IS_DISPLAY_VER(i915, 11, 13))
1403                 icl_force_disable_sagv(i915, state);
1404
1405         return 0;
1406 }