drivers/gpu/drm/i915/display/intel_bw.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <drm/drm_atomic_state_helper.h>
   7
   8 #include "i915_drv.h"
   9 #include "i915_reg.h"
  10 #include "i915_utils.h"
  11 #include "intel_atomic.h"
  12 #include "intel_bw.h"
  13 #include "intel_cdclk.h"
  14 #include "intel_display_core.h"
  15 #include "intel_display_types.h"
  16 #include "skl_watermark.h"
  17 #include "intel_mchbar_regs.h"
  18 #include "intel_pcode.h"
  19
  20 /* Parameters for Qclk Geyserville (QGV) */
  21 struct intel_qgv_point {
  22         u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd;
  23 };
  24
  25 #define DEPROGBWPCLIMIT         60
  26
  27 struct intel_psf_gv_point {
  28         u8 clk; /* clock in multiples of 16.6666 MHz */
  29 };
  30
  31 struct intel_qgv_info {
  32         struct intel_qgv_point points[I915_NUM_QGV_POINTS];
  33         struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS];
  34         u8 num_points;
  35         u8 num_psf_points;
  36         u8 t_bl;
  37         u8 max_numchannels;
  38         u8 channel_width;
  39         u8 deinterleave;
  40 };
  41
  42 static int dg1_mchbar_read_qgv_point_info(struct drm_i915_private *dev_priv,
  43                                           struct intel_qgv_point *sp,
  44                                           int point)
  45 {
  46         u32 dclk_ratio, dclk_reference;
  47         u32 val;
  48
  49         val = intel_uncore_read(&dev_priv->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC);
  50         dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val);
  51         if (val & DG1_QCLK_REFERENCE)
  52                 dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */
  53         else
  54                 dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */
  55         sp->dclk = DIV_ROUND_UP((16667 * dclk_ratio * dclk_reference) + 500, 1000);
  56
  57         val = intel_uncore_read(&dev_priv->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
  58         if (val & DG1_GEAR_TYPE)
  59                 sp->dclk *= 2;
  60
  61         if (sp->dclk == 0)
  62                 return -EINVAL;
  63
  64         val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR);
  65         sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val);
  66         sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val);
  67
  68         val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH);
  69         sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val);
  70         sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val);
  71
  72         sp->t_rc = sp->t_rp + sp->t_ras;
  73
  74         return 0;
  75 }
  76
  77 static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv,
  78                                          struct intel_qgv_point *sp,
  79                                          int point)
  80 {
  81         u32 val = 0, val2 = 0;
  82         u16 dclk;
  83         int ret;
  84
  85         ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
  86                              ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point),
  87                              &val, &val2);
  88         if (ret)
  89                 return ret;
  90
  91         dclk = val & 0xffff;
  92         sp->dclk = DIV_ROUND_UP((16667 * dclk) + (DISPLAY_VER(dev_priv) >= 12 ? 500 : 0),
  93                                 1000);
  94         sp->t_rp = (val & 0xff0000) >> 16;
  95         sp->t_rcd = (val & 0xff000000) >> 24;
  96
  97         sp->t_rdpre = val2 & 0xff;
  98         sp->t_ras = (val2 & 0xff00) >> 8;
  99
 100         sp->t_rc = sp->t_rp + sp->t_ras;
 101
 102         return 0;
 103 }
 104
 105 static int adls_pcode_read_psf_gv_point_info(struct drm_i915_private *dev_priv,
 106                                             struct intel_psf_gv_point *points)
 107 {
 108         u32 val = 0;
 109         int ret;
 110         int i;
 111
 112         ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
 113                              ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL);
 114         if (ret)
 115                 return ret;
 116
 117         for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) {
 118                 points[i].clk = val & 0xff;
 119                 val >>= 8;
 120         }
 121
 122         return 0;
 123 }
 124
 125 static u16 icl_qgv_points_mask(struct drm_i915_private *i915)
 126 {
 127         unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
 128         unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
 129         u16 qgv_points = 0, psf_points = 0;
 130
 131         /*
 132          * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects
 133          * it with failure if we try masking any unadvertised points.
 134          * So need to operate only with those returned from PCode.
 135          */
 136         if (num_qgv_points > 0)
 137                 qgv_points = GENMASK(num_qgv_points - 1, 0);
 138
 139         if (num_psf_gv_points > 0)
 140                 psf_points = GENMASK(num_psf_gv_points - 1, 0);
 141
 142         return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points);
 143 }
 144
 145 static bool is_sagv_enabled(struct drm_i915_private *i915, u16 points_mask)
 146 {
 147         return !is_power_of_2(~points_mask & icl_qgv_points_mask(i915) &
 148                               ICL_PCODE_REQ_QGV_PT_MASK);
 149 }
 150
 151 int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv,
 152                                   u32 points_mask)
 153 {
 154         int ret;
 155
 156         if (DISPLAY_VER(dev_priv) >= 14)
 157                 return 0;
 158
 159         /* bspec says to keep retrying for at least 1 ms */
 160         ret = skl_pcode_request(&dev_priv->uncore, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG,
 161                                 points_mask,
 162                                 ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK,
 163                                 ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE,
 164                                 1);
 165
 166         if (ret < 0) {
 167                 drm_err(&dev_priv->drm,
 168                         "Failed to disable qgv points (0x%x) points: 0x%x\n",
 169                         ret, points_mask);
 170                 return ret;
 171         }
 172
 173         dev_priv->display.sagv.status = is_sagv_enabled(dev_priv, points_mask) ?
 174                 I915_SAGV_ENABLED : I915_SAGV_DISABLED;
 175
 176         return 0;
 177 }
 178
 179 static int mtl_read_qgv_point_info(struct drm_i915_private *dev_priv,
 180                                    struct intel_qgv_point *sp, int point)
 181 {
 182         u32 val, val2;
 183         u16 dclk;
 184
 185         val = intel_uncore_read(&dev_priv->uncore,
 186                                 MTL_MEM_SS_INFO_QGV_POINT_LOW(point));
 187         val2 = intel_uncore_read(&dev_priv->uncore,
 188                                  MTL_MEM_SS_INFO_QGV_POINT_HIGH(point));
 189         dclk = REG_FIELD_GET(MTL_DCLK_MASK, val);
 190         sp->dclk = DIV_ROUND_CLOSEST(16667 * dclk, 1000);
 191         sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val);
 192         sp->t_rcd = REG_FIELD_GET(MTL_TRCD_MASK, val);
 193
 194         sp->t_rdpre = REG_FIELD_GET(MTL_TRDPRE_MASK, val2);
 195         sp->t_ras = REG_FIELD_GET(MTL_TRAS_MASK, val2);
 196
 197         sp->t_rc = sp->t_rp + sp->t_ras;
 198
 199         return 0;
 200 }
 201
 202 static int
 203 intel_read_qgv_point_info(struct drm_i915_private *dev_priv,
 204                           struct intel_qgv_point *sp,
 205                           int point)
 206 {
 207         if (DISPLAY_VER(dev_priv) >= 14)
 208                 return mtl_read_qgv_point_info(dev_priv, sp, point);
 209         else if (IS_DG1(dev_priv))
 210                 return dg1_mchbar_read_qgv_point_info(dev_priv, sp, point);
 211         else
 212                 return icl_pcode_read_qgv_point_info(dev_priv, sp, point);
 213 }
 214
 215 static int icl_get_qgv_points(struct drm_i915_private *dev_priv,
 216                               struct intel_qgv_info *qi,
 217                               bool is_y_tile)
 218 {
 219         const struct dram_info *dram_info = &dev_priv->dram_info;
 220         int i, ret;
 221
 222         qi->num_points = dram_info->num_qgv_points;
 223         qi->num_psf_points = dram_info->num_psf_gv_points;
 224
 225         if (DISPLAY_VER(dev_priv) >= 14) {
 226                 switch (dram_info->type) {
 227                 case INTEL_DRAM_DDR4:
 228                         qi->t_bl = 4;
 229                         qi->max_numchannels = 2;
 230                         qi->channel_width = 64;
 231                         qi->deinterleave = 2;
 232                         break;
 233                 case INTEL_DRAM_DDR5:
 234                         qi->t_bl = 8;
 235                         qi->max_numchannels = 4;
 236                         qi->channel_width = 32;
 237                         qi->deinterleave = 2;
 238                         break;
 239                 case INTEL_DRAM_LPDDR4:
 240                 case INTEL_DRAM_LPDDR5:
 241                         qi->t_bl = 16;
 242                         qi->max_numchannels = 8;
 243                         qi->channel_width = 16;
 244                         qi->deinterleave = 4;
 245                         break;
 246                 case INTEL_DRAM_GDDR:
 247                         qi->channel_width = 32;
 248                         break;
 249                 default:
 250                         MISSING_CASE(dram_info->type);
 251                         return -EINVAL;
 252                 }
 253         } else if (DISPLAY_VER(dev_priv) >= 12) {
 254                 switch (dram_info->type) {
 255                 case INTEL_DRAM_DDR4:
 256                         qi->t_bl = is_y_tile ? 8 : 4;
 257                         qi->max_numchannels = 2;
 258                         qi->channel_width = 64;
 259                         qi->deinterleave = is_y_tile ? 1 : 2;
 260                         break;
 261                 case INTEL_DRAM_DDR5:
 262                         qi->t_bl = is_y_tile ? 16 : 8;
 263                         qi->max_numchannels = 4;
 264                         qi->channel_width = 32;
 265                         qi->deinterleave = is_y_tile ? 1 : 2;
 266                         break;
 267                 case INTEL_DRAM_LPDDR4:
 268                         if (IS_ROCKETLAKE(dev_priv)) {
 269                                 qi->t_bl = 8;
 270                                 qi->max_numchannels = 4;
 271                                 qi->channel_width = 32;
 272                                 qi->deinterleave = 2;
 273                                 break;
 274                         }
 275                         fallthrough;
 276                 case INTEL_DRAM_LPDDR5:
 277                         qi->t_bl = 16;
 278                         qi->max_numchannels = 8;
 279                         qi->channel_width = 16;
 280                         qi->deinterleave = is_y_tile ? 2 : 4;
 281                         break;
 282                 default:
 283                         qi->t_bl = 16;
 284                         qi->max_numchannels = 1;
 285                         break;
 286                 }
 287         } else if (DISPLAY_VER(dev_priv) == 11) {
 288                 qi->t_bl = dev_priv->dram_info.type == INTEL_DRAM_DDR4 ? 4 : 8;
 289                 qi->max_numchannels = 1;
 290         }
 291
 292         if (drm_WARN_ON(&dev_priv->drm,
 293                         qi->num_points > ARRAY_SIZE(qi->points)))
 294                 qi->num_points = ARRAY_SIZE(qi->points);
 295
 296         for (i = 0; i < qi->num_points; i++) {
 297                 struct intel_qgv_point *sp = &qi->points[i];
 298
 299                 ret = intel_read_qgv_point_info(dev_priv, sp, i);
 300                 if (ret) {
 301                         drm_dbg_kms(&dev_priv->drm, "Could not read QGV %d info\n", i);
 302                         return ret;
 303                 }
 304
 305                 drm_dbg_kms(&dev_priv->drm,
 306                             "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n",
 307                             i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras,
 308                             sp->t_rcd, sp->t_rc);
 309         }
 310
 311         if (qi->num_psf_points > 0) {
 312                 ret = adls_pcode_read_psf_gv_point_info(dev_priv, qi->psf_points);
 313                 if (ret) {
 314                         drm_err(&dev_priv->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n");
 315                         qi->num_psf_points = 0;
 316                 }
 317
 318                 for (i = 0; i < qi->num_psf_points; i++)
 319                         drm_dbg_kms(&dev_priv->drm,
 320                                     "PSF GV %d: CLK=%d \n",
 321                                     i, qi->psf_points[i].clk);
 322         }
 323
 324         return 0;
 325 }
 326
 327 static int adl_calc_psf_bw(int clk)
 328 {
 329         /*
 330          * clk is multiples of 16.666MHz (100/6)
 331          * According to BSpec PSF GV bandwidth is
 332          * calculated as BW = 64 * clk * 16.666Mhz
 333          */
 334         return DIV_ROUND_CLOSEST(64 * clk * 100, 6);
 335 }
 336
 337 static int icl_sagv_max_dclk(const struct intel_qgv_info *qi)
 338 {
 339         u16 dclk = 0;
 340         int i;
 341
 342         for (i = 0; i < qi->num_points; i++)
 343                 dclk = max(dclk, qi->points[i].dclk);
 344
 345         return dclk;
 346 }
 347
 348 struct intel_sa_info {
 349         u16 displayrtids;
 350         u8 deburst, deprogbwlimit, derating;
 351 };
 352
 353 static const struct intel_sa_info icl_sa_info = {
 354         .deburst = 8,
 355         .deprogbwlimit = 25, /* GB/s */
 356         .displayrtids = 128,
 357         .derating = 10,
 358 };
 359
 360 static const struct intel_sa_info tgl_sa_info = {
 361         .deburst = 16,
 362         .deprogbwlimit = 34, /* GB/s */
 363         .displayrtids = 256,
 364         .derating = 10,
 365 };
 366
 367 static const struct intel_sa_info rkl_sa_info = {
 368         .deburst = 8,
 369         .deprogbwlimit = 20, /* GB/s */
 370         .displayrtids = 128,
 371         .derating = 10,
 372 };
 373
 374 static const struct intel_sa_info adls_sa_info = {
 375         .deburst = 16,
 376         .deprogbwlimit = 38, /* GB/s */
 377         .displayrtids = 256,
 378         .derating = 10,
 379 };
 380
 381 static const struct intel_sa_info adlp_sa_info = {
 382         .deburst = 16,
 383         .deprogbwlimit = 38, /* GB/s */
 384         .displayrtids = 256,
 385         .derating = 20,
 386 };
 387
 388 static const struct intel_sa_info mtl_sa_info = {
 389         .deburst = 32,
 390         .deprogbwlimit = 38, /* GB/s */
 391         .displayrtids = 256,
 392         .derating = 10,
 393 };
 394
 395 static const struct intel_sa_info xe2_hpd_sa_info = {
 396         .derating = 30,
 397         .deprogbwlimit = 53,
 398         /* Other values not used by simplified algorithm */
 399 };
 400
 401 static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
 402 {
 403         struct intel_qgv_info qi = {};
 404         bool is_y_tile = true; /* assume y tile may be used */
 405         int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
 406         int ipqdepth, ipqdepthpch = 16;
 407         int dclk_max;
 408         int maxdebw;
 409         int num_groups = ARRAY_SIZE(dev_priv->display.bw.max);
 410         int i, ret;
 411
 412         ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
 413         if (ret) {
 414                 drm_dbg_kms(&dev_priv->drm,
 415                             "Failed to get memory subsystem information, ignoring bandwidth limits");
 416                 return ret;
 417         }
 418
 419         dclk_max = icl_sagv_max_dclk(&qi);
 420         maxdebw = min(sa->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10);
 421         ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
 422         qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
 423
 424         for (i = 0; i < num_groups; i++) {
 425                 struct intel_bw_info *bi = &dev_priv->display.bw.max[i];
 426                 int clpchgroup;
 427                 int j;
 428
 429                 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
 430                 bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;
 431
 432                 bi->num_qgv_points = qi.num_points;
 433                 bi->num_psf_gv_points = qi.num_psf_points;
 434
 435                 for (j = 0; j < qi.num_points; j++) {
 436                         const struct intel_qgv_point *sp = &qi.points[j];
 437                         int ct, bw;
 438
 439                         /*
 440                          * Max row cycle time
 441                          *
 442                          * FIXME what is the logic behind the
 443                          * assumed burst length?
 444                          */
 445                         ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
 446                                    (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
 447                         bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
 448
 449                         bi->deratedbw[j] = min(maxdebw,
 450                                                bw * (100 - sa->derating) / 100);
 451
 452                         drm_dbg_kms(&dev_priv->drm,
 453                                     "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
 454                                     i, j, bi->num_planes, bi->deratedbw[j]);
 455                 }
 456         }
 457         /*
 458          * In case if SAGV is disabled in BIOS, we always get 1
 459          * SAGV point, but we can't send PCode commands to restrict it
 460          * as it will fail and pointless anyway.
 461          */
 462         if (qi.num_points == 1)
 463                 dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 464         else
 465                 dev_priv->display.sagv.status = I915_SAGV_ENABLED;
 466
 467         return 0;
 468 }
 469
 470 static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
 471 {
 472         struct intel_qgv_info qi = {};
 473         const struct dram_info *dram_info = &dev_priv->dram_info;
 474         bool is_y_tile = true; /* assume y tile may be used */
 475         int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
 476         int ipqdepth, ipqdepthpch = 16;
 477         int dclk_max;
 478         int maxdebw, peakbw;
 479         int clperchgroup;
 480         int num_groups = ARRAY_SIZE(dev_priv->display.bw.max);
 481         int i, ret;
 482
 483         ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
 484         if (ret) {
 485                 drm_dbg_kms(&dev_priv->drm,
 486                             "Failed to get memory subsystem information, ignoring bandwidth limits");
 487                 return ret;
 488         }
 489
 490         if (DISPLAY_VER(dev_priv) < 14 &&
 491             (dram_info->type == INTEL_DRAM_LPDDR4 || dram_info->type == INTEL_DRAM_LPDDR5))
 492                 num_channels *= 2;
 493
 494         qi.deinterleave = qi.deinterleave ? : DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
 495
 496         if (num_channels < qi.max_numchannels && DISPLAY_VER(dev_priv) >= 12)
 497                 qi.deinterleave = max(DIV_ROUND_UP(qi.deinterleave, 2), 1);
 498
 499         if (DISPLAY_VER(dev_priv) >= 12 && num_channels > qi.max_numchannels)
 500                 drm_warn(&dev_priv->drm, "Number of channels exceeds max number of channels.");
 501         if (qi.max_numchannels != 0)
 502                 num_channels = min_t(u8, num_channels, qi.max_numchannels);
 503
 504         dclk_max = icl_sagv_max_dclk(&qi);
 505
 506         peakbw = num_channels * DIV_ROUND_UP(qi.channel_width, 8) * dclk_max;
 507         maxdebw = min(sa->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 100);
 508
 509         ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
 510         /*
 511          * clperchgroup = 4kpagespermempage * clperchperblock,
 512          * clperchperblock = 8 / num_channels * interleave
 513          */
 514         clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave;
 515
 516         for (i = 0; i < num_groups; i++) {
 517                 struct intel_bw_info *bi = &dev_priv->display.bw.max[i];
 518                 struct intel_bw_info *bi_next;
 519                 int clpchgroup;
 520                 int j;
 521
 522                 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
 523
 524                 if (i < num_groups - 1) {
 525                         bi_next = &dev_priv->display.bw.max[i + 1];
 526
 527                         if (clpchgroup < clperchgroup)
 528                                 bi_next->num_planes = (ipqdepth - clpchgroup) /
 529                                                        clpchgroup + 1;
 530                         else
 531                                 bi_next->num_planes = 0;
 532                 }
 533
 534                 bi->num_qgv_points = qi.num_points;
 535                 bi->num_psf_gv_points = qi.num_psf_points;
 536
 537                 for (j = 0; j < qi.num_points; j++) {
 538                         const struct intel_qgv_point *sp = &qi.points[j];
 539                         int ct, bw;
 540
 541                         /*
 542                          * Max row cycle time
 543                          *
 544                          * FIXME what is the logic behind the
 545                          * assumed burst length?
 546                          */
 547                         ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
 548                                    (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
 549                         bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
 550
 551                         bi->deratedbw[j] = min(maxdebw,
 552                                                bw * (100 - sa->derating) / 100);
 553                         bi->peakbw[j] = DIV_ROUND_CLOSEST(sp->dclk *
 554                                                           num_channels *
 555                                                           qi.channel_width, 8);
 556
 557                         drm_dbg_kms(&dev_priv->drm,
 558                                     "BW%d / QGV %d: num_planes=%d deratedbw=%u peakbw: %u\n",
 559                                     i, j, bi->num_planes, bi->deratedbw[j],
 560                                     bi->peakbw[j]);
 561                 }
 562
 563                 for (j = 0; j < qi.num_psf_points; j++) {
 564                         const struct intel_psf_gv_point *sp = &qi.psf_points[j];
 565
 566                         bi->psf_bw[j] = adl_calc_psf_bw(sp->clk);
 567
 568                         drm_dbg_kms(&dev_priv->drm,
 569                                     "BW%d / PSF GV %d: num_planes=%d bw=%u\n",
 570                                     i, j, bi->num_planes, bi->psf_bw[j]);
 571                 }
 572         }
 573
 574         /*
 575          * In case if SAGV is disabled in BIOS, we always get 1
 576          * SAGV point, but we can't send PCode commands to restrict it
 577          * as it will fail and pointless anyway.
 578          */
 579         if (qi.num_points == 1)
 580                 dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 581         else
 582                 dev_priv->display.sagv.status = I915_SAGV_ENABLED;
 583
 584         return 0;
 585 }
 586
 587 static void dg2_get_bw_info(struct drm_i915_private *i915)
 588 {
 589         unsigned int deratedbw = IS_DG2_G11(i915) ? 38000 : 50000;
 590         int num_groups = ARRAY_SIZE(i915->display.bw.max);
 591         int i;
 592
 593         /*
 594          * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth
 595          * that doesn't depend on the number of planes enabled. So fill all the
 596          * plane group with constant bw information for uniformity with other
 597          * platforms. DG2-G10 platforms have a constant 50 GB/s bandwidth,
 598          * whereas DG2-G11 platforms have 38 GB/s.
 599          */
 600         for (i = 0; i < num_groups; i++) {
 601                 struct intel_bw_info *bi = &i915->display.bw.max[i];
 602
 603                 bi->num_planes = 1;
 604                 /* Need only one dummy QGV point per group */
 605                 bi->num_qgv_points = 1;
 606                 bi->deratedbw[0] = deratedbw;
 607         }
 608
 609         i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 610 }
 611
 612 static int xe2_hpd_get_bw_info(struct drm_i915_private *i915,
 613                                const struct intel_sa_info *sa)
 614 {
 615         struct intel_qgv_info qi = {};
 616         int num_channels = i915->dram_info.num_channels;
 617         int peakbw, maxdebw;
 618         int ret, i;
 619
 620         ret = icl_get_qgv_points(i915, &qi, true);
 621         if (ret) {
 622                 drm_dbg_kms(&i915->drm,
 623                             "Failed to get memory subsystem information, ignoring bandwidth limits");
 624                 return ret;
 625         }
 626
 627         peakbw = num_channels * qi.channel_width / 8 * icl_sagv_max_dclk(&qi);
 628         maxdebw = min(sa->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 10);
 629
 630         for (i = 0; i < qi.num_points; i++) {
 631                 const struct intel_qgv_point *point = &qi.points[i];
 632                 int bw = num_channels * (qi.channel_width / 8) * point->dclk;
 633
 634                 i915->display.bw.max[0].deratedbw[i] =
 635                         min(maxdebw, (100 - sa->derating) * bw / 100);
 636                 i915->display.bw.max[0].peakbw[i] = bw;
 637
 638                 drm_dbg_kms(&i915->drm, "QGV %d: deratedbw=%u peakbw: %u\n",
 639                             i, i915->display.bw.max[0].deratedbw[i],
 640                             i915->display.bw.max[0].peakbw[i]);
 641         }
 642
 643         /* Bandwidth does not depend on # of planes; set all groups the same */
 644         i915->display.bw.max[0].num_planes = 1;
 645         i915->display.bw.max[0].num_qgv_points = qi.num_points;
 646         for (i = 1; i < ARRAY_SIZE(i915->display.bw.max); i++)
 647                 memcpy(&i915->display.bw.max[i], &i915->display.bw.max[0],
 648                        sizeof(i915->display.bw.max[0]));
 649
 650         /*
 651          * Xe2_HPD should always have exactly two QGV points representing
 652          * battery and plugged-in operation.
 653          */
 654         drm_WARN_ON(&i915->drm, qi.num_points != 2);
 655         i915->display.sagv.status = I915_SAGV_ENABLED;
 656
 657         return 0;
 658 }
 659
 660 static unsigned int icl_max_bw_index(struct drm_i915_private *dev_priv,
 661                                      int num_planes, int qgv_point)
 662 {
 663         int i;
 664
 665         /*
 666          * Let's return max bw for 0 planes
 667          */
 668         num_planes = max(1, num_planes);
 669
 670         for (i = 0; i < ARRAY_SIZE(dev_priv->display.bw.max); i++) {
 671                 const struct intel_bw_info *bi =
 672                         &dev_priv->display.bw.max[i];
 673
 674                 /*
 675                  * Pcode will not expose all QGV points when
 676                  * SAGV is forced to off/min/med/max.
 677                  */
 678                 if (qgv_point >= bi->num_qgv_points)
 679                         return UINT_MAX;
 680
 681                 if (num_planes >= bi->num_planes)
 682                         return i;
 683         }
 684
 685         return UINT_MAX;
 686 }
 687
 688 static unsigned int tgl_max_bw_index(struct drm_i915_private *dev_priv,
 689                                      int num_planes, int qgv_point)
 690 {
 691         int i;
 692
 693         /*
 694          * Let's return max bw for 0 planes
 695          */
 696         num_planes = max(1, num_planes);
 697
 698         for (i = ARRAY_SIZE(dev_priv->display.bw.max) - 1; i >= 0; i--) {
 699                 const struct intel_bw_info *bi =
 700                         &dev_priv->display.bw.max[i];
 701
 702                 /*
 703                  * Pcode will not expose all QGV points when
 704                  * SAGV is forced to off/min/med/max.
 705                  */
 706                 if (qgv_point >= bi->num_qgv_points)
 707                         return UINT_MAX;
 708
 709                 if (num_planes <= bi->num_planes)
 710                         return i;
 711         }
 712
 713         return 0;
 714 }
 715
 716 static unsigned int adl_psf_bw(struct drm_i915_private *dev_priv,
 717                                int psf_gv_point)
 718 {
 719         const struct intel_bw_info *bi =
 720                         &dev_priv->display.bw.max[0];
 721
 722         return bi->psf_bw[psf_gv_point];
 723 }
 724
 725 static unsigned int icl_qgv_bw(struct drm_i915_private *i915,
 726                                int num_active_planes, int qgv_point)
 727 {
 728         unsigned int idx;
 729
 730         if (DISPLAY_VER(i915) >= 12)
 731                 idx = tgl_max_bw_index(i915, num_active_planes, qgv_point);
 732         else
 733                 idx = icl_max_bw_index(i915, num_active_planes, qgv_point);
 734
 735         if (idx >= ARRAY_SIZE(i915->display.bw.max))
 736                 return 0;
 737
 738         return i915->display.bw.max[idx].deratedbw[qgv_point];
 739 }
 740
 741 void intel_bw_init_hw(struct drm_i915_private *dev_priv)
 742 {
 743         if (!HAS_DISPLAY(dev_priv))
 744                 return;
 745
 746         if (DISPLAY_VERx100(dev_priv) >= 1401 && IS_DGFX(dev_priv))
 747                 xe2_hpd_get_bw_info(dev_priv, &xe2_hpd_sa_info);
 748         else if (DISPLAY_VER(dev_priv) >= 14)
 749                 tgl_get_bw_info(dev_priv, &mtl_sa_info);
 750         else if (IS_DG2(dev_priv))
 751                 dg2_get_bw_info(dev_priv);
 752         else if (IS_ALDERLAKE_P(dev_priv))
 753                 tgl_get_bw_info(dev_priv, &adlp_sa_info);
 754         else if (IS_ALDERLAKE_S(dev_priv))
 755                 tgl_get_bw_info(dev_priv, &adls_sa_info);
 756         else if (IS_ROCKETLAKE(dev_priv))
 757                 tgl_get_bw_info(dev_priv, &rkl_sa_info);
 758         else if (DISPLAY_VER(dev_priv) == 12)
 759                 tgl_get_bw_info(dev_priv, &tgl_sa_info);
 760         else if (DISPLAY_VER(dev_priv) == 11)
 761                 icl_get_bw_info(dev_priv, &icl_sa_info);
 762 }
 763
 764 static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state)
 765 {
 766         /*
 767          * We assume cursors are small enough
 768          * to not not cause bandwidth problems.
 769          */
 770         return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR));
 771 }
 772
 773 static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state)
 774 {
 775         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 776         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 777         unsigned int data_rate = 0;
 778         enum plane_id plane_id;
 779
 780         for_each_plane_id_on_crtc(crtc, plane_id) {
 781                 /*
 782                  * We assume cursors are small enough
 783                  * to not not cause bandwidth problems.
 784                  */
 785                 if (plane_id == PLANE_CURSOR)
 786                         continue;
 787
 788                 data_rate += crtc_state->data_rate[plane_id];
 789
 790                 if (DISPLAY_VER(i915) < 11)
 791                         data_rate += crtc_state->data_rate_y[plane_id];
 792         }
 793
 794         return data_rate;
 795 }
 796
 797 /* "Maximum Pipe Read Bandwidth" */
 798 static int intel_bw_crtc_min_cdclk(const struct intel_crtc_state *crtc_state)
 799 {
 800         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 801         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 802
 803         if (DISPLAY_VER(i915) < 12)
 804                 return 0;
 805
 806         return DIV_ROUND_UP_ULL(mul_u32_u32(intel_bw_crtc_data_rate(crtc_state), 10), 512);
 807 }
 808
 809 void intel_bw_crtc_update(struct intel_bw_state *bw_state,
 810                           const struct intel_crtc_state *crtc_state)
 811 {
 812         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 813         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 814
 815         bw_state->data_rate[crtc->pipe] =
 816                 intel_bw_crtc_data_rate(crtc_state);
 817         bw_state->num_active_planes[crtc->pipe] =
 818                 intel_bw_crtc_num_active_planes(crtc_state);
 819         bw_state->force_check_qgv = true;
 820
 821         drm_dbg_kms(&i915->drm, "pipe %c data rate %u num active planes %u\n",
 822                     pipe_name(crtc->pipe),
 823                     bw_state->data_rate[crtc->pipe],
 824                     bw_state->num_active_planes[crtc->pipe]);
 825 }
 826
 827 static unsigned int intel_bw_num_active_planes(struct drm_i915_private *dev_priv,
 828                                                const struct intel_bw_state *bw_state)
 829 {
 830         unsigned int num_active_planes = 0;
 831         enum pipe pipe;
 832
 833         for_each_pipe(dev_priv, pipe)
 834                 num_active_planes += bw_state->num_active_planes[pipe];
 835
 836         return num_active_planes;
 837 }
 838
 839 static unsigned int intel_bw_data_rate(struct drm_i915_private *dev_priv,
 840                                        const struct intel_bw_state *bw_state)
 841 {
 842         unsigned int data_rate = 0;
 843         enum pipe pipe;
 844
 845         for_each_pipe(dev_priv, pipe)
 846                 data_rate += bw_state->data_rate[pipe];
 847
 848         if (DISPLAY_VER(dev_priv) >= 13 && i915_vtd_active(dev_priv))
 849                 data_rate = DIV_ROUND_UP(data_rate * 105, 100);
 850
 851         return data_rate;
 852 }
 853
 854 struct intel_bw_state *
 855 intel_atomic_get_old_bw_state(struct intel_atomic_state *state)
 856 {
 857         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 858         struct intel_global_state *bw_state;
 859
 860         bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->display.bw.obj);
 861
 862         return to_intel_bw_state(bw_state);
 863 }
 864
 865 struct intel_bw_state *
 866 intel_atomic_get_new_bw_state(struct intel_atomic_state *state)
 867 {
 868         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 869         struct intel_global_state *bw_state;
 870
 871         bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->display.bw.obj);
 872
 873         return to_intel_bw_state(bw_state);
 874 }
 875
 876 struct intel_bw_state *
 877 intel_atomic_get_bw_state(struct intel_atomic_state *state)
 878 {
 879         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 880         struct intel_global_state *bw_state;
 881
 882         bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.bw.obj);
 883         if (IS_ERR(bw_state))
 884                 return ERR_CAST(bw_state);
 885
 886         return to_intel_bw_state(bw_state);
 887 }
 888
 889 static unsigned int icl_max_bw_qgv_point_mask(struct drm_i915_private *i915,
 890                                               int num_active_planes)
 891 {
 892         unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
 893         unsigned int max_bw_point = 0;
 894         unsigned int max_bw = 0;
 895         int i;
 896
 897         for (i = 0; i < num_qgv_points; i++) {
 898                 unsigned int max_data_rate =
 899                         icl_qgv_bw(i915, num_active_planes, i);
 900
 901                 /*
 902                  * We need to know which qgv point gives us
 903                  * maximum bandwidth in order to disable SAGV
 904                  * if we find that we exceed SAGV block time
 905                  * with watermarks. By that moment we already
 906                  * have those, as it is calculated earlier in
 907                  * intel_atomic_check,
 908                  */
 909                 if (max_data_rate > max_bw) {
 910                         max_bw_point = BIT(i);
 911                         max_bw = max_data_rate;
 912                 }
 913         }
 914
 915         return max_bw_point;
 916 }
 917
 918 static u16 icl_prepare_qgv_points_mask(struct drm_i915_private *i915,
 919                                        unsigned int qgv_points,
 920                                        unsigned int psf_points)
 921 {
 922         return ~(ICL_PCODE_REQ_QGV_PT(qgv_points) |
 923                  ADLS_PCODE_REQ_PSF_PT(psf_points)) & icl_qgv_points_mask(i915);
 924 }
 925
 926 static unsigned int icl_max_bw_psf_gv_point_mask(struct drm_i915_private *i915)
 927 {
 928         unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
 929         unsigned int max_bw_point_mask = 0;
 930         unsigned int max_bw = 0;
 931         int i;
 932
 933         for (i = 0; i < num_psf_gv_points; i++) {
 934                 unsigned int max_data_rate = adl_psf_bw(i915, i);
 935
 936                 if (max_data_rate > max_bw) {
 937                         max_bw_point_mask = BIT(i);
 938                         max_bw = max_data_rate;
 939                 } else if (max_data_rate == max_bw) {
 940                         max_bw_point_mask |= BIT(i);
 941                 }
 942         }
 943
 944         return max_bw_point_mask;
 945 }
 946
 947 static void icl_force_disable_sagv(struct drm_i915_private *i915,
 948                                    struct intel_bw_state *bw_state)
 949 {
 950         unsigned int qgv_points = icl_max_bw_qgv_point_mask(i915, 0);
 951         unsigned int psf_points = icl_max_bw_psf_gv_point_mask(i915);
 952
 953         bw_state->qgv_points_mask = icl_prepare_qgv_points_mask(i915,
 954                                                                 qgv_points,
 955                                                                 psf_points);
 956
 957         drm_dbg_kms(&i915->drm, "Forcing SAGV disable: mask 0x%x\n",
 958                     bw_state->qgv_points_mask);
 959
 960         icl_pcode_restrict_qgv_points(i915, bw_state->qgv_points_mask);
 961 }
 962
 963 static int mtl_find_qgv_points(struct drm_i915_private *i915,
 964                                unsigned int data_rate,
 965                                unsigned int num_active_planes,
 966                                struct intel_bw_state *new_bw_state)
 967 {
 968         unsigned int best_rate = UINT_MAX;
 969         unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
 970         unsigned int qgv_peak_bw  = 0;
 971         int i;
 972         int ret;
 973
 974         ret = intel_atomic_lock_global_state(&new_bw_state->base);
 975         if (ret)
 976                 return ret;
 977
 978         /*
 979          * If SAGV cannot be enabled, disable the pcode SAGV by passing all 1's
 980          * for qgv peak bw in PM Demand request. So assign UINT_MAX if SAGV is
 981          * not enabled. PM Demand code will clamp the value for the register
 982          */
 983         if (!intel_can_enable_sagv(i915, new_bw_state)) {
 984                 new_bw_state->qgv_point_peakbw = U16_MAX;
 985                 drm_dbg_kms(&i915->drm, "No SAGV, use UINT_MAX as peak bw.");
 986                 return 0;
 987         }
 988
 989         /*
 990          * Find the best QGV point by comparing the data_rate with max data rate
 991          * offered per plane group
 992          */
 993         for (i = 0; i < num_qgv_points; i++) {
 994                 unsigned int bw_index =
 995                         tgl_max_bw_index(i915, num_active_planes, i);
 996                 unsigned int max_data_rate;
 997
 998                 if (bw_index >= ARRAY_SIZE(i915->display.bw.max))
 999                         continue;
1000
1001                 max_data_rate = i915->display.bw.max[bw_index].deratedbw[i];
1002
1003                 if (max_data_rate < data_rate)
1004                         continue;
1005
1006                 if (max_data_rate - data_rate < best_rate) {
1007                         best_rate = max_data_rate - data_rate;
1008                         qgv_peak_bw = i915->display.bw.max[bw_index].peakbw[i];
1009                 }
1010
1011                 drm_dbg_kms(&i915->drm, "QGV point %d: max bw %d required %d qgv_peak_bw: %d\n",
1012                             i, max_data_rate, data_rate, qgv_peak_bw);
1013         }
1014
1015         drm_dbg_kms(&i915->drm, "Matching peaks QGV bw: %d for required data rate: %d\n",
1016                     qgv_peak_bw, data_rate);
1017
1018         /*
1019          * The display configuration cannot be supported if no QGV point
1020          * satisfying the required data rate is found
1021          */
1022         if (qgv_peak_bw == 0) {
1023                 drm_dbg_kms(&i915->drm, "No QGV points for bw %d for display configuration(%d active planes).\n",
1024                             data_rate, num_active_planes);
1025                 return -EINVAL;
1026         }
1027
1028         /* MTL PM DEMAND expects QGV BW parameter in multiples of 100 mbps */
1029         new_bw_state->qgv_point_peakbw = DIV_ROUND_CLOSEST(qgv_peak_bw, 100);
1030
1031         return 0;
1032 }
1033
1034 static int icl_find_qgv_points(struct drm_i915_private *i915,
1035                                unsigned int data_rate,
1036                                unsigned int num_active_planes,
1037                                const struct intel_bw_state *old_bw_state,
1038                                struct intel_bw_state *new_bw_state)
1039 {
1040         unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
1041         unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
1042         u16 psf_points = 0;
1043         u16 qgv_points = 0;
1044         int i;
1045         int ret;
1046
1047         ret = intel_atomic_lock_global_state(&new_bw_state->base);
1048         if (ret)
1049                 return ret;
1050
1051         for (i = 0; i < num_qgv_points; i++) {
1052                 unsigned int max_data_rate = icl_qgv_bw(i915,
1053                                                         num_active_planes, i);
1054                 if (max_data_rate >= data_rate)
1055                         qgv_points |= BIT(i);
1056
1057                 drm_dbg_kms(&i915->drm, "QGV point %d: max bw %d required %d\n",
1058                             i, max_data_rate, data_rate);
1059         }
1060
1061         for (i = 0; i < num_psf_gv_points; i++) {
1062                 unsigned int max_data_rate = adl_psf_bw(i915, i);
1063
1064                 if (max_data_rate >= data_rate)
1065                         psf_points |= BIT(i);
1066
1067                 drm_dbg_kms(&i915->drm, "PSF GV point %d: max bw %d"
1068                             " required %d\n",
1069                             i, max_data_rate, data_rate);
1070         }
1071
1072         /*
1073          * BSpec states that we always should have at least one allowed point
1074          * left, so if we couldn't - simply reject the configuration for obvious
1075          * reasons.
1076          */
1077         if (qgv_points == 0) {
1078                 drm_dbg_kms(&i915->drm, "No QGV points provide sufficient memory"
1079                             " bandwidth %d for display configuration(%d active planes).\n",
1080                             data_rate, num_active_planes);
1081                 return -EINVAL;
1082         }
1083
1084         if (num_psf_gv_points > 0 && psf_points == 0) {
1085                 drm_dbg_kms(&i915->drm, "No PSF GV points provide sufficient memory"
1086                             " bandwidth %d for display configuration(%d active planes).\n",
1087                             data_rate, num_active_planes);
1088                 return -EINVAL;
1089         }
1090
1091         /*
1092          * Leave only single point with highest bandwidth, if
1093          * we can't enable SAGV due to the increased memory latency it may
1094          * cause.
1095          */
1096         if (!intel_can_enable_sagv(i915, new_bw_state)) {
1097                 qgv_points = icl_max_bw_qgv_point_mask(i915, num_active_planes);
1098                 drm_dbg_kms(&i915->drm, "No SAGV, using single QGV point mask 0x%x\n",
1099                             qgv_points);
1100         }
1101
1102         /*
1103          * We store the ones which need to be masked as that is what PCode
1104          * actually accepts as a parameter.
1105          */
1106         new_bw_state->qgv_points_mask = icl_prepare_qgv_points_mask(i915,
1107                                                                     qgv_points,
1108                                                                     psf_points);
1109         /*
1110          * If the actual mask had changed we need to make sure that
1111          * the commits are serialized(in case this is a nomodeset, nonblocking)
1112          */
1113         if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) {
1114                 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
1115                 if (ret)
1116                         return ret;
1117         }
1118
1119         return 0;
1120 }
1121
1122 static int intel_bw_check_qgv_points(struct drm_i915_private *i915,
1123                                      const struct intel_bw_state *old_bw_state,
1124                                      struct intel_bw_state *new_bw_state)
1125 {
1126         unsigned int data_rate = intel_bw_data_rate(i915, new_bw_state);
1127         unsigned int num_active_planes =
1128                         intel_bw_num_active_planes(i915, new_bw_state);
1129
1130         data_rate = DIV_ROUND_UP(data_rate, 1000);
1131
1132         if (DISPLAY_VER(i915) >= 14)
1133                 return mtl_find_qgv_points(i915, data_rate, num_active_planes,
1134                                            new_bw_state);
1135         else
1136                 return icl_find_qgv_points(i915, data_rate, num_active_planes,
1137                                            old_bw_state, new_bw_state);
1138 }
1139
1140 static bool intel_bw_state_changed(struct drm_i915_private *i915,
1141                                    const struct intel_bw_state *old_bw_state,
1142                                    const struct intel_bw_state *new_bw_state)
1143 {
1144         enum pipe pipe;
1145
1146         for_each_pipe(i915, pipe) {
1147                 const struct intel_dbuf_bw *old_crtc_bw =
1148                         &old_bw_state->dbuf_bw[pipe];
1149                 const struct intel_dbuf_bw *new_crtc_bw =
1150                         &new_bw_state->dbuf_bw[pipe];
1151                 enum dbuf_slice slice;
1152
1153                 for_each_dbuf_slice(i915, slice) {
1154                         if (old_crtc_bw->max_bw[slice] != new_crtc_bw->max_bw[slice] ||
1155                             old_crtc_bw->active_planes[slice] != new_crtc_bw->active_planes[slice])
1156                                 return true;
1157                 }
1158
1159                 if (old_bw_state->min_cdclk[pipe] != new_bw_state->min_cdclk[pipe])
1160                         return true;
1161         }
1162
1163         return false;
1164 }
1165
1166 static void skl_plane_calc_dbuf_bw(struct intel_bw_state *bw_state,
1167                                    struct intel_crtc *crtc,
1168                                    enum plane_id plane_id,
1169                                    const struct skl_ddb_entry *ddb,
1170                                    unsigned int data_rate)
1171 {
1172         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1173         struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
1174         unsigned int dbuf_mask = skl_ddb_dbuf_slice_mask(i915, ddb);
1175         enum dbuf_slice slice;
1176
1177         /*
1178          * The arbiter can only really guarantee an
1179          * equal share of the total bw to each plane.
1180          */
1181         for_each_dbuf_slice_in_mask(i915, slice, dbuf_mask) {
1182                 crtc_bw->max_bw[slice] = max(crtc_bw->max_bw[slice], data_rate);
1183                 crtc_bw->active_planes[slice] |= BIT(plane_id);
1184         }
1185 }
1186
1187 static void skl_crtc_calc_dbuf_bw(struct intel_bw_state *bw_state,
1188                                   const struct intel_crtc_state *crtc_state)
1189 {
1190         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1191         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1192         struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
1193         enum plane_id plane_id;
1194
1195         memset(crtc_bw, 0, sizeof(*crtc_bw));
1196
1197         if (!crtc_state->hw.active)
1198                 return;
1199
1200         for_each_plane_id_on_crtc(crtc, plane_id) {
1201                 /*
1202                  * We assume cursors are small enough
1203                  * to not cause bandwidth problems.
1204                  */
1205                 if (plane_id == PLANE_CURSOR)
1206                         continue;
1207
1208                 skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
1209                                        &crtc_state->wm.skl.plane_ddb[plane_id],
1210                                        crtc_state->data_rate[plane_id]);
1211
1212                 if (DISPLAY_VER(i915) < 11)
1213                         skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
1214                                                &crtc_state->wm.skl.plane_ddb_y[plane_id],
1215                                                crtc_state->data_rate[plane_id]);
1216         }
1217 }
1218
1219 /* "Maximum Data Buffer Bandwidth" */
1220 static int
1221 intel_bw_dbuf_min_cdclk(struct drm_i915_private *i915,
1222                         const struct intel_bw_state *bw_state)
1223 {
1224         unsigned int total_max_bw = 0;
1225         enum dbuf_slice slice;
1226
1227         for_each_dbuf_slice(i915, slice) {
1228                 int num_active_planes = 0;
1229                 unsigned int max_bw = 0;
1230                 enum pipe pipe;
1231
1232                 /*
1233                  * The arbiter can only really guarantee an
1234                  * equal share of the total bw to each plane.
1235                  */
1236                 for_each_pipe(i915, pipe) {
1237                         const struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[pipe];
1238
1239                         max_bw = max(crtc_bw->max_bw[slice], max_bw);
1240                         num_active_planes += hweight8(crtc_bw->active_planes[slice]);
1241                 }
1242                 max_bw *= num_active_planes;
1243
1244                 total_max_bw = max(total_max_bw, max_bw);
1245         }
1246
1247         return DIV_ROUND_UP(total_max_bw, 64);
1248 }
1249
1250 int intel_bw_min_cdclk(struct drm_i915_private *i915,
1251                        const struct intel_bw_state *bw_state)
1252 {
1253         enum pipe pipe;
1254         int min_cdclk;
1255
1256         min_cdclk = intel_bw_dbuf_min_cdclk(i915, bw_state);
1257
1258         for_each_pipe(i915, pipe)
1259                 min_cdclk = max(min_cdclk, bw_state->min_cdclk[pipe]);
1260
1261         return min_cdclk;
1262 }
1263
1264 int intel_bw_calc_min_cdclk(struct intel_atomic_state *state,
1265                             bool *need_cdclk_calc)
1266 {
1267         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
1268         struct intel_bw_state *new_bw_state = NULL;
1269         const struct intel_bw_state *old_bw_state = NULL;
1270         const struct intel_cdclk_state *cdclk_state;
1271         const struct intel_crtc_state *crtc_state;
1272         int old_min_cdclk, new_min_cdclk;
1273         struct intel_crtc *crtc;
1274         int i;
1275
1276         if (DISPLAY_VER(dev_priv) < 9)
1277                 return 0;
1278
1279         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
1280                 new_bw_state = intel_atomic_get_bw_state(state);
1281                 if (IS_ERR(new_bw_state))
1282                         return PTR_ERR(new_bw_state);
1283
1284                 old_bw_state = intel_atomic_get_old_bw_state(state);
1285
1286                 skl_crtc_calc_dbuf_bw(new_bw_state, crtc_state);
1287
1288                 new_bw_state->min_cdclk[crtc->pipe] =
1289                         intel_bw_crtc_min_cdclk(crtc_state);
1290         }
1291
1292         if (!old_bw_state)
1293                 return 0;
1294
1295         if (intel_bw_state_changed(dev_priv, old_bw_state, new_bw_state)) {
1296                 int ret = intel_atomic_lock_global_state(&new_bw_state->base);
1297                 if (ret)
1298                         return ret;
1299         }
1300
1301         old_min_cdclk = intel_bw_min_cdclk(dev_priv, old_bw_state);
1302         new_min_cdclk = intel_bw_min_cdclk(dev_priv, new_bw_state);
1303
1304         /*
1305          * No need to check against the cdclk state if
1306          * the min cdclk doesn't increase.
1307          *
1308          * Ie. we only ever increase the cdclk due to bandwidth
1309          * requirements. This can reduce back and forth
1310          * display blinking due to constant cdclk changes.
1311          */
1312         if (new_min_cdclk <= old_min_cdclk)
1313                 return 0;
1314
1315         cdclk_state = intel_atomic_get_cdclk_state(state);
1316         if (IS_ERR(cdclk_state))
1317                 return PTR_ERR(cdclk_state);
1318
1319         /*
1320          * No need to recalculate the cdclk state if
1321          * the min cdclk doesn't increase.
1322          *
1323          * Ie. we only ever increase the cdclk due to bandwidth
1324          * requirements. This can reduce back and forth
1325          * display blinking due to constant cdclk changes.
1326          */
1327         if (new_min_cdclk <= cdclk_state->bw_min_cdclk)
1328                 return 0;
1329
1330         drm_dbg_kms(&dev_priv->drm,
1331                     "new bandwidth min cdclk (%d kHz) > old min cdclk (%d kHz)\n",
1332                     new_min_cdclk, cdclk_state->bw_min_cdclk);
1333         *need_cdclk_calc = true;
1334
1335         return 0;
1336 }
1337
1338 static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed)
1339 {
1340         struct drm_i915_private *i915 = to_i915(state->base.dev);
1341         const struct intel_crtc_state *new_crtc_state, *old_crtc_state;
1342         struct intel_crtc *crtc;
1343         int i;
1344
1345         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
1346                                             new_crtc_state, i) {
1347                 unsigned int old_data_rate =
1348                         intel_bw_crtc_data_rate(old_crtc_state);
1349                 unsigned int new_data_rate =
1350                         intel_bw_crtc_data_rate(new_crtc_state);
1351                 unsigned int old_active_planes =
1352                         intel_bw_crtc_num_active_planes(old_crtc_state);
1353                 unsigned int new_active_planes =
1354                         intel_bw_crtc_num_active_planes(new_crtc_state);
1355                 struct intel_bw_state *new_bw_state;
1356
1357                 /*
1358                  * Avoid locking the bw state when
1359                  * nothing significant has changed.
1360                  */
1361                 if (old_data_rate == new_data_rate &&
1362                     old_active_planes == new_active_planes)
1363                         continue;
1364
1365                 new_bw_state = intel_atomic_get_bw_state(state);
1366                 if (IS_ERR(new_bw_state))
1367                         return PTR_ERR(new_bw_state);
1368
1369                 new_bw_state->data_rate[crtc->pipe] = new_data_rate;
1370                 new_bw_state->num_active_planes[crtc->pipe] = new_active_planes;
1371
1372                 *changed = true;
1373
1374                 drm_dbg_kms(&i915->drm,
1375                             "[CRTC:%d:%s] data rate %u num active planes %u\n",
1376                             crtc->base.base.id, crtc->base.name,
1377                             new_bw_state->data_rate[crtc->pipe],
1378                             new_bw_state->num_active_planes[crtc->pipe]);
1379         }
1380
1381         return 0;
1382 }
1383
1384 int intel_bw_atomic_check(struct intel_atomic_state *state)
1385 {
1386         bool changed = false;
1387         struct drm_i915_private *i915 = to_i915(state->base.dev);
1388         struct intel_bw_state *new_bw_state;
1389         const struct intel_bw_state *old_bw_state;
1390         int ret;
1391
1392         /* FIXME earlier gens need some checks too */
1393         if (DISPLAY_VER(i915) < 11)
1394                 return 0;
1395
1396         ret = intel_bw_check_data_rate(state, &changed);
1397         if (ret)
1398                 return ret;
1399
1400         old_bw_state = intel_atomic_get_old_bw_state(state);
1401         new_bw_state = intel_atomic_get_new_bw_state(state);
1402
1403         if (new_bw_state &&
1404             (intel_can_enable_sagv(i915, old_bw_state) !=
1405              intel_can_enable_sagv(i915, new_bw_state) ||
1406              new_bw_state->force_check_qgv))
1407                 changed = true;
1408
1409         /*
1410          * If none of our inputs (data rates, number of active
1411          * planes, SAGV yes/no) changed then nothing to do here.
1412          */
1413         if (!changed)
1414                 return 0;
1415
1416         ret = intel_bw_check_qgv_points(i915, old_bw_state, new_bw_state);
1417         if (ret)
1418                 return ret;
1419
1420         new_bw_state->force_check_qgv = false;
1421
1422         return 0;
1423 }
1424
1425 static struct intel_global_state *
1426 intel_bw_duplicate_state(struct intel_global_obj *obj)
1427 {
1428         struct intel_bw_state *state;
1429
1430         state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
1431         if (!state)
1432                 return NULL;
1433
1434         return &state->base;
1435 }
1436
1437 static void intel_bw_destroy_state(struct intel_global_obj *obj,
1438                                    struct intel_global_state *state)
1439 {
1440         kfree(state);
1441 }
1442
1443 static const struct intel_global_state_funcs intel_bw_funcs = {
1444         .atomic_duplicate_state = intel_bw_duplicate_state,
1445         .atomic_destroy_state = intel_bw_destroy_state,
1446 };
1447
1448 int intel_bw_init(struct drm_i915_private *i915)
1449 {
1450         struct intel_display *display = &i915->display;
1451         struct intel_bw_state *state;
1452
1453         state = kzalloc(sizeof(*state), GFP_KERNEL);
1454         if (!state)
1455                 return -ENOMEM;
1456
1457         intel_atomic_global_obj_init(display, &display->bw.obj,
1458                                      &state->base, &intel_bw_funcs);
1459
1460         /*
1461          * Limit this only if we have SAGV. And for Display version 14 onwards
1462          * sagv is handled though pmdemand requests
1463          */
1464         if (intel_has_sagv(i915) && IS_DISPLAY_VER(i915, 11, 13))
1465                 icl_force_disable_sagv(i915, state);
1466
1467         return 0;
1468 }