drivers/gpu/drm/i915/display/skl_watermark.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2022 Intel Corporation
   4  */
   5
   6 #include <drm/drm_blend.h>
   7
   8 #include "i915_drv.h"
   9 #include "i915_fixed.h"
  10 #include "i915_reg.h"
  11 #include "i9xx_wm.h"
  12 #include "intel_atomic.h"
  13 #include "intel_atomic_plane.h"
  14 #include "intel_bw.h"
  15 #include "intel_crtc.h"
  16 #include "intel_de.h"
  17 #include "intel_display.h"
  18 #include "intel_display_power.h"
  19 #include "intel_display_types.h"
  20 #include "intel_fb.h"
  21 #include "intel_pcode.h"
  22 #include "intel_wm.h"
  23 #include "skl_watermark.h"
  24 #include "skl_watermark_regs.h"
  25
  26 static void skl_sagv_disable(struct drm_i915_private *i915);
  27
  28 /* Stores plane specific WM parameters */
  29 struct skl_wm_params {
  30         bool x_tiled, y_tiled;
  31         bool rc_surface;
  32         bool is_planar;
  33         u32 width;
  34         u8 cpp;
  35         u32 plane_pixel_rate;
  36         u32 y_min_scanlines;
  37         u32 plane_bytes_per_line;
  38         uint_fixed_16_16_t plane_blocks_per_line;
  39         uint_fixed_16_16_t y_tile_minimum;
  40         u32 linetime_us;
  41         u32 dbuf_block_size;
  42 };
  43
  44 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *i915)
  45 {
  46         u8 enabled_slices = 0;
  47         enum dbuf_slice slice;
  48
  49         for_each_dbuf_slice(i915, slice) {
  50                 if (intel_de_read(i915, DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
  51                         enabled_slices |= BIT(slice);
  52         }
  53
  54         return enabled_slices;
  55 }
  56
  57 /*
  58  * FIXME: We still don't have the proper code detect if we need to apply the WA,
  59  * so assume we'll always need it in order to avoid underruns.
  60  */
  61 static bool skl_needs_memory_bw_wa(struct drm_i915_private *i915)
  62 {
  63         return DISPLAY_VER(i915) == 9;
  64 }
  65
  66 static bool
  67 intel_has_sagv(struct drm_i915_private *i915)
  68 {
  69         return HAS_SAGV(i915) &&
  70                 i915->display.sagv.status != I915_SAGV_NOT_CONTROLLED;
  71 }
  72
  73 static u32
  74 intel_sagv_block_time(struct drm_i915_private *i915)
  75 {
  76         if (DISPLAY_VER(i915) >= 14) {
  77                 u32 val;
  78
  79                 val = intel_de_read(i915, MTL_LATENCY_SAGV);
  80
  81                 return REG_FIELD_GET(MTL_LATENCY_QCLK_SAGV, val);
  82         } else if (DISPLAY_VER(i915) >= 12) {
  83                 u32 val = 0;
  84                 int ret;
  85
  86                 ret = snb_pcode_read(&i915->uncore,
  87                                      GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
  88                                      &val, NULL);
  89                 if (ret) {
  90                         drm_dbg_kms(&i915->drm, "Couldn't read SAGV block time!\n");
  91                         return 0;
  92                 }
  93
  94                 return val;
  95         } else if (DISPLAY_VER(i915) == 11) {
  96                 return 10;
  97         } else if (HAS_SAGV(i915)) {
  98                 return 30;
  99         } else {
 100                 return 0;
 101         }
 102 }
 103
 104 static void intel_sagv_init(struct drm_i915_private *i915)
 105 {
 106         if (!HAS_SAGV(i915))
 107                 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 108
 109         /*
 110          * Probe to see if we have working SAGV control.
 111          * For icl+ this was already determined by intel_bw_init_hw().
 112          */
 113         if (DISPLAY_VER(i915) < 11)
 114                 skl_sagv_disable(i915);
 115
 116         drm_WARN_ON(&i915->drm, i915->display.sagv.status == I915_SAGV_UNKNOWN);
 117
 118         i915->display.sagv.block_time_us = intel_sagv_block_time(i915);
 119
 120         drm_dbg_kms(&i915->drm, "SAGV supported: %s, original SAGV block time: %u us\n",
 121                     str_yes_no(intel_has_sagv(i915)), i915->display.sagv.block_time_us);
 122
 123         /* avoid overflow when adding with wm0 latency/etc. */
 124         if (drm_WARN(&i915->drm, i915->display.sagv.block_time_us > U16_MAX,
 125                      "Excessive SAGV block time %u, ignoring\n",
 126                      i915->display.sagv.block_time_us))
 127                 i915->display.sagv.block_time_us = 0;
 128
 129         if (!intel_has_sagv(i915))
 130                 i915->display.sagv.block_time_us = 0;
 131 }
 132
 133 /*
 134  * SAGV dynamically adjusts the system agent voltage and clock frequencies
 135  * depending on power and performance requirements. The display engine access
 136  * to system memory is blocked during the adjustment time. Because of the
 137  * blocking time, having this enabled can cause full system hangs and/or pipe
 138  * underruns if we don't meet all of the following requirements:
 139  *
 140  *  - <= 1 pipe enabled
 141  *  - All planes can enable watermarks for latencies >= SAGV engine block time
 142  *  - We're not using an interlaced display configuration
 143  */
 144 static void skl_sagv_enable(struct drm_i915_private *i915)
 145 {
 146         int ret;
 147
 148         if (!intel_has_sagv(i915))
 149                 return;
 150
 151         if (i915->display.sagv.status == I915_SAGV_ENABLED)
 152                 return;
 153
 154         drm_dbg_kms(&i915->drm, "Enabling SAGV\n");
 155         ret = snb_pcode_write(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
 156                               GEN9_SAGV_ENABLE);
 157
 158         /* We don't need to wait for SAGV when enabling */
 159
 160         /*
 161          * Some skl systems, pre-release machines in particular,
 162          * don't actually have SAGV.
 163          */
 164         if (IS_SKYLAKE(i915) && ret == -ENXIO) {
 165                 drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
 166                 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 167                 return;
 168         } else if (ret < 0) {
 169                 drm_err(&i915->drm, "Failed to enable SAGV\n");
 170                 return;
 171         }
 172
 173         i915->display.sagv.status = I915_SAGV_ENABLED;
 174 }
 175
 176 static void skl_sagv_disable(struct drm_i915_private *i915)
 177 {
 178         int ret;
 179
 180         if (!intel_has_sagv(i915))
 181                 return;
 182
 183         if (i915->display.sagv.status == I915_SAGV_DISABLED)
 184                 return;
 185
 186         drm_dbg_kms(&i915->drm, "Disabling SAGV\n");
 187         /* bspec says to keep retrying for at least 1 ms */
 188         ret = skl_pcode_request(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
 189                                 GEN9_SAGV_DISABLE,
 190                                 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
 191                                 1);
 192         /*
 193          * Some skl systems, pre-release machines in particular,
 194          * don't actually have SAGV.
 195          */
 196         if (IS_SKYLAKE(i915) && ret == -ENXIO) {
 197                 drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
 198                 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 199                 return;
 200         } else if (ret < 0) {
 201                 drm_err(&i915->drm, "Failed to disable SAGV (%d)\n", ret);
 202                 return;
 203         }
 204
 205         i915->display.sagv.status = I915_SAGV_DISABLED;
 206 }
 207
 208 static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
 209 {
 210         struct drm_i915_private *i915 = to_i915(state->base.dev);
 211         const struct intel_bw_state *new_bw_state =
 212                 intel_atomic_get_new_bw_state(state);
 213
 214         if (!new_bw_state)
 215                 return;
 216
 217         if (!intel_can_enable_sagv(i915, new_bw_state))
 218                 skl_sagv_disable(i915);
 219 }
 220
 221 static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
 222 {
 223         struct drm_i915_private *i915 = to_i915(state->base.dev);
 224         const struct intel_bw_state *new_bw_state =
 225                 intel_atomic_get_new_bw_state(state);
 226
 227         if (!new_bw_state)
 228                 return;
 229
 230         if (intel_can_enable_sagv(i915, new_bw_state))
 231                 skl_sagv_enable(i915);
 232 }
 233
 234 static void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
 235 {
 236         struct drm_i915_private *i915 = to_i915(state->base.dev);
 237         const struct intel_bw_state *old_bw_state =
 238                 intel_atomic_get_old_bw_state(state);
 239         const struct intel_bw_state *new_bw_state =
 240                 intel_atomic_get_new_bw_state(state);
 241         u16 old_mask, new_mask;
 242
 243         if (!new_bw_state)
 244                 return;
 245
 246         old_mask = old_bw_state->qgv_points_mask;
 247         new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
 248
 249         if (old_mask == new_mask)
 250                 return;
 251
 252         WARN_ON(!new_bw_state->base.changed);
 253
 254         drm_dbg_kms(&i915->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
 255                     old_mask, new_mask);
 256
 257         /*
 258          * Restrict required qgv points before updating the configuration.
 259          * According to BSpec we can't mask and unmask qgv points at the same
 260          * time. Also masking should be done before updating the configuration
 261          * and unmasking afterwards.
 262          */
 263         icl_pcode_restrict_qgv_points(i915, new_mask);
 264 }
 265
 266 static void icl_sagv_post_plane_update(struct intel_atomic_state *state)
 267 {
 268         struct drm_i915_private *i915 = to_i915(state->base.dev);
 269         const struct intel_bw_state *old_bw_state =
 270                 intel_atomic_get_old_bw_state(state);
 271         const struct intel_bw_state *new_bw_state =
 272                 intel_atomic_get_new_bw_state(state);
 273         u16 old_mask, new_mask;
 274
 275         if (!new_bw_state)
 276                 return;
 277
 278         old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
 279         new_mask = new_bw_state->qgv_points_mask;
 280
 281         if (old_mask == new_mask)
 282                 return;
 283
 284         WARN_ON(!new_bw_state->base.changed);
 285
 286         drm_dbg_kms(&i915->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
 287                     old_mask, new_mask);
 288
 289         /*
 290          * Allow required qgv points after updating the configuration.
 291          * According to BSpec we can't mask and unmask qgv points at the same
 292          * time. Also masking should be done before updating the configuration
 293          * and unmasking afterwards.
 294          */
 295         icl_pcode_restrict_qgv_points(i915, new_mask);
 296 }
 297
 298 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
 299 {
 300         struct drm_i915_private *i915 = to_i915(state->base.dev);
 301
 302         /*
 303          * Just return if we can't control SAGV or don't have it.
 304          * This is different from situation when we have SAGV but just can't
 305          * afford it due to DBuf limitation - in case if SAGV is completely
 306          * disabled in a BIOS, we are not even allowed to send a PCode request,
 307          * as it will throw an error. So have to check it here.
 308          */
 309         if (!intel_has_sagv(i915))
 310                 return;
 311
 312         if (DISPLAY_VER(i915) >= 11)
 313                 icl_sagv_pre_plane_update(state);
 314         else
 315                 skl_sagv_pre_plane_update(state);
 316 }
 317
 318 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
 319 {
 320         struct drm_i915_private *i915 = to_i915(state->base.dev);
 321
 322         /*
 323          * Just return if we can't control SAGV or don't have it.
 324          * This is different from situation when we have SAGV but just can't
 325          * afford it due to DBuf limitation - in case if SAGV is completely
 326          * disabled in a BIOS, we are not even allowed to send a PCode request,
 327          * as it will throw an error. So have to check it here.
 328          */
 329         if (!intel_has_sagv(i915))
 330                 return;
 331
 332         if (DISPLAY_VER(i915) >= 11)
 333                 icl_sagv_post_plane_update(state);
 334         else
 335                 skl_sagv_post_plane_update(state);
 336 }
 337
 338 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 339 {
 340         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 341         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 342         enum plane_id plane_id;
 343         int max_level = INT_MAX;
 344
 345         if (!intel_has_sagv(i915))
 346                 return false;
 347
 348         if (!crtc_state->hw.active)
 349                 return true;
 350
 351         if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
 352                 return false;
 353
 354         for_each_plane_id_on_crtc(crtc, plane_id) {
 355                 const struct skl_plane_wm *wm =
 356                         &crtc_state->wm.skl.optimal.planes[plane_id];
 357                 int level;
 358
 359                 /* Skip this plane if it's not enabled */
 360                 if (!wm->wm[0].enable)
 361                         continue;
 362
 363                 /* Find the highest enabled wm level for this plane */
 364                 for (level = i915->display.wm.num_levels - 1;
 365                      !wm->wm[level].enable; --level)
 366                      { }
 367
 368                 /* Highest common enabled wm level for all planes */
 369                 max_level = min(level, max_level);
 370         }
 371
 372         /* No enabled planes? */
 373         if (max_level == INT_MAX)
 374                 return true;
 375
 376         for_each_plane_id_on_crtc(crtc, plane_id) {
 377                 const struct skl_plane_wm *wm =
 378                         &crtc_state->wm.skl.optimal.planes[plane_id];
 379
 380                 /*
 381                  * All enabled planes must have enabled a common wm level that
 382                  * can tolerate memory latencies higher than sagv_block_time_us
 383                  */
 384                 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
 385                         return false;
 386         }
 387
 388         return true;
 389 }
 390
 391 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 392 {
 393         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 394         enum plane_id plane_id;
 395
 396         if (!crtc_state->hw.active)
 397                 return true;
 398
 399         for_each_plane_id_on_crtc(crtc, plane_id) {
 400                 const struct skl_plane_wm *wm =
 401                         &crtc_state->wm.skl.optimal.planes[plane_id];
 402
 403                 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
 404                         return false;
 405         }
 406
 407         return true;
 408 }
 409
 410 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 411 {
 412         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 413         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 414
 415         if (!i915->params.enable_sagv)
 416                 return false;
 417
 418         if (DISPLAY_VER(i915) >= 12)
 419                 return tgl_crtc_can_enable_sagv(crtc_state);
 420         else
 421                 return skl_crtc_can_enable_sagv(crtc_state);
 422 }
 423
 424 bool intel_can_enable_sagv(struct drm_i915_private *i915,
 425                            const struct intel_bw_state *bw_state)
 426 {
 427         if (DISPLAY_VER(i915) < 11 &&
 428             bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
 429                 return false;
 430
 431         return bw_state->pipe_sagv_reject == 0;
 432 }
 433
 434 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
 435 {
 436         struct drm_i915_private *i915 = to_i915(state->base.dev);
 437         int ret;
 438         struct intel_crtc *crtc;
 439         struct intel_crtc_state *new_crtc_state;
 440         struct intel_bw_state *new_bw_state = NULL;
 441         const struct intel_bw_state *old_bw_state = NULL;
 442         int i;
 443
 444         for_each_new_intel_crtc_in_state(state, crtc,
 445                                          new_crtc_state, i) {
 446                 new_bw_state = intel_atomic_get_bw_state(state);
 447                 if (IS_ERR(new_bw_state))
 448                         return PTR_ERR(new_bw_state);
 449
 450                 old_bw_state = intel_atomic_get_old_bw_state(state);
 451
 452                 if (intel_crtc_can_enable_sagv(new_crtc_state))
 453                         new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
 454                 else
 455                         new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
 456         }
 457
 458         if (!new_bw_state)
 459                 return 0;
 460
 461         new_bw_state->active_pipes =
 462                 intel_calc_active_pipes(state, old_bw_state->active_pipes);
 463
 464         if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
 465                 ret = intel_atomic_lock_global_state(&new_bw_state->base);
 466                 if (ret)
 467                         return ret;
 468         }
 469
 470         if (intel_can_enable_sagv(i915, new_bw_state) !=
 471             intel_can_enable_sagv(i915, old_bw_state)) {
 472                 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
 473                 if (ret)
 474                         return ret;
 475         } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
 476                 ret = intel_atomic_lock_global_state(&new_bw_state->base);
 477                 if (ret)
 478                         return ret;
 479         }
 480
 481         for_each_new_intel_crtc_in_state(state, crtc,
 482                                          new_crtc_state, i) {
 483                 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
 484
 485                 /*
 486                  * We store use_sagv_wm in the crtc state rather than relying on
 487                  * that bw state since we have no convenient way to get at the
 488                  * latter from the plane commit hooks (especially in the legacy
 489                  * cursor case)
 490                  */
 491                 pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(i915) &&
 492                         DISPLAY_VER(i915) >= 12 &&
 493                         intel_can_enable_sagv(i915, new_bw_state);
 494         }
 495
 496         return 0;
 497 }
 498
 499 static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
 500                               u16 start, u16 end)
 501 {
 502         entry->start = start;
 503         entry->end = end;
 504
 505         return end;
 506 }
 507
 508 static int intel_dbuf_slice_size(struct drm_i915_private *i915)
 509 {
 510         return DISPLAY_INFO(i915)->dbuf.size /
 511                 hweight8(DISPLAY_INFO(i915)->dbuf.slice_mask);
 512 }
 513
 514 static void
 515 skl_ddb_entry_for_slices(struct drm_i915_private *i915, u8 slice_mask,
 516                          struct skl_ddb_entry *ddb)
 517 {
 518         int slice_size = intel_dbuf_slice_size(i915);
 519
 520         if (!slice_mask) {
 521                 ddb->start = 0;
 522                 ddb->end = 0;
 523                 return;
 524         }
 525
 526         ddb->start = (ffs(slice_mask) - 1) * slice_size;
 527         ddb->end = fls(slice_mask) * slice_size;
 528
 529         WARN_ON(ddb->start >= ddb->end);
 530         WARN_ON(ddb->end > DISPLAY_INFO(i915)->dbuf.size);
 531 }
 532
 533 static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
 534 {
 535         struct skl_ddb_entry ddb;
 536
 537         if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
 538                 slice_mask = BIT(DBUF_S1);
 539         else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
 540                 slice_mask = BIT(DBUF_S3);
 541
 542         skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
 543
 544         return ddb.start;
 545 }
 546
 547 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *i915,
 548                             const struct skl_ddb_entry *entry)
 549 {
 550         int slice_size = intel_dbuf_slice_size(i915);
 551         enum dbuf_slice start_slice, end_slice;
 552         u8 slice_mask = 0;
 553
 554         if (!skl_ddb_entry_size(entry))
 555                 return 0;
 556
 557         start_slice = entry->start / slice_size;
 558         end_slice = (entry->end - 1) / slice_size;
 559
 560         /*
 561          * Per plane DDB entry can in a really worst case be on multiple slices
 562          * but single entry is anyway contigious.
 563          */
 564         while (start_slice <= end_slice) {
 565                 slice_mask |= BIT(start_slice);
 566                 start_slice++;
 567         }
 568
 569         return slice_mask;
 570 }
 571
 572 static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
 573 {
 574         const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
 575         int hdisplay, vdisplay;
 576
 577         if (!crtc_state->hw.active)
 578                 return 0;
 579
 580         /*
 581          * Watermark/ddb requirement highly depends upon width of the
 582          * framebuffer, So instead of allocating DDB equally among pipes
 583          * distribute DDB based on resolution/width of the display.
 584          */
 585         drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
 586
 587         return hdisplay;
 588 }
 589
 590 static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
 591                                     enum pipe for_pipe,
 592                                     unsigned int *weight_start,
 593                                     unsigned int *weight_end,
 594                                     unsigned int *weight_total)
 595 {
 596         struct drm_i915_private *i915 =
 597                 to_i915(dbuf_state->base.state->base.dev);
 598         enum pipe pipe;
 599
 600         *weight_start = 0;
 601         *weight_end = 0;
 602         *weight_total = 0;
 603
 604         for_each_pipe(i915, pipe) {
 605                 int weight = dbuf_state->weight[pipe];
 606
 607                 /*
 608                  * Do not account pipes using other slice sets
 609                  * luckily as of current BSpec slice sets do not partially
 610                  * intersect(pipes share either same one slice or same slice set
 611                  * i.e no partial intersection), so it is enough to check for
 612                  * equality for now.
 613                  */
 614                 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
 615                         continue;
 616
 617                 *weight_total += weight;
 618                 if (pipe < for_pipe) {
 619                         *weight_start += weight;
 620                         *weight_end += weight;
 621                 } else if (pipe == for_pipe) {
 622                         *weight_end += weight;
 623                 }
 624         }
 625 }
 626
 627 static int
 628 skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
 629 {
 630         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 631         unsigned int weight_total, weight_start, weight_end;
 632         const struct intel_dbuf_state *old_dbuf_state =
 633                 intel_atomic_get_old_dbuf_state(state);
 634         struct intel_dbuf_state *new_dbuf_state =
 635                 intel_atomic_get_new_dbuf_state(state);
 636         struct intel_crtc_state *crtc_state;
 637         struct skl_ddb_entry ddb_slices;
 638         enum pipe pipe = crtc->pipe;
 639         unsigned int mbus_offset = 0;
 640         u32 ddb_range_size;
 641         u32 dbuf_slice_mask;
 642         u32 start, end;
 643         int ret;
 644
 645         if (new_dbuf_state->weight[pipe] == 0) {
 646                 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0);
 647                 goto out;
 648         }
 649
 650         dbuf_slice_mask = new_dbuf_state->slices[pipe];
 651
 652         skl_ddb_entry_for_slices(i915, dbuf_slice_mask, &ddb_slices);
 653         mbus_offset = mbus_ddb_offset(i915, dbuf_slice_mask);
 654         ddb_range_size = skl_ddb_entry_size(&ddb_slices);
 655
 656         intel_crtc_dbuf_weights(new_dbuf_state, pipe,
 657                                 &weight_start, &weight_end, &weight_total);
 658
 659         start = ddb_range_size * weight_start / weight_total;
 660         end = ddb_range_size * weight_end / weight_total;
 661
 662         skl_ddb_entry_init(&new_dbuf_state->ddb[pipe],
 663                            ddb_slices.start - mbus_offset + start,
 664                            ddb_slices.start - mbus_offset + end);
 665
 666 out:
 667         if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
 668             skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
 669                                 &new_dbuf_state->ddb[pipe]))
 670                 return 0;
 671
 672         ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
 673         if (ret)
 674                 return ret;
 675
 676         crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
 677         if (IS_ERR(crtc_state))
 678                 return PTR_ERR(crtc_state);
 679
 680         /*
 681          * Used for checking overlaps, so we need absolute
 682          * offsets instead of MBUS relative offsets.
 683          */
 684         crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
 685         crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
 686
 687         drm_dbg_kms(&i915->drm,
 688                     "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
 689                     crtc->base.base.id, crtc->base.name,
 690                     old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
 691                     old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
 692                     new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
 693                     old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
 694
 695         return 0;
 696 }
 697
 698 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
 699                                  int width, const struct drm_format_info *format,
 700                                  u64 modifier, unsigned int rotation,
 701                                  u32 plane_pixel_rate, struct skl_wm_params *wp,
 702                                  int color_plane);
 703
 704 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
 705                                  struct intel_plane *plane,
 706                                  int level,
 707                                  unsigned int latency,
 708                                  const struct skl_wm_params *wp,
 709                                  const struct skl_wm_level *result_prev,
 710                                  struct skl_wm_level *result /* out */);
 711
 712 static unsigned int skl_wm_latency(struct drm_i915_private *i915, int level,
 713                                    const struct skl_wm_params *wp)
 714 {
 715         unsigned int latency = i915->display.wm.skl_latency[level];
 716
 717         if (latency == 0)
 718                 return 0;
 719
 720         /*
 721          * WaIncreaseLatencyIPCEnabled: kbl,cfl
 722          * Display WA #1141: kbl,cfl
 723          */
 724         if ((IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) &&
 725             skl_watermark_ipc_enabled(i915))
 726                 latency += 4;
 727
 728         if (skl_needs_memory_bw_wa(i915) && wp && wp->x_tiled)
 729                 latency += 15;
 730
 731         return latency;
 732 }
 733
 734 static unsigned int
 735 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
 736                       int num_active)
 737 {
 738         struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
 739         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
 740         struct skl_wm_level wm = {};
 741         int ret, min_ddb_alloc = 0;
 742         struct skl_wm_params wp;
 743         int level;
 744
 745         ret = skl_compute_wm_params(crtc_state, 256,
 746                                     drm_format_info(DRM_FORMAT_ARGB8888),
 747                                     DRM_FORMAT_MOD_LINEAR,
 748                                     DRM_MODE_ROTATE_0,
 749                                     crtc_state->pixel_rate, &wp, 0);
 750         drm_WARN_ON(&i915->drm, ret);
 751
 752         for (level = 0; level < i915->display.wm.num_levels; level++) {
 753                 unsigned int latency = skl_wm_latency(i915, level, &wp);
 754
 755                 skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm);
 756                 if (wm.min_ddb_alloc == U16_MAX)
 757                         break;
 758
 759                 min_ddb_alloc = wm.min_ddb_alloc;
 760         }
 761
 762         return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
 763 }
 764
 765 static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
 766 {
 767         skl_ddb_entry_init(entry,
 768                            REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
 769                            REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
 770         if (entry->end)
 771                 entry->end++;
 772 }
 773
 774 static void
 775 skl_ddb_get_hw_plane_state(struct drm_i915_private *i915,
 776                            const enum pipe pipe,
 777                            const enum plane_id plane_id,
 778                            struct skl_ddb_entry *ddb,
 779                            struct skl_ddb_entry *ddb_y)
 780 {
 781         u32 val;
 782
 783         /* Cursor doesn't support NV12/planar, so no extra calculation needed */
 784         if (plane_id == PLANE_CURSOR) {
 785                 val = intel_de_read(i915, CUR_BUF_CFG(pipe));
 786                 skl_ddb_entry_init_from_hw(ddb, val);
 787                 return;
 788         }
 789
 790         val = intel_de_read(i915, PLANE_BUF_CFG(pipe, plane_id));
 791         skl_ddb_entry_init_from_hw(ddb, val);
 792
 793         if (DISPLAY_VER(i915) >= 11)
 794                 return;
 795
 796         val = intel_de_read(i915, PLANE_NV12_BUF_CFG(pipe, plane_id));
 797         skl_ddb_entry_init_from_hw(ddb_y, val);
 798 }
 799
 800 static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
 801                                       struct skl_ddb_entry *ddb,
 802                                       struct skl_ddb_entry *ddb_y)
 803 {
 804         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 805         enum intel_display_power_domain power_domain;
 806         enum pipe pipe = crtc->pipe;
 807         intel_wakeref_t wakeref;
 808         enum plane_id plane_id;
 809
 810         power_domain = POWER_DOMAIN_PIPE(pipe);
 811         wakeref = intel_display_power_get_if_enabled(i915, power_domain);
 812         if (!wakeref)
 813                 return;
 814
 815         for_each_plane_id_on_crtc(crtc, plane_id)
 816                 skl_ddb_get_hw_plane_state(i915, pipe,
 817                                            plane_id,
 818                                            &ddb[plane_id],
 819                                            &ddb_y[plane_id]);
 820
 821         intel_display_power_put(i915, power_domain, wakeref);
 822 }
 823
 824 struct dbuf_slice_conf_entry {
 825         u8 active_pipes;
 826         u8 dbuf_mask[I915_MAX_PIPES];
 827         bool join_mbus;
 828 };
 829
 830 /*
 831  * Table taken from Bspec 12716
 832  * Pipes do have some preferred DBuf slice affinity,
 833  * plus there are some hardcoded requirements on how
 834  * those should be distributed for multipipe scenarios.
 835  * For more DBuf slices algorithm can get even more messy
 836  * and less readable, so decided to use a table almost
 837  * as is from BSpec itself - that way it is at least easier
 838  * to compare, change and check.
 839  */
 840 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
 841 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
 842 {
 843         {
 844                 .active_pipes = BIT(PIPE_A),
 845                 .dbuf_mask = {
 846                         [PIPE_A] = BIT(DBUF_S1),
 847                 },
 848         },
 849         {
 850                 .active_pipes = BIT(PIPE_B),
 851                 .dbuf_mask = {
 852                         [PIPE_B] = BIT(DBUF_S1),
 853                 },
 854         },
 855         {
 856                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
 857                 .dbuf_mask = {
 858                         [PIPE_A] = BIT(DBUF_S1),
 859                         [PIPE_B] = BIT(DBUF_S2),
 860                 },
 861         },
 862         {
 863                 .active_pipes = BIT(PIPE_C),
 864                 .dbuf_mask = {
 865                         [PIPE_C] = BIT(DBUF_S2),
 866                 },
 867         },
 868         {
 869                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
 870                 .dbuf_mask = {
 871                         [PIPE_A] = BIT(DBUF_S1),
 872                         [PIPE_C] = BIT(DBUF_S2),
 873                 },
 874         },
 875         {
 876                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
 877                 .dbuf_mask = {
 878                         [PIPE_B] = BIT(DBUF_S1),
 879                         [PIPE_C] = BIT(DBUF_S2),
 880                 },
 881         },
 882         {
 883                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
 884                 .dbuf_mask = {
 885                         [PIPE_A] = BIT(DBUF_S1),
 886                         [PIPE_B] = BIT(DBUF_S1),
 887                         [PIPE_C] = BIT(DBUF_S2),
 888                 },
 889         },
 890         {}
 891 };
 892
 893 /*
 894  * Table taken from Bspec 49255
 895  * Pipes do have some preferred DBuf slice affinity,
 896  * plus there are some hardcoded requirements on how
 897  * those should be distributed for multipipe scenarios.
 898  * For more DBuf slices algorithm can get even more messy
 899  * and less readable, so decided to use a table almost
 900  * as is from BSpec itself - that way it is at least easier
 901  * to compare, change and check.
 902  */
 903 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
 904 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
 905 {
 906         {
 907                 .active_pipes = BIT(PIPE_A),
 908                 .dbuf_mask = {
 909                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
 910                 },
 911         },
 912         {
 913                 .active_pipes = BIT(PIPE_B),
 914                 .dbuf_mask = {
 915                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
 916                 },
 917         },
 918         {
 919                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
 920                 .dbuf_mask = {
 921                         [PIPE_A] = BIT(DBUF_S2),
 922                         [PIPE_B] = BIT(DBUF_S1),
 923                 },
 924         },
 925         {
 926                 .active_pipes = BIT(PIPE_C),
 927                 .dbuf_mask = {
 928                         [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
 929                 },
 930         },
 931         {
 932                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
 933                 .dbuf_mask = {
 934                         [PIPE_A] = BIT(DBUF_S1),
 935                         [PIPE_C] = BIT(DBUF_S2),
 936                 },
 937         },
 938         {
 939                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
 940                 .dbuf_mask = {
 941                         [PIPE_B] = BIT(DBUF_S1),
 942                         [PIPE_C] = BIT(DBUF_S2),
 943                 },
 944         },
 945         {
 946                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
 947                 .dbuf_mask = {
 948                         [PIPE_A] = BIT(DBUF_S1),
 949                         [PIPE_B] = BIT(DBUF_S1),
 950                         [PIPE_C] = BIT(DBUF_S2),
 951                 },
 952         },
 953         {
 954                 .active_pipes = BIT(PIPE_D),
 955                 .dbuf_mask = {
 956                         [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
 957                 },
 958         },
 959         {
 960                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
 961                 .dbuf_mask = {
 962                         [PIPE_A] = BIT(DBUF_S1),
 963                         [PIPE_D] = BIT(DBUF_S2),
 964                 },
 965         },
 966         {
 967                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
 968                 .dbuf_mask = {
 969                         [PIPE_B] = BIT(DBUF_S1),
 970                         [PIPE_D] = BIT(DBUF_S2),
 971                 },
 972         },
 973         {
 974                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
 975                 .dbuf_mask = {
 976                         [PIPE_A] = BIT(DBUF_S1),
 977                         [PIPE_B] = BIT(DBUF_S1),
 978                         [PIPE_D] = BIT(DBUF_S2),
 979                 },
 980         },
 981         {
 982                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
 983                 .dbuf_mask = {
 984                         [PIPE_C] = BIT(DBUF_S1),
 985                         [PIPE_D] = BIT(DBUF_S2),
 986                 },
 987         },
 988         {
 989                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
 990                 .dbuf_mask = {
 991                         [PIPE_A] = BIT(DBUF_S1),
 992                         [PIPE_C] = BIT(DBUF_S2),
 993                         [PIPE_D] = BIT(DBUF_S2),
 994                 },
 995         },
 996         {
 997                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
 998                 .dbuf_mask = {
 999                         [PIPE_B] = BIT(DBUF_S1),
1000                         [PIPE_C] = BIT(DBUF_S2),
1001                         [PIPE_D] = BIT(DBUF_S2),
1002                 },
1003         },
1004         {
1005                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1006                 .dbuf_mask = {
1007                         [PIPE_A] = BIT(DBUF_S1),
1008                         [PIPE_B] = BIT(DBUF_S1),
1009                         [PIPE_C] = BIT(DBUF_S2),
1010                         [PIPE_D] = BIT(DBUF_S2),
1011                 },
1012         },
1013         {}
1014 };
1015
1016 static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
1017         {
1018                 .active_pipes = BIT(PIPE_A),
1019                 .dbuf_mask = {
1020                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1021                 },
1022         },
1023         {
1024                 .active_pipes = BIT(PIPE_B),
1025                 .dbuf_mask = {
1026                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1027                 },
1028         },
1029         {
1030                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1031                 .dbuf_mask = {
1032                         [PIPE_A] = BIT(DBUF_S1),
1033                         [PIPE_B] = BIT(DBUF_S2),
1034                 },
1035         },
1036         {
1037                 .active_pipes = BIT(PIPE_C),
1038                 .dbuf_mask = {
1039                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1040                 },
1041         },
1042         {
1043                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1044                 .dbuf_mask = {
1045                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1046                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1047                 },
1048         },
1049         {
1050                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1051                 .dbuf_mask = {
1052                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1053                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1054                 },
1055         },
1056         {
1057                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1058                 .dbuf_mask = {
1059                         [PIPE_A] = BIT(DBUF_S1),
1060                         [PIPE_B] = BIT(DBUF_S2),
1061                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1062                 },
1063         },
1064         {
1065                 .active_pipes = BIT(PIPE_D),
1066                 .dbuf_mask = {
1067                         [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1068                 },
1069         },
1070         {
1071                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1072                 .dbuf_mask = {
1073                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1074                         [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1075                 },
1076         },
1077         {
1078                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1079                 .dbuf_mask = {
1080                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1081                         [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1082                 },
1083         },
1084         {
1085                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1086                 .dbuf_mask = {
1087                         [PIPE_A] = BIT(DBUF_S1),
1088                         [PIPE_B] = BIT(DBUF_S2),
1089                         [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1090                 },
1091         },
1092         {
1093                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1094                 .dbuf_mask = {
1095                         [PIPE_C] = BIT(DBUF_S3),
1096                         [PIPE_D] = BIT(DBUF_S4),
1097                 },
1098         },
1099         {
1100                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1101                 .dbuf_mask = {
1102                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1103                         [PIPE_C] = BIT(DBUF_S3),
1104                         [PIPE_D] = BIT(DBUF_S4),
1105                 },
1106         },
1107         {
1108                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1109                 .dbuf_mask = {
1110                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1111                         [PIPE_C] = BIT(DBUF_S3),
1112                         [PIPE_D] = BIT(DBUF_S4),
1113                 },
1114         },
1115         {
1116                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1117                 .dbuf_mask = {
1118                         [PIPE_A] = BIT(DBUF_S1),
1119                         [PIPE_B] = BIT(DBUF_S2),
1120                         [PIPE_C] = BIT(DBUF_S3),
1121                         [PIPE_D] = BIT(DBUF_S4),
1122                 },
1123         },
1124         {}
1125 };
1126
1127 static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
1128         /*
1129          * Keep the join_mbus cases first so check_mbus_joined()
1130          * will prefer them over the !join_mbus cases.
1131          */
1132         {
1133                 .active_pipes = BIT(PIPE_A),
1134                 .dbuf_mask = {
1135                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1136                 },
1137                 .join_mbus = true,
1138         },
1139         {
1140                 .active_pipes = BIT(PIPE_B),
1141                 .dbuf_mask = {
1142                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1143                 },
1144                 .join_mbus = true,
1145         },
1146         {
1147                 .active_pipes = BIT(PIPE_A),
1148                 .dbuf_mask = {
1149                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1150                 },
1151                 .join_mbus = false,
1152         },
1153         {
1154                 .active_pipes = BIT(PIPE_B),
1155                 .dbuf_mask = {
1156                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1157                 },
1158                 .join_mbus = false,
1159         },
1160         {
1161                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1162                 .dbuf_mask = {
1163                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1164                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1165                 },
1166         },
1167         {
1168                 .active_pipes = BIT(PIPE_C),
1169                 .dbuf_mask = {
1170                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1171                 },
1172         },
1173         {
1174                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1175                 .dbuf_mask = {
1176                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1177                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1178                 },
1179         },
1180         {
1181                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1182                 .dbuf_mask = {
1183                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1184                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1185                 },
1186         },
1187         {
1188                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1189                 .dbuf_mask = {
1190                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1191                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1192                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1193                 },
1194         },
1195         {
1196                 .active_pipes = BIT(PIPE_D),
1197                 .dbuf_mask = {
1198                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1199                 },
1200         },
1201         {
1202                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1203                 .dbuf_mask = {
1204                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1205                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1206                 },
1207         },
1208         {
1209                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1210                 .dbuf_mask = {
1211                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1212                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1213                 },
1214         },
1215         {
1216                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1217                 .dbuf_mask = {
1218                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1219                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1220                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1221                 },
1222         },
1223         {
1224                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1225                 .dbuf_mask = {
1226                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1227                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1228                 },
1229         },
1230         {
1231                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1232                 .dbuf_mask = {
1233                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1234                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1235                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1236                 },
1237         },
1238         {
1239                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1240                 .dbuf_mask = {
1241                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1242                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1243                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1244                 },
1245         },
1246         {
1247                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1248                 .dbuf_mask = {
1249                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1250                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1251                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1252                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1253                 },
1254         },
1255         {}
1256
1257 };
1258
1259 static bool check_mbus_joined(u8 active_pipes,
1260                               const struct dbuf_slice_conf_entry *dbuf_slices)
1261 {
1262         int i;
1263
1264         for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1265                 if (dbuf_slices[i].active_pipes == active_pipes)
1266                         return dbuf_slices[i].join_mbus;
1267         }
1268         return false;
1269 }
1270
1271 static bool adlp_check_mbus_joined(u8 active_pipes)
1272 {
1273         return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
1274 }
1275
1276 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
1277                               const struct dbuf_slice_conf_entry *dbuf_slices)
1278 {
1279         int i;
1280
1281         for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1282                 if (dbuf_slices[i].active_pipes == active_pipes &&
1283                     dbuf_slices[i].join_mbus == join_mbus)
1284                         return dbuf_slices[i].dbuf_mask[pipe];
1285         }
1286         return 0;
1287 }
1288
1289 /*
1290  * This function finds an entry with same enabled pipe configuration and
1291  * returns correspondent DBuf slice mask as stated in BSpec for particular
1292  * platform.
1293  */
1294 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1295 {
1296         /*
1297          * FIXME: For ICL this is still a bit unclear as prev BSpec revision
1298          * required calculating "pipe ratio" in order to determine
1299          * if one or two slices can be used for single pipe configurations
1300          * as additional constraint to the existing table.
1301          * However based on recent info, it should be not "pipe ratio"
1302          * but rather ratio between pixel_rate and cdclk with additional
1303          * constants, so for now we are using only table until this is
1304          * clarified. Also this is the reason why crtc_state param is
1305          * still here - we will need it once those additional constraints
1306          * pop up.
1307          */
1308         return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1309                                    icl_allowed_dbufs);
1310 }
1311
1312 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1313 {
1314         return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1315                                    tgl_allowed_dbufs);
1316 }
1317
1318 static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1319 {
1320         return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1321                                    adlp_allowed_dbufs);
1322 }
1323
1324 static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1325 {
1326         return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1327                                    dg2_allowed_dbufs);
1328 }
1329
1330 static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
1331 {
1332         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1333         enum pipe pipe = crtc->pipe;
1334
1335         if (IS_DG2(i915))
1336                 return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1337         else if (DISPLAY_VER(i915) >= 13)
1338                 return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1339         else if (DISPLAY_VER(i915) == 12)
1340                 return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1341         else if (DISPLAY_VER(i915) == 11)
1342                 return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1343         /*
1344          * For anything else just return one slice yet.
1345          * Should be extended for other platforms.
1346          */
1347         return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
1348 }
1349
1350 static bool
1351 use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
1352                      struct intel_plane *plane)
1353 {
1354         struct drm_i915_private *i915 = to_i915(plane->base.dev);
1355
1356         return DISPLAY_VER(i915) >= 13 &&
1357                crtc_state->uapi.async_flip &&
1358                plane->async_flip;
1359 }
1360
1361 static u64
1362 skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state)
1363 {
1364         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1365         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1366         enum plane_id plane_id;
1367         u64 data_rate = 0;
1368
1369         for_each_plane_id_on_crtc(crtc, plane_id) {
1370                 if (plane_id == PLANE_CURSOR && DISPLAY_VER(i915) < 20)
1371                         continue;
1372
1373                 data_rate += crtc_state->rel_data_rate[plane_id];
1374
1375                 if (DISPLAY_VER(i915) < 11)
1376                         data_rate += crtc_state->rel_data_rate_y[plane_id];
1377         }
1378
1379         return data_rate;
1380 }
1381
1382 static const struct skl_wm_level *
1383 skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
1384                    enum plane_id plane_id,
1385                    int level)
1386 {
1387         const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1388
1389         if (level == 0 && pipe_wm->use_sagv_wm)
1390                 return &wm->sagv.wm0;
1391
1392         return &wm->wm[level];
1393 }
1394
1395 static const struct skl_wm_level *
1396 skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
1397                    enum plane_id plane_id)
1398 {
1399         const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1400
1401         if (pipe_wm->use_sagv_wm)
1402                 return &wm->sagv.trans_wm;
1403
1404         return &wm->trans_wm;
1405 }
1406
1407 /*
1408  * We only disable the watermarks for each plane if
1409  * they exceed the ddb allocation of said plane. This
1410  * is done so that we don't end up touching cursor
1411  * watermarks needlessly when some other plane reduces
1412  * our max possible watermark level.
1413  *
1414  * Bspec has this to say about the PLANE_WM enable bit:
1415  * "All the watermarks at this level for all enabled
1416  *  planes must be enabled before the level will be used."
1417  * So this is actually safe to do.
1418  */
1419 static void
1420 skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb)
1421 {
1422         if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb))
1423                 memset(wm, 0, sizeof(*wm));
1424 }
1425
1426 static void
1427 skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
1428                         const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb)
1429 {
1430         if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb_y) ||
1431             uv_wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1432                 memset(wm, 0, sizeof(*wm));
1433                 memset(uv_wm, 0, sizeof(*uv_wm));
1434         }
1435 }
1436
1437 static bool skl_need_wm_copy_wa(struct drm_i915_private *i915, int level,
1438                                 const struct skl_plane_wm *wm)
1439 {
1440         /*
1441          * Wa_1408961008:icl, ehl
1442          * Wa_14012656716:tgl, adl
1443          * Wa_14017887344:icl
1444          * Wa_14017868169:adl, tgl
1445          * Due to some power saving optimizations, different subsystems
1446          * like PSR, might still use even disabled wm level registers,
1447          * for "reference", so lets keep at least the values sane.
1448          * Considering amount of WA requiring us to do similar things, was
1449          * decided to simply do it for all of the platforms, as those wm
1450          * levels are disabled, this isn't going to do harm anyway.
1451          */
1452         return level > 0 && !wm->wm[level].enable;
1453 }
1454
1455 struct skl_plane_ddb_iter {
1456         u64 data_rate;
1457         u16 start, size;
1458 };
1459
1460 static void
1461 skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
1462                        struct skl_ddb_entry *ddb,
1463                        const struct skl_wm_level *wm,
1464                        u64 data_rate)
1465 {
1466         u16 size, extra = 0;
1467
1468         if (data_rate) {
1469                 extra = min_t(u16, iter->size,
1470                               DIV64_U64_ROUND_UP(iter->size * data_rate,
1471                                                  iter->data_rate));
1472                 iter->size -= extra;
1473                 iter->data_rate -= data_rate;
1474         }
1475
1476         /*
1477          * Keep ddb entry of all disabled planes explicitly zeroed
1478          * to avoid skl_ddb_add_affected_planes() adding them to
1479          * the state when other planes change their allocations.
1480          */
1481         size = wm->min_ddb_alloc + extra;
1482         if (size)
1483                 iter->start = skl_ddb_entry_init(ddb, iter->start,
1484                                                  iter->start + size);
1485 }
1486
1487 static int
1488 skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
1489                             struct intel_crtc *crtc)
1490 {
1491         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1492         struct intel_crtc_state *crtc_state =
1493                 intel_atomic_get_new_crtc_state(state, crtc);
1494         const struct intel_dbuf_state *dbuf_state =
1495                 intel_atomic_get_new_dbuf_state(state);
1496         const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
1497         int num_active = hweight8(dbuf_state->active_pipes);
1498         struct skl_plane_ddb_iter iter;
1499         enum plane_id plane_id;
1500         u16 cursor_size;
1501         u32 blocks;
1502         int level;
1503
1504         /* Clear the partitioning for disabled planes. */
1505         memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb));
1506         memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
1507
1508         if (!crtc_state->hw.active)
1509                 return 0;
1510
1511         iter.start = alloc->start;
1512         iter.size = skl_ddb_entry_size(alloc);
1513         if (iter.size == 0)
1514                 return 0;
1515
1516         /* Allocate fixed number of blocks for cursor. */
1517         if (DISPLAY_VER(i915) < 20) {
1518                 cursor_size = skl_cursor_allocation(crtc_state, num_active);
1519                 iter.size -= cursor_size;
1520                 skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[PLANE_CURSOR],
1521                                    alloc->end - cursor_size, alloc->end);
1522         }
1523
1524         iter.data_rate = skl_total_relative_data_rate(crtc_state);
1525
1526         /*
1527          * Find the highest watermark level for which we can satisfy the block
1528          * requirement of active planes.
1529          */
1530         for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
1531                 blocks = 0;
1532                 for_each_plane_id_on_crtc(crtc, plane_id) {
1533                         const struct skl_plane_wm *wm =
1534                                 &crtc_state->wm.skl.optimal.planes[plane_id];
1535
1536                         if (plane_id == PLANE_CURSOR && DISPLAY_VER(i915) < 20) {
1537                                 const struct skl_ddb_entry *ddb =
1538                                         &crtc_state->wm.skl.plane_ddb[plane_id];
1539
1540                                 if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1541                                         drm_WARN_ON(&i915->drm,
1542                                                     wm->wm[level].min_ddb_alloc != U16_MAX);
1543                                         blocks = U32_MAX;
1544                                         break;
1545                                 }
1546                                 continue;
1547                         }
1548
1549                         blocks += wm->wm[level].min_ddb_alloc;
1550                         blocks += wm->uv_wm[level].min_ddb_alloc;
1551                 }
1552
1553                 if (blocks <= iter.size) {
1554                         iter.size -= blocks;
1555                         break;
1556                 }
1557         }
1558
1559         if (level < 0) {
1560                 drm_dbg_kms(&i915->drm,
1561                             "Requested display configuration exceeds system DDB limitations");
1562                 drm_dbg_kms(&i915->drm, "minimum required %d/%d\n",
1563                             blocks, iter.size);
1564                 return -EINVAL;
1565         }
1566
1567         /* avoid the WARN later when we don't allocate any extra DDB */
1568         if (iter.data_rate == 0)
1569                 iter.size = 0;
1570
1571         /*
1572          * Grant each plane the blocks it requires at the highest achievable
1573          * watermark level, plus an extra share of the leftover blocks
1574          * proportional to its relative data rate.
1575          */
1576         for_each_plane_id_on_crtc(crtc, plane_id) {
1577                 struct skl_ddb_entry *ddb =
1578                         &crtc_state->wm.skl.plane_ddb[plane_id];
1579                 struct skl_ddb_entry *ddb_y =
1580                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
1581                 const struct skl_plane_wm *wm =
1582                         &crtc_state->wm.skl.optimal.planes[plane_id];
1583
1584                 if (plane_id == PLANE_CURSOR && DISPLAY_VER(i915) < 20)
1585                         continue;
1586
1587                 if (DISPLAY_VER(i915) < 11 &&
1588                     crtc_state->nv12_planes & BIT(plane_id)) {
1589                         skl_allocate_plane_ddb(&iter, ddb_y, &wm->wm[level],
1590                                                crtc_state->rel_data_rate_y[plane_id]);
1591                         skl_allocate_plane_ddb(&iter, ddb, &wm->uv_wm[level],
1592                                                crtc_state->rel_data_rate[plane_id]);
1593                 } else {
1594                         skl_allocate_plane_ddb(&iter, ddb, &wm->wm[level],
1595                                                crtc_state->rel_data_rate[plane_id]);
1596                 }
1597         }
1598         drm_WARN_ON(&i915->drm, iter.size != 0 || iter.data_rate != 0);
1599
1600         /*
1601          * When we calculated watermark values we didn't know how high
1602          * of a level we'd actually be able to hit, so we just marked
1603          * all levels as "enabled."  Go back now and disable the ones
1604          * that aren't actually possible.
1605          */
1606         for (level++; level < i915->display.wm.num_levels; level++) {
1607                 for_each_plane_id_on_crtc(crtc, plane_id) {
1608                         const struct skl_ddb_entry *ddb =
1609                                 &crtc_state->wm.skl.plane_ddb[plane_id];
1610                         const struct skl_ddb_entry *ddb_y =
1611                                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
1612                         struct skl_plane_wm *wm =
1613                                 &crtc_state->wm.skl.optimal.planes[plane_id];
1614
1615                         if (DISPLAY_VER(i915) < 11 &&
1616                             crtc_state->nv12_planes & BIT(plane_id))
1617                                 skl_check_nv12_wm_level(&wm->wm[level],
1618                                                         &wm->uv_wm[level],
1619                                                         ddb_y, ddb);
1620                         else
1621                                 skl_check_wm_level(&wm->wm[level], ddb);
1622
1623                         if (skl_need_wm_copy_wa(i915, level, wm)) {
1624                                 wm->wm[level].blocks = wm->wm[level - 1].blocks;
1625                                 wm->wm[level].lines = wm->wm[level - 1].lines;
1626                                 wm->wm[level].ignore_lines = wm->wm[level - 1].ignore_lines;
1627                         }
1628                 }
1629         }
1630
1631         /*
1632          * Go back and disable the transition and SAGV watermarks
1633          * if it turns out we don't have enough DDB blocks for them.
1634          */
1635         for_each_plane_id_on_crtc(crtc, plane_id) {
1636                 const struct skl_ddb_entry *ddb =
1637                         &crtc_state->wm.skl.plane_ddb[plane_id];
1638                 const struct skl_ddb_entry *ddb_y =
1639                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
1640                 struct skl_plane_wm *wm =
1641                         &crtc_state->wm.skl.optimal.planes[plane_id];
1642
1643                 if (DISPLAY_VER(i915) < 11 &&
1644                     crtc_state->nv12_planes & BIT(plane_id)) {
1645                         skl_check_wm_level(&wm->trans_wm, ddb_y);
1646                 } else {
1647                         WARN_ON(skl_ddb_entry_size(ddb_y));
1648
1649                         skl_check_wm_level(&wm->trans_wm, ddb);
1650                 }
1651
1652                 skl_check_wm_level(&wm->sagv.wm0, ddb);
1653                 skl_check_wm_level(&wm->sagv.trans_wm, ddb);
1654         }
1655
1656         return 0;
1657 }
1658
1659 /*
1660  * The max latency should be 257 (max the punit can code is 255 and we add 2us
1661  * for the read latency) and cpp should always be <= 8, so that
1662  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
1663  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
1664  */
1665 static uint_fixed_16_16_t
1666 skl_wm_method1(const struct drm_i915_private *i915, u32 pixel_rate,
1667                u8 cpp, u32 latency, u32 dbuf_block_size)
1668 {
1669         u32 wm_intermediate_val;
1670         uint_fixed_16_16_t ret;
1671
1672         if (latency == 0)
1673                 return FP_16_16_MAX;
1674
1675         wm_intermediate_val = latency * pixel_rate * cpp;
1676         ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
1677
1678         if (DISPLAY_VER(i915) >= 10)
1679                 ret = add_fixed16_u32(ret, 1);
1680
1681         return ret;
1682 }
1683
1684 static uint_fixed_16_16_t
1685 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
1686                uint_fixed_16_16_t plane_blocks_per_line)
1687 {
1688         u32 wm_intermediate_val;
1689         uint_fixed_16_16_t ret;
1690
1691         if (latency == 0)
1692                 return FP_16_16_MAX;
1693
1694         wm_intermediate_val = latency * pixel_rate;
1695         wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
1696                                            pipe_htotal * 1000);
1697         ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
1698         return ret;
1699 }
1700
1701 static uint_fixed_16_16_t
1702 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
1703 {
1704         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1705         u32 pixel_rate;
1706         u32 crtc_htotal;
1707         uint_fixed_16_16_t linetime_us;
1708
1709         if (!crtc_state->hw.active)
1710                 return u32_to_fixed16(0);
1711
1712         pixel_rate = crtc_state->pixel_rate;
1713
1714         if (drm_WARN_ON(&i915->drm, pixel_rate == 0))
1715                 return u32_to_fixed16(0);
1716
1717         crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
1718         linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
1719
1720         return linetime_us;
1721 }
1722
1723 static int
1724 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
1725                       int width, const struct drm_format_info *format,
1726                       u64 modifier, unsigned int rotation,
1727                       u32 plane_pixel_rate, struct skl_wm_params *wp,
1728                       int color_plane)
1729 {
1730         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1731         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1732         u32 interm_pbpl;
1733
1734         /* only planar format has two planes */
1735         if (color_plane == 1 &&
1736             !intel_format_info_is_yuv_semiplanar(format, modifier)) {
1737                 drm_dbg_kms(&i915->drm,
1738                             "Non planar format have single plane\n");
1739                 return -EINVAL;
1740         }
1741
1742         wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
1743         wp->y_tiled = modifier != I915_FORMAT_MOD_X_TILED &&
1744                 intel_fb_is_tiled_modifier(modifier);
1745         wp->rc_surface = intel_fb_is_ccs_modifier(modifier);
1746         wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
1747
1748         wp->width = width;
1749         if (color_plane == 1 && wp->is_planar)
1750                 wp->width /= 2;
1751
1752         wp->cpp = format->cpp[color_plane];
1753         wp->plane_pixel_rate = plane_pixel_rate;
1754
1755         if (DISPLAY_VER(i915) >= 11 &&
1756             modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
1757                 wp->dbuf_block_size = 256;
1758         else
1759                 wp->dbuf_block_size = 512;
1760
1761         if (drm_rotation_90_or_270(rotation)) {
1762                 switch (wp->cpp) {
1763                 case 1:
1764                         wp->y_min_scanlines = 16;
1765                         break;
1766                 case 2:
1767                         wp->y_min_scanlines = 8;
1768                         break;
1769                 case 4:
1770                         wp->y_min_scanlines = 4;
1771                         break;
1772                 default:
1773                         MISSING_CASE(wp->cpp);
1774                         return -EINVAL;
1775                 }
1776         } else {
1777                 wp->y_min_scanlines = 4;
1778         }
1779
1780         if (skl_needs_memory_bw_wa(i915))
1781                 wp->y_min_scanlines *= 2;
1782
1783         wp->plane_bytes_per_line = wp->width * wp->cpp;
1784         if (wp->y_tiled) {
1785                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
1786                                            wp->y_min_scanlines,
1787                                            wp->dbuf_block_size);
1788
1789                 if (DISPLAY_VER(i915) >= 10)
1790                         interm_pbpl++;
1791
1792                 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
1793                                                         wp->y_min_scanlines);
1794         } else {
1795                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
1796                                            wp->dbuf_block_size);
1797
1798                 if (!wp->x_tiled || DISPLAY_VER(i915) >= 10)
1799                         interm_pbpl++;
1800
1801                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
1802         }
1803
1804         wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
1805                                              wp->plane_blocks_per_line);
1806
1807         wp->linetime_us = fixed16_to_u32_round_up(intel_get_linetime_us(crtc_state));
1808
1809         return 0;
1810 }
1811
1812 static int
1813 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
1814                             const struct intel_plane_state *plane_state,
1815                             struct skl_wm_params *wp, int color_plane)
1816 {
1817         const struct drm_framebuffer *fb = plane_state->hw.fb;
1818         int width;
1819
1820         /*
1821          * Src coordinates are already rotated by 270 degrees for
1822          * the 90/270 degree plane rotation cases (to match the
1823          * GTT mapping), hence no need to account for rotation here.
1824          */
1825         width = drm_rect_width(&plane_state->uapi.src) >> 16;
1826
1827         return skl_compute_wm_params(crtc_state, width,
1828                                      fb->format, fb->modifier,
1829                                      plane_state->hw.rotation,
1830                                      intel_plane_pixel_rate(crtc_state, plane_state),
1831                                      wp, color_plane);
1832 }
1833
1834 static bool skl_wm_has_lines(struct drm_i915_private *i915, int level)
1835 {
1836         if (DISPLAY_VER(i915) >= 10)
1837                 return true;
1838
1839         /* The number of lines are ignored for the level 0 watermark. */
1840         return level > 0;
1841 }
1842
1843 static int skl_wm_max_lines(struct drm_i915_private *i915)
1844 {
1845         if (DISPLAY_VER(i915) >= 13)
1846                 return 255;
1847         else
1848                 return 31;
1849 }
1850
1851 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
1852                                  struct intel_plane *plane,
1853                                  int level,
1854                                  unsigned int latency,
1855                                  const struct skl_wm_params *wp,
1856                                  const struct skl_wm_level *result_prev,
1857                                  struct skl_wm_level *result /* out */)
1858 {
1859         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1860         uint_fixed_16_16_t method1, method2;
1861         uint_fixed_16_16_t selected_result;
1862         u32 blocks, lines, min_ddb_alloc = 0;
1863
1864         if (latency == 0 ||
1865             (use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
1866                 /* reject it */
1867                 result->min_ddb_alloc = U16_MAX;
1868                 return;
1869         }
1870
1871         method1 = skl_wm_method1(i915, wp->plane_pixel_rate,
1872                                  wp->cpp, latency, wp->dbuf_block_size);
1873         method2 = skl_wm_method2(wp->plane_pixel_rate,
1874                                  crtc_state->hw.pipe_mode.crtc_htotal,
1875                                  latency,
1876                                  wp->plane_blocks_per_line);
1877
1878         if (wp->y_tiled) {
1879                 selected_result = max_fixed16(method2, wp->y_tile_minimum);
1880         } else {
1881                 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
1882                      wp->dbuf_block_size < 1) &&
1883                      (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
1884                         selected_result = method2;
1885                 } else if (latency >= wp->linetime_us) {
1886                         if (DISPLAY_VER(i915) == 9)
1887                                 selected_result = min_fixed16(method1, method2);
1888                         else
1889                                 selected_result = method2;
1890                 } else {
1891                         selected_result = method1;
1892                 }
1893         }
1894
1895         blocks = fixed16_to_u32_round_up(selected_result) + 1;
1896         /*
1897          * Lets have blocks at minimum equivalent to plane_blocks_per_line
1898          * as there will be at minimum one line for lines configuration. This
1899          * is a work around for FIFO underruns observed with resolutions like
1900          * 4k 60 Hz in single channel DRAM configurations.
1901          *
1902          * As per the Bspec 49325, if the ddb allocation can hold at least
1903          * one plane_blocks_per_line, we should have selected method2 in
1904          * the above logic. Assuming that modern versions have enough dbuf
1905          * and method2 guarantees blocks equivalent to at least 1 line,
1906          * select the blocks as plane_blocks_per_line.
1907          *
1908          * TODO: Revisit the logic when we have better understanding on DRAM
1909          * channels' impact on the level 0 memory latency and the relevant
1910          * wm calculations.
1911          */
1912         if (skl_wm_has_lines(i915, level))
1913                 blocks = max(blocks,
1914                              fixed16_to_u32_round_up(wp->plane_blocks_per_line));
1915         lines = div_round_up_fixed16(selected_result,
1916                                      wp->plane_blocks_per_line);
1917
1918         if (DISPLAY_VER(i915) == 9) {
1919                 /* Display WA #1125: skl,bxt,kbl */
1920                 if (level == 0 && wp->rc_surface)
1921                         blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
1922
1923                 /* Display WA #1126: skl,bxt,kbl */
1924                 if (level >= 1 && level <= 7) {
1925                         if (wp->y_tiled) {
1926                                 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
1927                                 lines += wp->y_min_scanlines;
1928                         } else {
1929                                 blocks++;
1930                         }
1931
1932                         /*
1933                          * Make sure result blocks for higher latency levels are
1934                          * at least as high as level below the current level.
1935                          * Assumption in DDB algorithm optimization for special
1936                          * cases. Also covers Display WA #1125 for RC.
1937                          */
1938                         if (result_prev->blocks > blocks)
1939                                 blocks = result_prev->blocks;
1940                 }
1941         }
1942
1943         if (DISPLAY_VER(i915) >= 11) {
1944                 if (wp->y_tiled) {
1945                         int extra_lines;
1946
1947                         if (lines % wp->y_min_scanlines == 0)
1948                                 extra_lines = wp->y_min_scanlines;
1949                         else
1950                                 extra_lines = wp->y_min_scanlines * 2 -
1951                                         lines % wp->y_min_scanlines;
1952
1953                         min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
1954                                                                  wp->plane_blocks_per_line);
1955                 } else {
1956                         min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
1957                 }
1958         }
1959
1960         if (!skl_wm_has_lines(i915, level))
1961                 lines = 0;
1962
1963         if (lines > skl_wm_max_lines(i915)) {
1964                 /* reject it */
1965                 result->min_ddb_alloc = U16_MAX;
1966                 return;
1967         }
1968
1969         /*
1970          * If lines is valid, assume we can use this watermark level
1971          * for now.  We'll come back and disable it after we calculate the
1972          * DDB allocation if it turns out we don't actually have enough
1973          * blocks to satisfy it.
1974          */
1975         result->blocks = blocks;
1976         result->lines = lines;
1977         /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
1978         result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
1979         result->enable = true;
1980
1981         if (DISPLAY_VER(i915) < 12 && i915->display.sagv.block_time_us)
1982                 result->can_sagv = latency >= i915->display.sagv.block_time_us;
1983 }
1984
1985 static void
1986 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
1987                       struct intel_plane *plane,
1988                       const struct skl_wm_params *wm_params,
1989                       struct skl_wm_level *levels)
1990 {
1991         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1992         struct skl_wm_level *result_prev = &levels[0];
1993         int level;
1994
1995         for (level = 0; level < i915->display.wm.num_levels; level++) {
1996                 struct skl_wm_level *result = &levels[level];
1997                 unsigned int latency = skl_wm_latency(i915, level, wm_params);
1998
1999                 skl_compute_plane_wm(crtc_state, plane, level, latency,
2000                                      wm_params, result_prev, result);
2001
2002                 result_prev = result;
2003         }
2004 }
2005
2006 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
2007                                 struct intel_plane *plane,
2008                                 const struct skl_wm_params *wm_params,
2009                                 struct skl_plane_wm *plane_wm)
2010 {
2011         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
2012         struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
2013         struct skl_wm_level *levels = plane_wm->wm;
2014         unsigned int latency = 0;
2015
2016         if (i915->display.sagv.block_time_us)
2017                 latency = i915->display.sagv.block_time_us +
2018                         skl_wm_latency(i915, 0, wm_params);
2019
2020         skl_compute_plane_wm(crtc_state, plane, 0, latency,
2021                              wm_params, &levels[0],
2022                              sagv_wm);
2023 }
2024
2025 static void skl_compute_transition_wm(struct drm_i915_private *i915,
2026                                       struct skl_wm_level *trans_wm,
2027                                       const struct skl_wm_level *wm0,
2028                                       const struct skl_wm_params *wp)
2029 {
2030         u16 trans_min, trans_amount, trans_y_tile_min;
2031         u16 wm0_blocks, trans_offset, blocks;
2032
2033         /* Transition WM don't make any sense if ipc is disabled */
2034         if (!skl_watermark_ipc_enabled(i915))
2035                 return;
2036
2037         /*
2038          * WaDisableTWM:skl,kbl,cfl,bxt
2039          * Transition WM are not recommended by HW team for GEN9
2040          */
2041         if (DISPLAY_VER(i915) == 9)
2042                 return;
2043
2044         if (DISPLAY_VER(i915) >= 11)
2045                 trans_min = 4;
2046         else
2047                 trans_min = 14;
2048
2049         /* Display WA #1140: glk,cnl */
2050         if (DISPLAY_VER(i915) == 10)
2051                 trans_amount = 0;
2052         else
2053                 trans_amount = 10; /* This is configurable amount */
2054
2055         trans_offset = trans_min + trans_amount;
2056
2057         /*
2058          * The spec asks for Selected Result Blocks for wm0 (the real value),
2059          * not Result Blocks (the integer value). Pay attention to the capital
2060          * letters. The value wm_l0->blocks is actually Result Blocks, but
2061          * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
2062          * and since we later will have to get the ceiling of the sum in the
2063          * transition watermarks calculation, we can just pretend Selected
2064          * Result Blocks is Result Blocks minus 1 and it should work for the
2065          * current platforms.
2066          */
2067         wm0_blocks = wm0->blocks - 1;
2068
2069         if (wp->y_tiled) {
2070                 trans_y_tile_min =
2071                         (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
2072                 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
2073         } else {
2074                 blocks = wm0_blocks + trans_offset;
2075         }
2076         blocks++;
2077
2078         /*
2079          * Just assume we can enable the transition watermark.  After
2080          * computing the DDB we'll come back and disable it if that
2081          * assumption turns out to be false.
2082          */
2083         trans_wm->blocks = blocks;
2084         trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
2085         trans_wm->enable = true;
2086 }
2087
2088 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
2089                                      const struct intel_plane_state *plane_state,
2090                                      struct intel_plane *plane, int color_plane)
2091 {
2092         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2093         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2094         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2095         struct skl_wm_params wm_params;
2096         int ret;
2097
2098         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2099                                           &wm_params, color_plane);
2100         if (ret)
2101                 return ret;
2102
2103         skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm);
2104
2105         skl_compute_transition_wm(i915, &wm->trans_wm,
2106                                   &wm->wm[0], &wm_params);
2107
2108         if (DISPLAY_VER(i915) >= 12) {
2109                 tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm);
2110
2111                 skl_compute_transition_wm(i915, &wm->sagv.trans_wm,
2112                                           &wm->sagv.wm0, &wm_params);
2113         }
2114
2115         return 0;
2116 }
2117
2118 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
2119                                  const struct intel_plane_state *plane_state,
2120                                  struct intel_plane *plane)
2121 {
2122         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2123         struct skl_wm_params wm_params;
2124         int ret;
2125
2126         wm->is_planar = true;
2127
2128         /* uv plane watermarks must also be validated for NV12/Planar */
2129         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2130                                           &wm_params, 1);
2131         if (ret)
2132                 return ret;
2133
2134         skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm);
2135
2136         return 0;
2137 }
2138
2139 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
2140                               const struct intel_plane_state *plane_state)
2141 {
2142         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2143         enum plane_id plane_id = plane->id;
2144         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2145         const struct drm_framebuffer *fb = plane_state->hw.fb;
2146         int ret;
2147
2148         memset(wm, 0, sizeof(*wm));
2149
2150         if (!intel_wm_plane_visible(crtc_state, plane_state))
2151                 return 0;
2152
2153         ret = skl_build_plane_wm_single(crtc_state, plane_state,
2154                                         plane, 0);
2155         if (ret)
2156                 return ret;
2157
2158         if (fb->format->is_yuv && fb->format->num_planes > 1) {
2159                 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
2160                                             plane);
2161                 if (ret)
2162                         return ret;
2163         }
2164
2165         return 0;
2166 }
2167
2168 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
2169                               const struct intel_plane_state *plane_state)
2170 {
2171         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2172         struct drm_i915_private *i915 = to_i915(plane->base.dev);
2173         enum plane_id plane_id = plane->id;
2174         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2175         int ret;
2176
2177         /* Watermarks calculated in master */
2178         if (plane_state->planar_slave)
2179                 return 0;
2180
2181         memset(wm, 0, sizeof(*wm));
2182
2183         if (plane_state->planar_linked_plane) {
2184                 const struct drm_framebuffer *fb = plane_state->hw.fb;
2185
2186                 drm_WARN_ON(&i915->drm,
2187                             !intel_wm_plane_visible(crtc_state, plane_state));
2188                 drm_WARN_ON(&i915->drm, !fb->format->is_yuv ||
2189                             fb->format->num_planes == 1);
2190
2191                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2192                                                 plane_state->planar_linked_plane, 0);
2193                 if (ret)
2194                         return ret;
2195
2196                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2197                                                 plane, 1);
2198                 if (ret)
2199                         return ret;
2200         } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
2201                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2202                                                 plane, 0);
2203                 if (ret)
2204                         return ret;
2205         }
2206
2207         return 0;
2208 }
2209
2210 static bool
2211 skl_is_vblank_too_short(const struct intel_crtc_state *crtc_state,
2212                         int wm0_lines, int latency)
2213 {
2214         const struct drm_display_mode *adjusted_mode =
2215                 &crtc_state->hw.adjusted_mode;
2216
2217         /* FIXME missing scaler and DSC pre-fill time */
2218         return crtc_state->framestart_delay +
2219                 intel_usecs_to_scanlines(adjusted_mode, latency) +
2220                 wm0_lines >
2221                 adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vblank_start;
2222 }
2223
2224 static int skl_max_wm0_lines(const struct intel_crtc_state *crtc_state)
2225 {
2226         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2227         enum plane_id plane_id;
2228         int wm0_lines = 0;
2229
2230         for_each_plane_id_on_crtc(crtc, plane_id) {
2231                 const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
2232
2233                 /* FIXME what about !skl_wm_has_lines() platforms? */
2234                 wm0_lines = max_t(int, wm0_lines, wm->wm[0].lines);
2235         }
2236
2237         return wm0_lines;
2238 }
2239
2240 static int skl_max_wm_level_for_vblank(struct intel_crtc_state *crtc_state,
2241                                        int wm0_lines)
2242 {
2243         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2244         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2245         int level;
2246
2247         for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
2248                 int latency;
2249
2250                 /* FIXME should we care about the latency w/a's? */
2251                 latency = skl_wm_latency(i915, level, NULL);
2252                 if (latency == 0)
2253                         continue;
2254
2255                 /* FIXME is it correct to use 0 latency for wm0 here? */
2256                 if (level == 0)
2257                         latency = 0;
2258
2259                 if (!skl_is_vblank_too_short(crtc_state, wm0_lines, latency))
2260                         return level;
2261         }
2262
2263         return -EINVAL;
2264 }
2265
2266 static int skl_wm_check_vblank(struct intel_crtc_state *crtc_state)
2267 {
2268         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2269         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2270         int wm0_lines, level;
2271
2272         if (!crtc_state->hw.active)
2273                 return 0;
2274
2275         wm0_lines = skl_max_wm0_lines(crtc_state);
2276
2277         level = skl_max_wm_level_for_vblank(crtc_state, wm0_lines);
2278         if (level < 0)
2279                 return level;
2280
2281         /*
2282          * PSR needs to toggle LATENCY_REPORTING_REMOVED_PIPE_*
2283          * based on whether we're limited by the vblank duration.
2284          */
2285         crtc_state->wm_level_disabled = level < i915->display.wm.num_levels - 1;
2286
2287         for (level++; level < i915->display.wm.num_levels; level++) {
2288                 enum plane_id plane_id;
2289
2290                 for_each_plane_id_on_crtc(crtc, plane_id) {
2291                         struct skl_plane_wm *wm =
2292                                 &crtc_state->wm.skl.optimal.planes[plane_id];
2293
2294                         /*
2295                          * FIXME just clear enable or flag the entire
2296                          * thing as bad via min_ddb_alloc=U16_MAX?
2297                          */
2298                         wm->wm[level].enable = false;
2299                         wm->uv_wm[level].enable = false;
2300                 }
2301         }
2302
2303         if (DISPLAY_VER(i915) >= 12 &&
2304             i915->display.sagv.block_time_us &&
2305             skl_is_vblank_too_short(crtc_state, wm0_lines,
2306                                     i915->display.sagv.block_time_us)) {
2307                 enum plane_id plane_id;
2308
2309                 for_each_plane_id_on_crtc(crtc, plane_id) {
2310                         struct skl_plane_wm *wm =
2311                                 &crtc_state->wm.skl.optimal.planes[plane_id];
2312
2313                         wm->sagv.wm0.enable = false;
2314                         wm->sagv.trans_wm.enable = false;
2315                 }
2316         }
2317
2318         return 0;
2319 }
2320
2321 static int skl_build_pipe_wm(struct intel_atomic_state *state,
2322                              struct intel_crtc *crtc)
2323 {
2324         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2325         struct intel_crtc_state *crtc_state =
2326                 intel_atomic_get_new_crtc_state(state, crtc);
2327         const struct intel_plane_state *plane_state;
2328         struct intel_plane *plane;
2329         int ret, i;
2330
2331         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2332                 /*
2333                  * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
2334                  * instead but we don't populate that correctly for NV12 Y
2335                  * planes so for now hack this.
2336                  */
2337                 if (plane->pipe != crtc->pipe)
2338                         continue;
2339
2340                 if (DISPLAY_VER(i915) >= 11)
2341                         ret = icl_build_plane_wm(crtc_state, plane_state);
2342                 else
2343                         ret = skl_build_plane_wm(crtc_state, plane_state);
2344                 if (ret)
2345                         return ret;
2346         }
2347
2348         crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
2349
2350         return skl_wm_check_vblank(crtc_state);
2351 }
2352
2353 static void skl_ddb_entry_write(struct drm_i915_private *i915,
2354                                 i915_reg_t reg,
2355                                 const struct skl_ddb_entry *entry)
2356 {
2357         if (entry->end)
2358                 intel_de_write_fw(i915, reg,
2359                                   PLANE_BUF_END(entry->end - 1) |
2360                                   PLANE_BUF_START(entry->start));
2361         else
2362                 intel_de_write_fw(i915, reg, 0);
2363 }
2364
2365 static void skl_write_wm_level(struct drm_i915_private *i915,
2366                                i915_reg_t reg,
2367                                const struct skl_wm_level *level)
2368 {
2369         u32 val = 0;
2370
2371         if (level->enable)
2372                 val |= PLANE_WM_EN;
2373         if (level->ignore_lines)
2374                 val |= PLANE_WM_IGNORE_LINES;
2375         val |= REG_FIELD_PREP(PLANE_WM_BLOCKS_MASK, level->blocks);
2376         val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
2377
2378         intel_de_write_fw(i915, reg, val);
2379 }
2380
2381 void skl_write_plane_wm(struct intel_plane *plane,
2382                         const struct intel_crtc_state *crtc_state)
2383 {
2384         struct drm_i915_private *i915 = to_i915(plane->base.dev);
2385         enum plane_id plane_id = plane->id;
2386         enum pipe pipe = plane->pipe;
2387         const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
2388         const struct skl_ddb_entry *ddb =
2389                 &crtc_state->wm.skl.plane_ddb[plane_id];
2390         const struct skl_ddb_entry *ddb_y =
2391                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
2392         int level;
2393
2394         for (level = 0; level < i915->display.wm.num_levels; level++)
2395                 skl_write_wm_level(i915, PLANE_WM(pipe, plane_id, level),
2396                                    skl_plane_wm_level(pipe_wm, plane_id, level));
2397
2398         skl_write_wm_level(i915, PLANE_WM_TRANS(pipe, plane_id),
2399                            skl_plane_trans_wm(pipe_wm, plane_id));
2400
2401         if (HAS_HW_SAGV_WM(i915)) {
2402                 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
2403
2404                 skl_write_wm_level(i915, PLANE_WM_SAGV(pipe, plane_id),
2405                                    &wm->sagv.wm0);
2406                 skl_write_wm_level(i915, PLANE_WM_SAGV_TRANS(pipe, plane_id),
2407                                    &wm->sagv.trans_wm);
2408         }
2409
2410         skl_ddb_entry_write(i915,
2411                             PLANE_BUF_CFG(pipe, plane_id), ddb);
2412
2413         if (DISPLAY_VER(i915) < 11)
2414                 skl_ddb_entry_write(i915,
2415                                     PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_y);
2416 }
2417
2418 void skl_write_cursor_wm(struct intel_plane *plane,
2419                          const struct intel_crtc_state *crtc_state)
2420 {
2421         struct drm_i915_private *i915 = to_i915(plane->base.dev);
2422         enum plane_id plane_id = plane->id;
2423         enum pipe pipe = plane->pipe;
2424         const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
2425         const struct skl_ddb_entry *ddb =
2426                 &crtc_state->wm.skl.plane_ddb[plane_id];
2427         int level;
2428
2429         for (level = 0; level < i915->display.wm.num_levels; level++)
2430                 skl_write_wm_level(i915, CUR_WM(pipe, level),
2431                                    skl_plane_wm_level(pipe_wm, plane_id, level));
2432
2433         skl_write_wm_level(i915, CUR_WM_TRANS(pipe),
2434                            skl_plane_trans_wm(pipe_wm, plane_id));
2435
2436         if (HAS_HW_SAGV_WM(i915)) {
2437                 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
2438
2439                 skl_write_wm_level(i915, CUR_WM_SAGV(pipe),
2440                                    &wm->sagv.wm0);
2441                 skl_write_wm_level(i915, CUR_WM_SAGV_TRANS(pipe),
2442                                    &wm->sagv.trans_wm);
2443         }
2444
2445         skl_ddb_entry_write(i915, CUR_BUF_CFG(pipe), ddb);
2446 }
2447
2448 static bool skl_wm_level_equals(const struct skl_wm_level *l1,
2449                                 const struct skl_wm_level *l2)
2450 {
2451         return l1->enable == l2->enable &&
2452                 l1->ignore_lines == l2->ignore_lines &&
2453                 l1->lines == l2->lines &&
2454                 l1->blocks == l2->blocks;
2455 }
2456
2457 static bool skl_plane_wm_equals(struct drm_i915_private *i915,
2458                                 const struct skl_plane_wm *wm1,
2459                                 const struct skl_plane_wm *wm2)
2460 {
2461         int level;
2462
2463         for (level = 0; level < i915->display.wm.num_levels; level++) {
2464                 /*
2465                  * We don't check uv_wm as the hardware doesn't actually
2466                  * use it. It only gets used for calculating the required
2467                  * ddb allocation.
2468                  */
2469                 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
2470                         return false;
2471         }
2472
2473         return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
2474                 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
2475                 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
2476 }
2477
2478 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
2479                                     const struct skl_ddb_entry *b)
2480 {
2481         return a->start < b->end && b->start < a->end;
2482 }
2483
2484 static void skl_ddb_entry_union(struct skl_ddb_entry *a,
2485                                 const struct skl_ddb_entry *b)
2486 {
2487         if (a->end && b->end) {
2488                 a->start = min(a->start, b->start);
2489                 a->end = max(a->end, b->end);
2490         } else if (b->end) {
2491                 a->start = b->start;
2492                 a->end = b->end;
2493         }
2494 }
2495
2496 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
2497                                  const struct skl_ddb_entry *entries,
2498                                  int num_entries, int ignore_idx)
2499 {
2500         int i;
2501
2502         for (i = 0; i < num_entries; i++) {
2503                 if (i != ignore_idx &&
2504                     skl_ddb_entries_overlap(ddb, &entries[i]))
2505                         return true;
2506         }
2507
2508         return false;
2509 }
2510
2511 static int
2512 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
2513                             struct intel_crtc_state *new_crtc_state)
2514 {
2515         struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
2516         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2517         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2518         struct intel_plane *plane;
2519
2520         for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2521                 struct intel_plane_state *plane_state;
2522                 enum plane_id plane_id = plane->id;
2523
2524                 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb[plane_id],
2525                                         &new_crtc_state->wm.skl.plane_ddb[plane_id]) &&
2526                     skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
2527                                         &new_crtc_state->wm.skl.plane_ddb_y[plane_id]))
2528                         continue;
2529
2530                 plane_state = intel_atomic_get_plane_state(state, plane);
2531                 if (IS_ERR(plane_state))
2532                         return PTR_ERR(plane_state);
2533
2534                 new_crtc_state->update_planes |= BIT(plane_id);
2535                 new_crtc_state->async_flip_planes = 0;
2536                 new_crtc_state->do_async_flip = false;
2537         }
2538
2539         return 0;
2540 }
2541
2542 static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
2543 {
2544         struct drm_i915_private *i915 = to_i915(dbuf_state->base.state->base.dev);
2545         u8 enabled_slices;
2546         enum pipe pipe;
2547
2548         /*
2549          * FIXME: For now we always enable slice S1 as per
2550          * the Bspec display initialization sequence.
2551          */
2552         enabled_slices = BIT(DBUF_S1);
2553
2554         for_each_pipe(i915, pipe)
2555                 enabled_slices |= dbuf_state->slices[pipe];
2556
2557         return enabled_slices;
2558 }
2559
2560 static int
2561 skl_compute_ddb(struct intel_atomic_state *state)
2562 {
2563         struct drm_i915_private *i915 = to_i915(state->base.dev);
2564         const struct intel_dbuf_state *old_dbuf_state;
2565         struct intel_dbuf_state *new_dbuf_state = NULL;
2566         const struct intel_crtc_state *old_crtc_state;
2567         struct intel_crtc_state *new_crtc_state;
2568         struct intel_crtc *crtc;
2569         int ret, i;
2570
2571         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2572                 new_dbuf_state = intel_atomic_get_dbuf_state(state);
2573                 if (IS_ERR(new_dbuf_state))
2574                         return PTR_ERR(new_dbuf_state);
2575
2576                 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
2577                 break;
2578         }
2579
2580         if (!new_dbuf_state)
2581                 return 0;
2582
2583         new_dbuf_state->active_pipes =
2584                 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
2585
2586         if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
2587                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2588                 if (ret)
2589                         return ret;
2590         }
2591
2592         if (HAS_MBUS_JOINING(i915))
2593                 new_dbuf_state->joined_mbus =
2594                         adlp_check_mbus_joined(new_dbuf_state->active_pipes);
2595
2596         for_each_intel_crtc(&i915->drm, crtc) {
2597                 enum pipe pipe = crtc->pipe;
2598
2599                 new_dbuf_state->slices[pipe] =
2600                         skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes,
2601                                                 new_dbuf_state->joined_mbus);
2602
2603                 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
2604                         continue;
2605
2606                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2607                 if (ret)
2608                         return ret;
2609         }
2610
2611         new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
2612
2613         if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
2614             old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2615                 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
2616                 if (ret)
2617                         return ret;
2618
2619                 if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2620                         /* TODO: Implement vblank synchronized MBUS joining changes */
2621                         ret = intel_modeset_all_pipes_late(state, "MBUS joining change");
2622                         if (ret)
2623                                 return ret;
2624                 }
2625
2626                 drm_dbg_kms(&i915->drm,
2627                             "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
2628                             old_dbuf_state->enabled_slices,
2629                             new_dbuf_state->enabled_slices,
2630                             DISPLAY_INFO(i915)->dbuf.slice_mask,
2631                             str_yes_no(old_dbuf_state->joined_mbus),
2632                             str_yes_no(new_dbuf_state->joined_mbus));
2633         }
2634
2635         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2636                 enum pipe pipe = crtc->pipe;
2637
2638                 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
2639
2640                 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
2641                         continue;
2642
2643                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2644                 if (ret)
2645                         return ret;
2646         }
2647
2648         for_each_intel_crtc(&i915->drm, crtc) {
2649                 ret = skl_crtc_allocate_ddb(state, crtc);
2650                 if (ret)
2651                         return ret;
2652         }
2653
2654         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2655                                             new_crtc_state, i) {
2656                 ret = skl_crtc_allocate_plane_ddb(state, crtc);
2657                 if (ret)
2658                         return ret;
2659
2660                 ret = skl_ddb_add_affected_planes(old_crtc_state,
2661                                                   new_crtc_state);
2662                 if (ret)
2663                         return ret;
2664         }
2665
2666         return 0;
2667 }
2668
2669 static char enast(bool enable)
2670 {
2671         return enable ? '*' : ' ';
2672 }
2673
2674 static void
2675 skl_print_wm_changes(struct intel_atomic_state *state)
2676 {
2677         struct drm_i915_private *i915 = to_i915(state->base.dev);
2678         const struct intel_crtc_state *old_crtc_state;
2679         const struct intel_crtc_state *new_crtc_state;
2680         struct intel_plane *plane;
2681         struct intel_crtc *crtc;
2682         int i;
2683
2684         if (!drm_debug_enabled(DRM_UT_KMS))
2685                 return;
2686
2687         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2688                                             new_crtc_state, i) {
2689                 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
2690
2691                 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
2692                 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
2693
2694                 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2695                         enum plane_id plane_id = plane->id;
2696                         const struct skl_ddb_entry *old, *new;
2697
2698                         old = &old_crtc_state->wm.skl.plane_ddb[plane_id];
2699                         new = &new_crtc_state->wm.skl.plane_ddb[plane_id];
2700
2701                         if (skl_ddb_entry_equal(old, new))
2702                                 continue;
2703
2704                         drm_dbg_kms(&i915->drm,
2705                                     "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
2706                                     plane->base.base.id, plane->base.name,
2707                                     old->start, old->end, new->start, new->end,
2708                                     skl_ddb_entry_size(old), skl_ddb_entry_size(new));
2709                 }
2710
2711                 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2712                         enum plane_id plane_id = plane->id;
2713                         const struct skl_plane_wm *old_wm, *new_wm;
2714
2715                         old_wm = &old_pipe_wm->planes[plane_id];
2716                         new_wm = &new_pipe_wm->planes[plane_id];
2717
2718                         if (skl_plane_wm_equals(i915, old_wm, new_wm))
2719                                 continue;
2720
2721                         drm_dbg_kms(&i915->drm,
2722                                     "[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
2723                                     " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
2724                                     plane->base.base.id, plane->base.name,
2725                                     enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
2726                                     enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
2727                                     enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
2728                                     enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
2729                                     enast(old_wm->trans_wm.enable),
2730                                     enast(old_wm->sagv.wm0.enable),
2731                                     enast(old_wm->sagv.trans_wm.enable),
2732                                     enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
2733                                     enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
2734                                     enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
2735                                     enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
2736                                     enast(new_wm->trans_wm.enable),
2737                                     enast(new_wm->sagv.wm0.enable),
2738                                     enast(new_wm->sagv.trans_wm.enable));
2739
2740                         drm_dbg_kms(&i915->drm,
2741                                     "[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
2742                                       " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
2743                                     plane->base.base.id, plane->base.name,
2744                                     enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
2745                                     enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
2746                                     enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
2747                                     enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
2748                                     enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
2749                                     enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
2750                                     enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
2751                                     enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
2752                                     enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
2753                                     enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
2754                                     enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
2755                                     enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
2756                                     enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
2757                                     enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
2758                                     enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
2759                                     enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
2760                                     enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
2761                                     enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
2762                                     enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
2763                                     enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
2764                                     enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
2765                                     enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
2766
2767                         drm_dbg_kms(&i915->drm,
2768                                     "[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2769                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2770                                     plane->base.base.id, plane->base.name,
2771                                     old_wm->wm[0].blocks, old_wm->wm[1].blocks,
2772                                     old_wm->wm[2].blocks, old_wm->wm[3].blocks,
2773                                     old_wm->wm[4].blocks, old_wm->wm[5].blocks,
2774                                     old_wm->wm[6].blocks, old_wm->wm[7].blocks,
2775                                     old_wm->trans_wm.blocks,
2776                                     old_wm->sagv.wm0.blocks,
2777                                     old_wm->sagv.trans_wm.blocks,
2778                                     new_wm->wm[0].blocks, new_wm->wm[1].blocks,
2779                                     new_wm->wm[2].blocks, new_wm->wm[3].blocks,
2780                                     new_wm->wm[4].blocks, new_wm->wm[5].blocks,
2781                                     new_wm->wm[6].blocks, new_wm->wm[7].blocks,
2782                                     new_wm->trans_wm.blocks,
2783                                     new_wm->sagv.wm0.blocks,
2784                                     new_wm->sagv.trans_wm.blocks);
2785
2786                         drm_dbg_kms(&i915->drm,
2787                                     "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2788                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2789                                     plane->base.base.id, plane->base.name,
2790                                     old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
2791                                     old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
2792                                     old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
2793                                     old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
2794                                     old_wm->trans_wm.min_ddb_alloc,
2795                                     old_wm->sagv.wm0.min_ddb_alloc,
2796                                     old_wm->sagv.trans_wm.min_ddb_alloc,
2797                                     new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
2798                                     new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
2799                                     new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
2800                                     new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
2801                                     new_wm->trans_wm.min_ddb_alloc,
2802                                     new_wm->sagv.wm0.min_ddb_alloc,
2803                                     new_wm->sagv.trans_wm.min_ddb_alloc);
2804                 }
2805         }
2806 }
2807
2808 static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
2809                                          const struct skl_pipe_wm *old_pipe_wm,
2810                                          const struct skl_pipe_wm *new_pipe_wm)
2811 {
2812         struct drm_i915_private *i915 = to_i915(plane->base.dev);
2813         int level;
2814
2815         for (level = 0; level < i915->display.wm.num_levels; level++) {
2816                 /*
2817                  * We don't check uv_wm as the hardware doesn't actually
2818                  * use it. It only gets used for calculating the required
2819                  * ddb allocation.
2820                  */
2821                 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
2822                                          skl_plane_wm_level(new_pipe_wm, plane->id, level)))
2823                         return false;
2824         }
2825
2826         if (HAS_HW_SAGV_WM(i915)) {
2827                 const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
2828                 const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
2829
2830                 if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
2831                     !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
2832                         return false;
2833         }
2834
2835         return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
2836                                    skl_plane_trans_wm(new_pipe_wm, plane->id));
2837 }
2838
2839 /*
2840  * To make sure the cursor watermark registers are always consistent
2841  * with our computed state the following scenario needs special
2842  * treatment:
2843  *
2844  * 1. enable cursor
2845  * 2. move cursor entirely offscreen
2846  * 3. disable cursor
2847  *
2848  * Step 2. does call .disable_plane() but does not zero the watermarks
2849  * (since we consider an offscreen cursor still active for the purposes
2850  * of watermarks). Step 3. would not normally call .disable_plane()
2851  * because the actual plane visibility isn't changing, and we don't
2852  * deallocate the cursor ddb until the pipe gets disabled. So we must
2853  * force step 3. to call .disable_plane() to update the watermark
2854  * registers properly.
2855  *
2856  * Other planes do not suffer from this issues as their watermarks are
2857  * calculated based on the actual plane visibility. The only time this
2858  * can trigger for the other planes is during the initial readout as the
2859  * default value of the watermarks registers is not zero.
2860  */
2861 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
2862                                       struct intel_crtc *crtc)
2863 {
2864         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2865         const struct intel_crtc_state *old_crtc_state =
2866                 intel_atomic_get_old_crtc_state(state, crtc);
2867         struct intel_crtc_state *new_crtc_state =
2868                 intel_atomic_get_new_crtc_state(state, crtc);
2869         struct intel_plane *plane;
2870
2871         for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2872                 struct intel_plane_state *plane_state;
2873                 enum plane_id plane_id = plane->id;
2874
2875                 /*
2876                  * Force a full wm update for every plane on modeset.
2877                  * Required because the reset value of the wm registers
2878                  * is non-zero, whereas we want all disabled planes to
2879                  * have zero watermarks. So if we turn off the relevant
2880                  * power well the hardware state will go out of sync
2881                  * with the software state.
2882                  */
2883                 if (!intel_crtc_needs_modeset(new_crtc_state) &&
2884                     skl_plane_selected_wm_equals(plane,
2885                                                  &old_crtc_state->wm.skl.optimal,
2886                                                  &new_crtc_state->wm.skl.optimal))
2887                         continue;
2888
2889                 plane_state = intel_atomic_get_plane_state(state, plane);
2890                 if (IS_ERR(plane_state))
2891                         return PTR_ERR(plane_state);
2892
2893                 new_crtc_state->update_planes |= BIT(plane_id);
2894                 new_crtc_state->async_flip_planes = 0;
2895                 new_crtc_state->do_async_flip = false;
2896         }
2897
2898         return 0;
2899 }
2900
2901 static int
2902 skl_compute_wm(struct intel_atomic_state *state)
2903 {
2904         struct intel_crtc *crtc;
2905         struct intel_crtc_state __maybe_unused *new_crtc_state;
2906         int ret, i;
2907
2908         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2909                 ret = skl_build_pipe_wm(state, crtc);
2910                 if (ret)
2911                         return ret;
2912         }
2913
2914         ret = skl_compute_ddb(state);
2915         if (ret)
2916                 return ret;
2917
2918         ret = intel_compute_sagv_mask(state);
2919         if (ret)
2920                 return ret;
2921
2922         /*
2923          * skl_compute_ddb() will have adjusted the final watermarks
2924          * based on how much ddb is available. Now we can actually
2925          * check if the final watermarks changed.
2926          */
2927         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2928                 ret = skl_wm_add_affected_planes(state, crtc);
2929                 if (ret)
2930                         return ret;
2931         }
2932
2933         skl_print_wm_changes(state);
2934
2935         return 0;
2936 }
2937
2938 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
2939 {
2940         level->enable = val & PLANE_WM_EN;
2941         level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
2942         level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val);
2943         level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
2944 }
2945
2946 static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
2947                                      struct skl_pipe_wm *out)
2948 {
2949         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2950         enum pipe pipe = crtc->pipe;
2951         enum plane_id plane_id;
2952         int level;
2953         u32 val;
2954
2955         for_each_plane_id_on_crtc(crtc, plane_id) {
2956                 struct skl_plane_wm *wm = &out->planes[plane_id];
2957
2958                 for (level = 0; level < i915->display.wm.num_levels; level++) {
2959                         if (plane_id != PLANE_CURSOR)
2960                                 val = intel_de_read(i915, PLANE_WM(pipe, plane_id, level));
2961                         else
2962                                 val = intel_de_read(i915, CUR_WM(pipe, level));
2963
2964                         skl_wm_level_from_reg_val(val, &wm->wm[level]);
2965                 }
2966
2967                 if (plane_id != PLANE_CURSOR)
2968                         val = intel_de_read(i915, PLANE_WM_TRANS(pipe, plane_id));
2969                 else
2970                         val = intel_de_read(i915, CUR_WM_TRANS(pipe));
2971
2972                 skl_wm_level_from_reg_val(val, &wm->trans_wm);
2973
2974                 if (HAS_HW_SAGV_WM(i915)) {
2975                         if (plane_id != PLANE_CURSOR)
2976                                 val = intel_de_read(i915, PLANE_WM_SAGV(pipe, plane_id));
2977                         else
2978                                 val = intel_de_read(i915, CUR_WM_SAGV(pipe));
2979
2980                         skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
2981
2982                         if (plane_id != PLANE_CURSOR)
2983                                 val = intel_de_read(i915, PLANE_WM_SAGV_TRANS(pipe, plane_id));
2984                         else
2985                                 val = intel_de_read(i915, CUR_WM_SAGV_TRANS(pipe));
2986
2987                         skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
2988                 } else if (DISPLAY_VER(i915) >= 12) {
2989                         wm->sagv.wm0 = wm->wm[0];
2990                         wm->sagv.trans_wm = wm->trans_wm;
2991                 }
2992         }
2993 }
2994
2995 static void skl_wm_get_hw_state(struct drm_i915_private *i915)
2996 {
2997         struct intel_dbuf_state *dbuf_state =
2998                 to_intel_dbuf_state(i915->display.dbuf.obj.state);
2999         struct intel_crtc *crtc;
3000
3001         if (HAS_MBUS_JOINING(i915))
3002                 dbuf_state->joined_mbus = intel_de_read(i915, MBUS_CTL) & MBUS_JOIN;
3003
3004         for_each_intel_crtc(&i915->drm, crtc) {
3005                 struct intel_crtc_state *crtc_state =
3006                         to_intel_crtc_state(crtc->base.state);
3007                 enum pipe pipe = crtc->pipe;
3008                 unsigned int mbus_offset;
3009                 enum plane_id plane_id;
3010                 u8 slices;
3011
3012                 memset(&crtc_state->wm.skl.optimal, 0,
3013                        sizeof(crtc_state->wm.skl.optimal));
3014                 if (crtc_state->hw.active)
3015                         skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
3016                 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
3017
3018                 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
3019
3020                 for_each_plane_id_on_crtc(crtc, plane_id) {
3021                         struct skl_ddb_entry *ddb =
3022                                 &crtc_state->wm.skl.plane_ddb[plane_id];
3023                         struct skl_ddb_entry *ddb_y =
3024                                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
3025
3026                         if (!crtc_state->hw.active)
3027                                 continue;
3028
3029                         skl_ddb_get_hw_plane_state(i915, crtc->pipe,
3030                                                    plane_id, ddb, ddb_y);
3031
3032                         skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb);
3033                         skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
3034                 }
3035
3036                 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
3037
3038                 /*
3039                  * Used for checking overlaps, so we need absolute
3040                  * offsets instead of MBUS relative offsets.
3041                  */
3042                 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
3043                                                  dbuf_state->joined_mbus);
3044                 mbus_offset = mbus_ddb_offset(i915, slices);
3045                 crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
3046                 crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
3047
3048                 /* The slices actually used by the planes on the pipe */
3049                 dbuf_state->slices[pipe] =
3050                         skl_ddb_dbuf_slice_mask(i915, &crtc_state->wm.skl.ddb);
3051
3052                 drm_dbg_kms(&i915->drm,
3053                             "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
3054                             crtc->base.base.id, crtc->base.name,
3055                             dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
3056                             dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
3057                             str_yes_no(dbuf_state->joined_mbus));
3058         }
3059
3060         dbuf_state->enabled_slices = i915->display.dbuf.enabled_slices;
3061 }
3062
3063 static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
3064 {
3065         const struct intel_dbuf_state *dbuf_state =
3066                 to_intel_dbuf_state(i915->display.dbuf.obj.state);
3067         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
3068         struct intel_crtc *crtc;
3069
3070         for_each_intel_crtc(&i915->drm, crtc) {
3071                 const struct intel_crtc_state *crtc_state =
3072                         to_intel_crtc_state(crtc->base.state);
3073
3074                 entries[crtc->pipe] = crtc_state->wm.skl.ddb;
3075         }
3076
3077         for_each_intel_crtc(&i915->drm, crtc) {
3078                 const struct intel_crtc_state *crtc_state =
3079                         to_intel_crtc_state(crtc->base.state);
3080                 u8 slices;
3081
3082                 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
3083                                                  dbuf_state->joined_mbus);
3084                 if (dbuf_state->slices[crtc->pipe] & ~slices)
3085                         return true;
3086
3087                 if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries,
3088                                                 I915_MAX_PIPES, crtc->pipe))
3089                         return true;
3090         }
3091
3092         return false;
3093 }
3094
3095 static void skl_wm_sanitize(struct drm_i915_private *i915)
3096 {
3097         struct intel_crtc *crtc;
3098
3099         /*
3100          * On TGL/RKL (at least) the BIOS likes to assign the planes
3101          * to the wrong DBUF slices. This will cause an infinite loop
3102          * in skl_commit_modeset_enables() as it can't find a way to
3103          * transition between the old bogus DBUF layout to the new
3104          * proper DBUF layout without DBUF allocation overlaps between
3105          * the planes (which cannot be allowed or else the hardware
3106          * may hang). If we detect a bogus DBUF layout just turn off
3107          * all the planes so that skl_commit_modeset_enables() can
3108          * simply ignore them.
3109          */
3110         if (!skl_dbuf_is_misconfigured(i915))
3111                 return;
3112
3113         drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
3114
3115         for_each_intel_crtc(&i915->drm, crtc) {
3116                 struct intel_plane *plane = to_intel_plane(crtc->base.primary);
3117                 const struct intel_plane_state *plane_state =
3118                         to_intel_plane_state(plane->base.state);
3119                 struct intel_crtc_state *crtc_state =
3120                         to_intel_crtc_state(crtc->base.state);
3121
3122                 if (plane_state->uapi.visible)
3123                         intel_plane_disable_noatomic(crtc, plane);
3124
3125                 drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
3126
3127                 memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
3128         }
3129 }
3130
3131 static void skl_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3132 {
3133         skl_wm_get_hw_state(i915);
3134         skl_wm_sanitize(i915);
3135 }
3136
3137 void intel_wm_state_verify(struct intel_atomic_state *state,
3138                            struct intel_crtc *crtc)
3139 {
3140         struct drm_i915_private *i915 = to_i915(state->base.dev);
3141         const struct intel_crtc_state *new_crtc_state =
3142                 intel_atomic_get_new_crtc_state(state, crtc);
3143         struct skl_hw_state {
3144                 struct skl_ddb_entry ddb[I915_MAX_PLANES];
3145                 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
3146                 struct skl_pipe_wm wm;
3147         } *hw;
3148         const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
3149         struct intel_plane *plane;
3150         u8 hw_enabled_slices;
3151         int level;
3152
3153         if (DISPLAY_VER(i915) < 9 || !new_crtc_state->hw.active)
3154                 return;
3155
3156         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
3157         if (!hw)
3158                 return;
3159
3160         skl_pipe_wm_get_hw_state(crtc, &hw->wm);
3161
3162         skl_pipe_ddb_get_hw_state(crtc, hw->ddb, hw->ddb_y);
3163
3164         hw_enabled_slices = intel_enabled_dbuf_slices_mask(i915);
3165
3166         if (DISPLAY_VER(i915) >= 11 &&
3167             hw_enabled_slices != i915->display.dbuf.enabled_slices)
3168                 drm_err(&i915->drm,
3169                         "mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
3170                         i915->display.dbuf.enabled_slices,
3171                         hw_enabled_slices);
3172
3173         for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
3174                 const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
3175                 const struct skl_wm_level *hw_wm_level, *sw_wm_level;
3176
3177                 /* Watermarks */
3178                 for (level = 0; level < i915->display.wm.num_levels; level++) {
3179                         hw_wm_level = &hw->wm.planes[plane->id].wm[level];
3180                         sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level);
3181
3182                         if (skl_wm_level_equals(hw_wm_level, sw_wm_level))
3183                                 continue;
3184
3185                         drm_err(&i915->drm,
3186                                 "[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3187                                 plane->base.base.id, plane->base.name, level,
3188                                 sw_wm_level->enable,
3189                                 sw_wm_level->blocks,
3190                                 sw_wm_level->lines,
3191                                 hw_wm_level->enable,
3192                                 hw_wm_level->blocks,
3193                                 hw_wm_level->lines);
3194                 }
3195
3196                 hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
3197                 sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id);
3198
3199                 if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3200                         drm_err(&i915->drm,
3201                                 "[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3202                                 plane->base.base.id, plane->base.name,
3203                                 sw_wm_level->enable,
3204                                 sw_wm_level->blocks,
3205                                 sw_wm_level->lines,
3206                                 hw_wm_level->enable,
3207                                 hw_wm_level->blocks,
3208                                 hw_wm_level->lines);
3209                 }
3210
3211                 hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
3212                 sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
3213
3214                 if (HAS_HW_SAGV_WM(i915) &&
3215                     !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3216                         drm_err(&i915->drm,
3217                                 "[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3218                                 plane->base.base.id, plane->base.name,
3219                                 sw_wm_level->enable,
3220                                 sw_wm_level->blocks,
3221                                 sw_wm_level->lines,
3222                                 hw_wm_level->enable,
3223                                 hw_wm_level->blocks,
3224                                 hw_wm_level->lines);
3225                 }
3226
3227                 hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
3228                 sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
3229
3230                 if (HAS_HW_SAGV_WM(i915) &&
3231                     !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3232                         drm_err(&i915->drm,
3233                                 "[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3234                                 plane->base.base.id, plane->base.name,
3235                                 sw_wm_level->enable,
3236                                 sw_wm_level->blocks,
3237                                 sw_wm_level->lines,
3238                                 hw_wm_level->enable,
3239                                 hw_wm_level->blocks,
3240                                 hw_wm_level->lines);
3241                 }
3242
3243                 /* DDB */
3244                 hw_ddb_entry = &hw->ddb[PLANE_CURSOR];
3245                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR];
3246
3247                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
3248                         drm_err(&i915->drm,
3249                                 "[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
3250                                 plane->base.base.id, plane->base.name,
3251                                 sw_ddb_entry->start, sw_ddb_entry->end,
3252                                 hw_ddb_entry->start, hw_ddb_entry->end);
3253                 }
3254         }
3255
3256         kfree(hw);
3257 }
3258
3259 bool skl_watermark_ipc_enabled(struct drm_i915_private *i915)
3260 {
3261         return i915->display.wm.ipc_enabled;
3262 }
3263
3264 void skl_watermark_ipc_update(struct drm_i915_private *i915)
3265 {
3266         if (!HAS_IPC(i915))
3267                 return;
3268
3269         intel_de_rmw(i915, DISP_ARB_CTL2, DISP_IPC_ENABLE,
3270                      skl_watermark_ipc_enabled(i915) ? DISP_IPC_ENABLE : 0);
3271 }
3272
3273 static bool skl_watermark_ipc_can_enable(struct drm_i915_private *i915)
3274 {
3275         /* Display WA #0477 WaDisableIPC: skl */
3276         if (IS_SKYLAKE(i915))
3277                 return false;
3278
3279         /* Display WA #1141: SKL:all KBL:all CFL */
3280         if (IS_KABYLAKE(i915) ||
3281             IS_COFFEELAKE(i915) ||
3282             IS_COMETLAKE(i915))
3283                 return i915->dram_info.symmetric_memory;
3284
3285         return true;
3286 }
3287
3288 void skl_watermark_ipc_init(struct drm_i915_private *i915)
3289 {
3290         if (!HAS_IPC(i915))
3291                 return;
3292
3293         i915->display.wm.ipc_enabled = skl_watermark_ipc_can_enable(i915);
3294
3295         skl_watermark_ipc_update(i915);
3296 }
3297
3298 static void
3299 adjust_wm_latency(struct drm_i915_private *i915,
3300                   u16 wm[], int num_levels, int read_latency)
3301 {
3302         bool wm_lv_0_adjust_needed = i915->dram_info.wm_lv_0_adjust_needed;
3303         int i, level;
3304
3305         /*
3306          * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
3307          * need to be disabled. We make sure to sanitize the values out
3308          * of the punit to satisfy this requirement.
3309          */
3310         for (level = 1; level < num_levels; level++) {
3311                 if (wm[level] == 0) {
3312                         for (i = level + 1; i < num_levels; i++)
3313                                 wm[i] = 0;
3314
3315                         num_levels = level;
3316                         break;
3317                 }
3318         }
3319
3320         /*
3321          * WaWmMemoryReadLatency
3322          *
3323          * punit doesn't take into account the read latency so we need
3324          * to add proper adjustement to each valid level we retrieve
3325          * from the punit when level 0 response data is 0us.
3326          */
3327         if (wm[0] == 0) {
3328                 for (level = 0; level < num_levels; level++)
3329                         wm[level] += read_latency;
3330         }
3331
3332         /*
3333          * WA Level-0 adjustment for 16GB DIMMs: SKL+
3334          * If we could not get dimm info enable this WA to prevent from
3335          * any underrun. If not able to get Dimm info assume 16GB dimm
3336          * to avoid any underrun.
3337          */
3338         if (wm_lv_0_adjust_needed)
3339                 wm[0] += 1;
3340 }
3341
3342 static void mtl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3343 {
3344         int num_levels = i915->display.wm.num_levels;
3345         u32 val;
3346
3347         val = intel_de_read(i915, MTL_LATENCY_LP0_LP1);
3348         wm[0] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3349         wm[1] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3350
3351         val = intel_de_read(i915, MTL_LATENCY_LP2_LP3);
3352         wm[2] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3353         wm[3] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3354
3355         val = intel_de_read(i915, MTL_LATENCY_LP4_LP5);
3356         wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3357         wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3358
3359         adjust_wm_latency(i915, wm, num_levels, 6);
3360 }
3361
3362 static void skl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3363 {
3364         int num_levels = i915->display.wm.num_levels;
3365         int read_latency = DISPLAY_VER(i915) >= 12 ? 3 : 2;
3366         int mult = IS_DG2(i915) ? 2 : 1;
3367         u32 val;
3368         int ret;
3369
3370         /* read the first set of memory latencies[0:3] */
3371         val = 0; /* data0 to be programmed to 0 for first set */
3372         ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3373         if (ret) {
3374                 drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3375                 return;
3376         }
3377
3378         wm[0] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3379         wm[1] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3380         wm[2] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3381         wm[3] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3382
3383         /* read the second set of memory latencies[4:7] */
3384         val = 1; /* data0 to be programmed to 1 for second set */
3385         ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3386         if (ret) {
3387                 drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3388                 return;
3389         }
3390
3391         wm[4] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3392         wm[5] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3393         wm[6] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3394         wm[7] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3395
3396         adjust_wm_latency(i915, wm, num_levels, read_latency);
3397 }
3398
3399 static void skl_setup_wm_latency(struct drm_i915_private *i915)
3400 {
3401         if (HAS_HW_SAGV_WM(i915))
3402                 i915->display.wm.num_levels = 6;
3403         else
3404                 i915->display.wm.num_levels = 8;
3405
3406         if (DISPLAY_VER(i915) >= 14)
3407                 mtl_read_wm_latency(i915, i915->display.wm.skl_latency);
3408         else
3409                 skl_read_wm_latency(i915, i915->display.wm.skl_latency);
3410
3411         intel_print_wm_latency(i915, "Gen9 Plane", i915->display.wm.skl_latency);
3412 }
3413
3414 static const struct intel_wm_funcs skl_wm_funcs = {
3415         .compute_global_watermarks = skl_compute_wm,
3416         .get_hw_state = skl_wm_get_hw_state_and_sanitize,
3417 };
3418
3419 void skl_wm_init(struct drm_i915_private *i915)
3420 {
3421         intel_sagv_init(i915);
3422
3423         skl_setup_wm_latency(i915);
3424
3425         i915->display.funcs.wm = &skl_wm_funcs;
3426 }
3427
3428 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
3429 {
3430         struct intel_dbuf_state *dbuf_state;
3431
3432         dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
3433         if (!dbuf_state)
3434                 return NULL;
3435
3436         return &dbuf_state->base;
3437 }
3438
3439 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
3440                                      struct intel_global_state *state)
3441 {
3442         kfree(state);
3443 }
3444
3445 static const struct intel_global_state_funcs intel_dbuf_funcs = {
3446         .atomic_duplicate_state = intel_dbuf_duplicate_state,
3447         .atomic_destroy_state = intel_dbuf_destroy_state,
3448 };
3449
3450 struct intel_dbuf_state *
3451 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
3452 {
3453         struct drm_i915_private *i915 = to_i915(state->base.dev);
3454         struct intel_global_state *dbuf_state;
3455
3456         dbuf_state = intel_atomic_get_global_obj_state(state, &i915->display.dbuf.obj);
3457         if (IS_ERR(dbuf_state))
3458                 return ERR_CAST(dbuf_state);
3459
3460         return to_intel_dbuf_state(dbuf_state);
3461 }
3462
3463 int intel_dbuf_init(struct drm_i915_private *i915)
3464 {
3465         struct intel_dbuf_state *dbuf_state;
3466
3467         dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
3468         if (!dbuf_state)
3469                 return -ENOMEM;
3470
3471         intel_atomic_global_obj_init(i915, &i915->display.dbuf.obj,
3472                                      &dbuf_state->base, &intel_dbuf_funcs);
3473
3474         return 0;
3475 }
3476
3477 /*
3478  * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
3479  * update the request state of all DBUS slices.
3480  */
3481 static void update_mbus_pre_enable(struct intel_atomic_state *state)
3482 {
3483         struct drm_i915_private *i915 = to_i915(state->base.dev);
3484         u32 mbus_ctl, dbuf_min_tracker_val;
3485         enum dbuf_slice slice;
3486         const struct intel_dbuf_state *dbuf_state =
3487                 intel_atomic_get_new_dbuf_state(state);
3488
3489         if (!HAS_MBUS_JOINING(i915))
3490                 return;
3491
3492         /*
3493          * TODO: Implement vblank synchronized MBUS joining changes.
3494          * Must be properly coordinated with dbuf reprogramming.
3495          */
3496         if (dbuf_state->joined_mbus) {
3497                 mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
3498                         MBUS_JOIN_PIPE_SELECT_NONE;
3499                 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
3500         } else {
3501                 mbus_ctl = MBUS_HASHING_MODE_2x2 |
3502                         MBUS_JOIN_PIPE_SELECT_NONE;
3503                 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
3504         }
3505
3506         intel_de_rmw(i915, MBUS_CTL,
3507                      MBUS_HASHING_MODE_MASK | MBUS_JOIN |
3508                      MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
3509
3510         for_each_dbuf_slice(i915, slice)
3511                 intel_de_rmw(i915, DBUF_CTL_S(slice),
3512                              DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
3513                              dbuf_min_tracker_val);
3514 }
3515
3516 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
3517 {
3518         struct drm_i915_private *i915 = to_i915(state->base.dev);
3519         const struct intel_dbuf_state *new_dbuf_state =
3520                 intel_atomic_get_new_dbuf_state(state);
3521         const struct intel_dbuf_state *old_dbuf_state =
3522                 intel_atomic_get_old_dbuf_state(state);
3523
3524         if (!new_dbuf_state ||
3525             (new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices &&
3526              new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))
3527                 return;
3528
3529         WARN_ON(!new_dbuf_state->base.changed);
3530
3531         update_mbus_pre_enable(state);
3532         gen9_dbuf_slices_update(i915,
3533                                 old_dbuf_state->enabled_slices |
3534                                 new_dbuf_state->enabled_slices);
3535 }
3536
3537 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
3538 {
3539         struct drm_i915_private *i915 = to_i915(state->base.dev);
3540         const struct intel_dbuf_state *new_dbuf_state =
3541                 intel_atomic_get_new_dbuf_state(state);
3542         const struct intel_dbuf_state *old_dbuf_state =
3543                 intel_atomic_get_old_dbuf_state(state);
3544
3545         if (!new_dbuf_state ||
3546             (new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices &&
3547              new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))
3548                 return;
3549
3550         WARN_ON(!new_dbuf_state->base.changed);
3551
3552         gen9_dbuf_slices_update(i915,
3553                                 new_dbuf_state->enabled_slices);
3554 }
3555
3556 static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes)
3557 {
3558         switch (pipe) {
3559         case PIPE_A:
3560                 return !(active_pipes & BIT(PIPE_D));
3561         case PIPE_D:
3562                 return !(active_pipes & BIT(PIPE_A));
3563         case PIPE_B:
3564                 return !(active_pipes & BIT(PIPE_C));
3565         case PIPE_C:
3566                 return !(active_pipes & BIT(PIPE_B));
3567         default: /* to suppress compiler warning */
3568                 MISSING_CASE(pipe);
3569                 break;
3570         }
3571
3572         return false;
3573 }
3574
3575 void intel_mbus_dbox_update(struct intel_atomic_state *state)
3576 {
3577         struct drm_i915_private *i915 = to_i915(state->base.dev);
3578         const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
3579         const struct intel_crtc_state *new_crtc_state;
3580         const struct intel_crtc *crtc;
3581         u32 val = 0;
3582         int i;
3583
3584         if (DISPLAY_VER(i915) < 11)
3585                 return;
3586
3587         new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
3588         old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
3589         if (!new_dbuf_state ||
3590             (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus &&
3591              new_dbuf_state->active_pipes == old_dbuf_state->active_pipes))
3592                 return;
3593
3594         if (DISPLAY_VER(i915) >= 14)
3595                 val |= MBUS_DBOX_I_CREDIT(2);
3596
3597         if (DISPLAY_VER(i915) >= 12) {
3598                 val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16);
3599                 val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1);
3600                 val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN;
3601         }
3602
3603         if (DISPLAY_VER(i915) >= 14)
3604                 val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(12) :
3605                                                      MBUS_DBOX_A_CREDIT(8);
3606         else if (IS_ALDERLAKE_P(i915))
3607                 /* Wa_22010947358:adl-p */
3608                 val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(6) :
3609                                                      MBUS_DBOX_A_CREDIT(4);
3610         else
3611                 val |= MBUS_DBOX_A_CREDIT(2);
3612
3613         if (DISPLAY_VER(i915) >= 14) {
3614                 val |= MBUS_DBOX_B_CREDIT(0xA);
3615         } else if (IS_ALDERLAKE_P(i915)) {
3616                 val |= MBUS_DBOX_BW_CREDIT(2);
3617                 val |= MBUS_DBOX_B_CREDIT(8);
3618         } else if (DISPLAY_VER(i915) >= 12) {
3619                 val |= MBUS_DBOX_BW_CREDIT(2);
3620                 val |= MBUS_DBOX_B_CREDIT(12);
3621         } else {
3622                 val |= MBUS_DBOX_BW_CREDIT(1);
3623                 val |= MBUS_DBOX_B_CREDIT(8);
3624         }
3625
3626         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
3627                 u32 pipe_val = val;
3628
3629                 if (!new_crtc_state->hw.active)
3630                         continue;
3631
3632                 if (DISPLAY_VER(i915) >= 14) {
3633                         if (xelpdp_is_only_pipe_per_dbuf_bank(crtc->pipe,
3634                                                               new_dbuf_state->active_pipes))
3635                                 pipe_val |= MBUS_DBOX_BW_8CREDITS_MTL;
3636                         else
3637                                 pipe_val |= MBUS_DBOX_BW_4CREDITS_MTL;
3638                 }
3639
3640                 intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe), pipe_val);
3641         }
3642 }
3643
3644 static int skl_watermark_ipc_status_show(struct seq_file *m, void *data)
3645 {
3646         struct drm_i915_private *i915 = m->private;
3647
3648         seq_printf(m, "Isochronous Priority Control: %s\n",
3649                    str_yes_no(skl_watermark_ipc_enabled(i915)));
3650         return 0;
3651 }
3652
3653 static int skl_watermark_ipc_status_open(struct inode *inode, struct file *file)
3654 {
3655         struct drm_i915_private *i915 = inode->i_private;
3656
3657         return single_open(file, skl_watermark_ipc_status_show, i915);
3658 }
3659
3660 static ssize_t skl_watermark_ipc_status_write(struct file *file,
3661                                               const char __user *ubuf,
3662                                               size_t len, loff_t *offp)
3663 {
3664         struct seq_file *m = file->private_data;
3665         struct drm_i915_private *i915 = m->private;
3666         intel_wakeref_t wakeref;
3667         bool enable;
3668         int ret;
3669
3670         ret = kstrtobool_from_user(ubuf, len, &enable);
3671         if (ret < 0)
3672                 return ret;
3673
3674         with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
3675                 if (!skl_watermark_ipc_enabled(i915) && enable)
3676                         drm_info(&i915->drm,
3677                                  "Enabling IPC: WM will be proper only after next commit\n");
3678                 i915->display.wm.ipc_enabled = enable;
3679                 skl_watermark_ipc_update(i915);
3680         }
3681
3682         return len;
3683 }
3684
3685 static const struct file_operations skl_watermark_ipc_status_fops = {
3686         .owner = THIS_MODULE,
3687         .open = skl_watermark_ipc_status_open,
3688         .read = seq_read,
3689         .llseek = seq_lseek,
3690         .release = single_release,
3691         .write = skl_watermark_ipc_status_write
3692 };
3693
3694 static int intel_sagv_status_show(struct seq_file *m, void *unused)
3695 {
3696         struct drm_i915_private *i915 = m->private;
3697         static const char * const sagv_status[] = {
3698                 [I915_SAGV_UNKNOWN] = "unknown",
3699                 [I915_SAGV_DISABLED] = "disabled",
3700                 [I915_SAGV_ENABLED] = "enabled",
3701                 [I915_SAGV_NOT_CONTROLLED] = "not controlled",
3702         };
3703
3704         seq_printf(m, "SAGV available: %s\n", str_yes_no(intel_has_sagv(i915)));
3705         seq_printf(m, "SAGV modparam: %s\n", str_enabled_disabled(i915->params.enable_sagv));
3706         seq_printf(m, "SAGV status: %s\n", sagv_status[i915->display.sagv.status]);
3707         seq_printf(m, "SAGV block time: %d usec\n", i915->display.sagv.block_time_us);
3708
3709         return 0;
3710 }
3711
3712 DEFINE_SHOW_ATTRIBUTE(intel_sagv_status);
3713
3714 void skl_watermark_debugfs_register(struct drm_i915_private *i915)
3715 {
3716         struct drm_minor *minor = i915->drm.primary;
3717
3718         if (HAS_IPC(i915))
3719                 debugfs_create_file("i915_ipc_status", 0644, minor->debugfs_root, i915,
3720                                     &skl_watermark_ipc_status_fops);
3721
3722         if (HAS_SAGV(i915))
3723                 debugfs_create_file("i915_sagv_status", 0444, minor->debugfs_root, i915,
3724                                     &intel_sagv_status_fops);
3725 }
3726
3727 unsigned int skl_watermark_max_latency(struct drm_i915_private *i915)
3728 {
3729         int level;
3730
3731         for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
3732                 unsigned int latency = skl_wm_latency(i915, level, NULL);
3733
3734                 if (latency)
3735                         return latency;
3736         }
3737
3738         return 0;
3739 }