drivers/gpu/drm/i915/display/skl_watermark.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2022 Intel Corporation
   4  */
   5
   6 #include <linux/debugfs.h>
   7
   8 #include <drm/drm_blend.h>
   9
  10 #include "i915_drv.h"
  11 #include "i915_reg.h"
  12 #include "i9xx_wm.h"
  13 #include "intel_atomic.h"
  14 #include "intel_atomic_plane.h"
  15 #include "intel_bw.h"
  16 #include "intel_cdclk.h"
  17 #include "intel_crtc.h"
  18 #include "intel_cursor_regs.h"
  19 #include "intel_de.h"
  20 #include "intel_display.h"
  21 #include "intel_display_power.h"
  22 #include "intel_display_types.h"
  23 #include "intel_fb.h"
  24 #include "intel_fixed.h"
  25 #include "intel_pcode.h"
  26 #include "intel_wm.h"
  27 #include "skl_universal_plane_regs.h"
  28 #include "skl_watermark.h"
  29 #include "skl_watermark_regs.h"
  30
  31 /*It is expected that DSB can do posted writes to every register in
  32  * the pipe and planes within 100us. For flip queue use case, the
  33  * recommended DSB execution time is 100us + one SAGV block time.
  34  */
  35 #define DSB_EXE_TIME 100
  36
  37 static void skl_sagv_disable(struct drm_i915_private *i915);
  38
  39 /* Stores plane specific WM parameters */
  40 struct skl_wm_params {
  41         bool x_tiled, y_tiled;
  42         bool rc_surface;
  43         bool is_planar;
  44         u32 width;
  45         u8 cpp;
  46         u32 plane_pixel_rate;
  47         u32 y_min_scanlines;
  48         u32 plane_bytes_per_line;
  49         uint_fixed_16_16_t plane_blocks_per_line;
  50         uint_fixed_16_16_t y_tile_minimum;
  51         u32 linetime_us;
  52         u32 dbuf_block_size;
  53 };
  54
  55 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *i915)
  56 {
  57         u8 enabled_slices = 0;
  58         enum dbuf_slice slice;
  59
  60         for_each_dbuf_slice(i915, slice) {
  61                 if (intel_de_read(i915, DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
  62                         enabled_slices |= BIT(slice);
  63         }
  64
  65         return enabled_slices;
  66 }
  67
  68 /*
  69  * FIXME: We still don't have the proper code detect if we need to apply the WA,
  70  * so assume we'll always need it in order to avoid underruns.
  71  */
  72 static bool skl_needs_memory_bw_wa(struct drm_i915_private *i915)
  73 {
  74         return DISPLAY_VER(i915) == 9;
  75 }
  76
  77 bool
  78 intel_has_sagv(struct drm_i915_private *i915)
  79 {
  80         return HAS_SAGV(i915) &&
  81                 i915->display.sagv.status != I915_SAGV_NOT_CONTROLLED;
  82 }
  83
  84 static u32
  85 intel_sagv_block_time(struct drm_i915_private *i915)
  86 {
  87         if (DISPLAY_VER(i915) >= 14) {
  88                 u32 val;
  89
  90                 val = intel_de_read(i915, MTL_LATENCY_SAGV);
  91
  92                 return REG_FIELD_GET(MTL_LATENCY_QCLK_SAGV, val);
  93         } else if (DISPLAY_VER(i915) >= 12) {
  94                 u32 val = 0;
  95                 int ret;
  96
  97                 ret = snb_pcode_read(&i915->uncore,
  98                                      GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
  99                                      &val, NULL);
 100                 if (ret) {
 101                         drm_dbg_kms(&i915->drm, "Couldn't read SAGV block time!\n");
 102                         return 0;
 103                 }
 104
 105                 return val;
 106         } else if (DISPLAY_VER(i915) == 11) {
 107                 return 10;
 108         } else if (HAS_SAGV(i915)) {
 109                 return 30;
 110         } else {
 111                 return 0;
 112         }
 113 }
 114
 115 static void intel_sagv_init(struct drm_i915_private *i915)
 116 {
 117         if (!HAS_SAGV(i915))
 118                 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 119
 120         /*
 121          * Probe to see if we have working SAGV control.
 122          * For icl+ this was already determined by intel_bw_init_hw().
 123          */
 124         if (DISPLAY_VER(i915) < 11)
 125                 skl_sagv_disable(i915);
 126
 127         drm_WARN_ON(&i915->drm, i915->display.sagv.status == I915_SAGV_UNKNOWN);
 128
 129         i915->display.sagv.block_time_us = intel_sagv_block_time(i915);
 130
 131         drm_dbg_kms(&i915->drm, "SAGV supported: %s, original SAGV block time: %u us\n",
 132                     str_yes_no(intel_has_sagv(i915)), i915->display.sagv.block_time_us);
 133
 134         /* avoid overflow when adding with wm0 latency/etc. */
 135         if (drm_WARN(&i915->drm, i915->display.sagv.block_time_us > U16_MAX,
 136                      "Excessive SAGV block time %u, ignoring\n",
 137                      i915->display.sagv.block_time_us))
 138                 i915->display.sagv.block_time_us = 0;
 139
 140         if (!intel_has_sagv(i915))
 141                 i915->display.sagv.block_time_us = 0;
 142 }
 143
 144 /*
 145  * SAGV dynamically adjusts the system agent voltage and clock frequencies
 146  * depending on power and performance requirements. The display engine access
 147  * to system memory is blocked during the adjustment time. Because of the
 148  * blocking time, having this enabled can cause full system hangs and/or pipe
 149  * underruns if we don't meet all of the following requirements:
 150  *
 151  *  - <= 1 pipe enabled
 152  *  - All planes can enable watermarks for latencies >= SAGV engine block time
 153  *  - We're not using an interlaced display configuration
 154  */
 155 static void skl_sagv_enable(struct drm_i915_private *i915)
 156 {
 157         int ret;
 158
 159         if (!intel_has_sagv(i915))
 160                 return;
 161
 162         if (i915->display.sagv.status == I915_SAGV_ENABLED)
 163                 return;
 164
 165         drm_dbg_kms(&i915->drm, "Enabling SAGV\n");
 166         ret = snb_pcode_write(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
 167                               GEN9_SAGV_ENABLE);
 168
 169         /* We don't need to wait for SAGV when enabling */
 170
 171         /*
 172          * Some skl systems, pre-release machines in particular,
 173          * don't actually have SAGV.
 174          */
 175         if (IS_SKYLAKE(i915) && ret == -ENXIO) {
 176                 drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
 177                 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 178                 return;
 179         } else if (ret < 0) {
 180                 drm_err(&i915->drm, "Failed to enable SAGV\n");
 181                 return;
 182         }
 183
 184         i915->display.sagv.status = I915_SAGV_ENABLED;
 185 }
 186
 187 static void skl_sagv_disable(struct drm_i915_private *i915)
 188 {
 189         int ret;
 190
 191         if (!intel_has_sagv(i915))
 192                 return;
 193
 194         if (i915->display.sagv.status == I915_SAGV_DISABLED)
 195                 return;
 196
 197         drm_dbg_kms(&i915->drm, "Disabling SAGV\n");
 198         /* bspec says to keep retrying for at least 1 ms */
 199         ret = skl_pcode_request(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
 200                                 GEN9_SAGV_DISABLE,
 201                                 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
 202                                 1);
 203         /*
 204          * Some skl systems, pre-release machines in particular,
 205          * don't actually have SAGV.
 206          */
 207         if (IS_SKYLAKE(i915) && ret == -ENXIO) {
 208                 drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
 209                 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 210                 return;
 211         } else if (ret < 0) {
 212                 drm_err(&i915->drm, "Failed to disable SAGV (%d)\n", ret);
 213                 return;
 214         }
 215
 216         i915->display.sagv.status = I915_SAGV_DISABLED;
 217 }
 218
 219 static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
 220 {
 221         struct drm_i915_private *i915 = to_i915(state->base.dev);
 222         const struct intel_bw_state *new_bw_state =
 223                 intel_atomic_get_new_bw_state(state);
 224
 225         if (!new_bw_state)
 226                 return;
 227
 228         if (!intel_can_enable_sagv(i915, new_bw_state))
 229                 skl_sagv_disable(i915);
 230 }
 231
 232 static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
 233 {
 234         struct drm_i915_private *i915 = to_i915(state->base.dev);
 235         const struct intel_bw_state *new_bw_state =
 236                 intel_atomic_get_new_bw_state(state);
 237
 238         if (!new_bw_state)
 239                 return;
 240
 241         if (intel_can_enable_sagv(i915, new_bw_state))
 242                 skl_sagv_enable(i915);
 243 }
 244
 245 static void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
 246 {
 247         struct drm_i915_private *i915 = to_i915(state->base.dev);
 248         const struct intel_bw_state *old_bw_state =
 249                 intel_atomic_get_old_bw_state(state);
 250         const struct intel_bw_state *new_bw_state =
 251                 intel_atomic_get_new_bw_state(state);
 252         u16 old_mask, new_mask;
 253
 254         if (!new_bw_state)
 255                 return;
 256
 257         old_mask = old_bw_state->qgv_points_mask;
 258         new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
 259
 260         if (old_mask == new_mask)
 261                 return;
 262
 263         WARN_ON(!new_bw_state->base.changed);
 264
 265         drm_dbg_kms(&i915->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
 266                     old_mask, new_mask);
 267
 268         /*
 269          * Restrict required qgv points before updating the configuration.
 270          * According to BSpec we can't mask and unmask qgv points at the same
 271          * time. Also masking should be done before updating the configuration
 272          * and unmasking afterwards.
 273          */
 274         icl_pcode_restrict_qgv_points(i915, new_mask);
 275 }
 276
 277 static void icl_sagv_post_plane_update(struct intel_atomic_state *state)
 278 {
 279         struct drm_i915_private *i915 = to_i915(state->base.dev);
 280         const struct intel_bw_state *old_bw_state =
 281                 intel_atomic_get_old_bw_state(state);
 282         const struct intel_bw_state *new_bw_state =
 283                 intel_atomic_get_new_bw_state(state);
 284         u16 old_mask, new_mask;
 285
 286         if (!new_bw_state)
 287                 return;
 288
 289         old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
 290         new_mask = new_bw_state->qgv_points_mask;
 291
 292         if (old_mask == new_mask)
 293                 return;
 294
 295         WARN_ON(!new_bw_state->base.changed);
 296
 297         drm_dbg_kms(&i915->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
 298                     old_mask, new_mask);
 299
 300         /*
 301          * Allow required qgv points after updating the configuration.
 302          * According to BSpec we can't mask and unmask qgv points at the same
 303          * time. Also masking should be done before updating the configuration
 304          * and unmasking afterwards.
 305          */
 306         icl_pcode_restrict_qgv_points(i915, new_mask);
 307 }
 308
 309 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
 310 {
 311         struct drm_i915_private *i915 = to_i915(state->base.dev);
 312
 313         /*
 314          * Just return if we can't control SAGV or don't have it.
 315          * This is different from situation when we have SAGV but just can't
 316          * afford it due to DBuf limitation - in case if SAGV is completely
 317          * disabled in a BIOS, we are not even allowed to send a PCode request,
 318          * as it will throw an error. So have to check it here.
 319          */
 320         if (!intel_has_sagv(i915))
 321                 return;
 322
 323         if (DISPLAY_VER(i915) >= 11)
 324                 icl_sagv_pre_plane_update(state);
 325         else
 326                 skl_sagv_pre_plane_update(state);
 327 }
 328
 329 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
 330 {
 331         struct drm_i915_private *i915 = to_i915(state->base.dev);
 332
 333         /*
 334          * Just return if we can't control SAGV or don't have it.
 335          * This is different from situation when we have SAGV but just can't
 336          * afford it due to DBuf limitation - in case if SAGV is completely
 337          * disabled in a BIOS, we are not even allowed to send a PCode request,
 338          * as it will throw an error. So have to check it here.
 339          */
 340         if (!intel_has_sagv(i915))
 341                 return;
 342
 343         if (DISPLAY_VER(i915) >= 11)
 344                 icl_sagv_post_plane_update(state);
 345         else
 346                 skl_sagv_post_plane_update(state);
 347 }
 348
 349 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 350 {
 351         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 352         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 353         enum plane_id plane_id;
 354         int max_level = INT_MAX;
 355
 356         if (!intel_has_sagv(i915))
 357                 return false;
 358
 359         if (!crtc_state->hw.active)
 360                 return true;
 361
 362         if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
 363                 return false;
 364
 365         for_each_plane_id_on_crtc(crtc, plane_id) {
 366                 const struct skl_plane_wm *wm =
 367                         &crtc_state->wm.skl.optimal.planes[plane_id];
 368                 int level;
 369
 370                 /* Skip this plane if it's not enabled */
 371                 if (!wm->wm[0].enable)
 372                         continue;
 373
 374                 /* Find the highest enabled wm level for this plane */
 375                 for (level = i915->display.wm.num_levels - 1;
 376                      !wm->wm[level].enable; --level)
 377                      { }
 378
 379                 /* Highest common enabled wm level for all planes */
 380                 max_level = min(level, max_level);
 381         }
 382
 383         /* No enabled planes? */
 384         if (max_level == INT_MAX)
 385                 return true;
 386
 387         for_each_plane_id_on_crtc(crtc, plane_id) {
 388                 const struct skl_plane_wm *wm =
 389                         &crtc_state->wm.skl.optimal.planes[plane_id];
 390
 391                 /*
 392                  * All enabled planes must have enabled a common wm level that
 393                  * can tolerate memory latencies higher than sagv_block_time_us
 394                  */
 395                 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
 396                         return false;
 397         }
 398
 399         return true;
 400 }
 401
 402 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 403 {
 404         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 405         enum plane_id plane_id;
 406
 407         if (!crtc_state->hw.active)
 408                 return true;
 409
 410         for_each_plane_id_on_crtc(crtc, plane_id) {
 411                 const struct skl_plane_wm *wm =
 412                         &crtc_state->wm.skl.optimal.planes[plane_id];
 413
 414                 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
 415                         return false;
 416         }
 417
 418         return true;
 419 }
 420
 421 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 422 {
 423         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 424         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 425
 426         if (!i915->display.params.enable_sagv)
 427                 return false;
 428
 429         if (DISPLAY_VER(i915) >= 12)
 430                 return tgl_crtc_can_enable_sagv(crtc_state);
 431         else
 432                 return skl_crtc_can_enable_sagv(crtc_state);
 433 }
 434
 435 bool intel_can_enable_sagv(struct drm_i915_private *i915,
 436                            const struct intel_bw_state *bw_state)
 437 {
 438         if (DISPLAY_VER(i915) < 11 &&
 439             bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
 440                 return false;
 441
 442         return bw_state->pipe_sagv_reject == 0;
 443 }
 444
 445 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
 446 {
 447         struct drm_i915_private *i915 = to_i915(state->base.dev);
 448         int ret;
 449         struct intel_crtc *crtc;
 450         struct intel_crtc_state *new_crtc_state;
 451         struct intel_bw_state *new_bw_state = NULL;
 452         const struct intel_bw_state *old_bw_state = NULL;
 453         int i;
 454
 455         for_each_new_intel_crtc_in_state(state, crtc,
 456                                          new_crtc_state, i) {
 457                 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
 458
 459                 new_bw_state = intel_atomic_get_bw_state(state);
 460                 if (IS_ERR(new_bw_state))
 461                         return PTR_ERR(new_bw_state);
 462
 463                 old_bw_state = intel_atomic_get_old_bw_state(state);
 464
 465                 /*
 466                  * We store use_sagv_wm in the crtc state rather than relying on
 467                  * that bw state since we have no convenient way to get at the
 468                  * latter from the plane commit hooks (especially in the legacy
 469                  * cursor case).
 470                  *
 471                  * drm_atomic_check_only() gets upset if we pull more crtcs
 472                  * into the state, so we have to calculate this based on the
 473                  * individual intel_crtc_can_enable_sagv() rather than
 474                  * the overall intel_can_enable_sagv(). Otherwise the
 475                  * crtcs not included in the commit would not switch to the
 476                  * SAGV watermarks when we are about to enable SAGV, and that
 477                  * would lead to underruns. This does mean extra power draw
 478                  * when only a subset of the crtcs are blocking SAGV as the
 479                  * other crtcs can't be allowed to use the more optimal
 480                  * normal (ie. non-SAGV) watermarks.
 481                  */
 482                 pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(i915) &&
 483                         DISPLAY_VER(i915) >= 12 &&
 484                         intel_crtc_can_enable_sagv(new_crtc_state);
 485
 486                 if (intel_crtc_can_enable_sagv(new_crtc_state))
 487                         new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
 488                 else
 489                         new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
 490         }
 491
 492         if (!new_bw_state)
 493                 return 0;
 494
 495         new_bw_state->active_pipes =
 496                 intel_calc_active_pipes(state, old_bw_state->active_pipes);
 497
 498         if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
 499                 ret = intel_atomic_lock_global_state(&new_bw_state->base);
 500                 if (ret)
 501                         return ret;
 502         }
 503
 504         if (intel_can_enable_sagv(i915, new_bw_state) !=
 505             intel_can_enable_sagv(i915, old_bw_state)) {
 506                 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
 507                 if (ret)
 508                         return ret;
 509         } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
 510                 ret = intel_atomic_lock_global_state(&new_bw_state->base);
 511                 if (ret)
 512                         return ret;
 513         }
 514
 515         return 0;
 516 }
 517
 518 static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
 519                               u16 start, u16 end)
 520 {
 521         entry->start = start;
 522         entry->end = end;
 523
 524         return end;
 525 }
 526
 527 static int intel_dbuf_slice_size(struct drm_i915_private *i915)
 528 {
 529         return DISPLAY_INFO(i915)->dbuf.size /
 530                 hweight8(DISPLAY_INFO(i915)->dbuf.slice_mask);
 531 }
 532
 533 static void
 534 skl_ddb_entry_for_slices(struct drm_i915_private *i915, u8 slice_mask,
 535                          struct skl_ddb_entry *ddb)
 536 {
 537         int slice_size = intel_dbuf_slice_size(i915);
 538
 539         if (!slice_mask) {
 540                 ddb->start = 0;
 541                 ddb->end = 0;
 542                 return;
 543         }
 544
 545         ddb->start = (ffs(slice_mask) - 1) * slice_size;
 546         ddb->end = fls(slice_mask) * slice_size;
 547
 548         WARN_ON(ddb->start >= ddb->end);
 549         WARN_ON(ddb->end > DISPLAY_INFO(i915)->dbuf.size);
 550 }
 551
 552 static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
 553 {
 554         struct skl_ddb_entry ddb;
 555
 556         if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
 557                 slice_mask = BIT(DBUF_S1);
 558         else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
 559                 slice_mask = BIT(DBUF_S3);
 560
 561         skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
 562
 563         return ddb.start;
 564 }
 565
 566 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *i915,
 567                             const struct skl_ddb_entry *entry)
 568 {
 569         int slice_size = intel_dbuf_slice_size(i915);
 570         enum dbuf_slice start_slice, end_slice;
 571         u8 slice_mask = 0;
 572
 573         if (!skl_ddb_entry_size(entry))
 574                 return 0;
 575
 576         start_slice = entry->start / slice_size;
 577         end_slice = (entry->end - 1) / slice_size;
 578
 579         /*
 580          * Per plane DDB entry can in a really worst case be on multiple slices
 581          * but single entry is anyway contigious.
 582          */
 583         while (start_slice <= end_slice) {
 584                 slice_mask |= BIT(start_slice);
 585                 start_slice++;
 586         }
 587
 588         return slice_mask;
 589 }
 590
 591 static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
 592 {
 593         const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
 594         int hdisplay, vdisplay;
 595
 596         if (!crtc_state->hw.active)
 597                 return 0;
 598
 599         /*
 600          * Watermark/ddb requirement highly depends upon width of the
 601          * framebuffer, So instead of allocating DDB equally among pipes
 602          * distribute DDB based on resolution/width of the display.
 603          */
 604         drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
 605
 606         return hdisplay;
 607 }
 608
 609 static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
 610                                     enum pipe for_pipe,
 611                                     unsigned int *weight_start,
 612                                     unsigned int *weight_end,
 613                                     unsigned int *weight_total)
 614 {
 615         struct drm_i915_private *i915 =
 616                 to_i915(dbuf_state->base.state->base.dev);
 617         enum pipe pipe;
 618
 619         *weight_start = 0;
 620         *weight_end = 0;
 621         *weight_total = 0;
 622
 623         for_each_pipe(i915, pipe) {
 624                 int weight = dbuf_state->weight[pipe];
 625
 626                 /*
 627                  * Do not account pipes using other slice sets
 628                  * luckily as of current BSpec slice sets do not partially
 629                  * intersect(pipes share either same one slice or same slice set
 630                  * i.e no partial intersection), so it is enough to check for
 631                  * equality for now.
 632                  */
 633                 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
 634                         continue;
 635
 636                 *weight_total += weight;
 637                 if (pipe < for_pipe) {
 638                         *weight_start += weight;
 639                         *weight_end += weight;
 640                 } else if (pipe == for_pipe) {
 641                         *weight_end += weight;
 642                 }
 643         }
 644 }
 645
 646 static int
 647 skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
 648 {
 649         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 650         unsigned int weight_total, weight_start, weight_end;
 651         const struct intel_dbuf_state *old_dbuf_state =
 652                 intel_atomic_get_old_dbuf_state(state);
 653         struct intel_dbuf_state *new_dbuf_state =
 654                 intel_atomic_get_new_dbuf_state(state);
 655         struct intel_crtc_state *crtc_state;
 656         struct skl_ddb_entry ddb_slices;
 657         enum pipe pipe = crtc->pipe;
 658         unsigned int mbus_offset = 0;
 659         u32 ddb_range_size;
 660         u32 dbuf_slice_mask;
 661         u32 start, end;
 662         int ret;
 663
 664         if (new_dbuf_state->weight[pipe] == 0) {
 665                 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0);
 666                 goto out;
 667         }
 668
 669         dbuf_slice_mask = new_dbuf_state->slices[pipe];
 670
 671         skl_ddb_entry_for_slices(i915, dbuf_slice_mask, &ddb_slices);
 672         mbus_offset = mbus_ddb_offset(i915, dbuf_slice_mask);
 673         ddb_range_size = skl_ddb_entry_size(&ddb_slices);
 674
 675         intel_crtc_dbuf_weights(new_dbuf_state, pipe,
 676                                 &weight_start, &weight_end, &weight_total);
 677
 678         start = ddb_range_size * weight_start / weight_total;
 679         end = ddb_range_size * weight_end / weight_total;
 680
 681         skl_ddb_entry_init(&new_dbuf_state->ddb[pipe],
 682                            ddb_slices.start - mbus_offset + start,
 683                            ddb_slices.start - mbus_offset + end);
 684
 685 out:
 686         if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
 687             skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
 688                                 &new_dbuf_state->ddb[pipe]))
 689                 return 0;
 690
 691         ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
 692         if (ret)
 693                 return ret;
 694
 695         crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
 696         if (IS_ERR(crtc_state))
 697                 return PTR_ERR(crtc_state);
 698
 699         /*
 700          * Used for checking overlaps, so we need absolute
 701          * offsets instead of MBUS relative offsets.
 702          */
 703         crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
 704         crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
 705
 706         drm_dbg_kms(&i915->drm,
 707                     "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
 708                     crtc->base.base.id, crtc->base.name,
 709                     old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
 710                     old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
 711                     new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
 712                     old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
 713
 714         return 0;
 715 }
 716
 717 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
 718                                  int width, const struct drm_format_info *format,
 719                                  u64 modifier, unsigned int rotation,
 720                                  u32 plane_pixel_rate, struct skl_wm_params *wp,
 721                                  int color_plane, unsigned int pan_x);
 722
 723 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
 724                                  struct intel_plane *plane,
 725                                  int level,
 726                                  unsigned int latency,
 727                                  const struct skl_wm_params *wp,
 728                                  const struct skl_wm_level *result_prev,
 729                                  struct skl_wm_level *result /* out */);
 730
 731 static unsigned int skl_wm_latency(struct drm_i915_private *i915, int level,
 732                                    const struct skl_wm_params *wp)
 733 {
 734         unsigned int latency = i915->display.wm.skl_latency[level];
 735
 736         if (latency == 0)
 737                 return 0;
 738
 739         /*
 740          * WaIncreaseLatencyIPCEnabled: kbl,cfl
 741          * Display WA #1141: kbl,cfl
 742          */
 743         if ((IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) &&
 744             skl_watermark_ipc_enabled(i915))
 745                 latency += 4;
 746
 747         if (skl_needs_memory_bw_wa(i915) && wp && wp->x_tiled)
 748                 latency += 15;
 749
 750         return latency;
 751 }
 752
 753 static unsigned int
 754 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
 755                       int num_active)
 756 {
 757         struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
 758         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
 759         struct skl_wm_level wm = {};
 760         int ret, min_ddb_alloc = 0;
 761         struct skl_wm_params wp;
 762         int level;
 763
 764         ret = skl_compute_wm_params(crtc_state, 256,
 765                                     drm_format_info(DRM_FORMAT_ARGB8888),
 766                                     DRM_FORMAT_MOD_LINEAR,
 767                                     DRM_MODE_ROTATE_0,
 768                                     crtc_state->pixel_rate, &wp, 0, 0);
 769         drm_WARN_ON(&i915->drm, ret);
 770
 771         for (level = 0; level < i915->display.wm.num_levels; level++) {
 772                 unsigned int latency = skl_wm_latency(i915, level, &wp);
 773
 774                 skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm);
 775                 if (wm.min_ddb_alloc == U16_MAX)
 776                         break;
 777
 778                 min_ddb_alloc = wm.min_ddb_alloc;
 779         }
 780
 781         return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
 782 }
 783
 784 static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
 785 {
 786         skl_ddb_entry_init(entry,
 787                            REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
 788                            REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
 789         if (entry->end)
 790                 entry->end++;
 791 }
 792
 793 static void
 794 skl_ddb_get_hw_plane_state(struct drm_i915_private *i915,
 795                            const enum pipe pipe,
 796                            const enum plane_id plane_id,
 797                            struct skl_ddb_entry *ddb,
 798                            struct skl_ddb_entry *ddb_y)
 799 {
 800         u32 val;
 801
 802         /* Cursor doesn't support NV12/planar, so no extra calculation needed */
 803         if (plane_id == PLANE_CURSOR) {
 804                 val = intel_de_read(i915, CUR_BUF_CFG(pipe));
 805                 skl_ddb_entry_init_from_hw(ddb, val);
 806                 return;
 807         }
 808
 809         val = intel_de_read(i915, PLANE_BUF_CFG(pipe, plane_id));
 810         skl_ddb_entry_init_from_hw(ddb, val);
 811
 812         if (DISPLAY_VER(i915) >= 11)
 813                 return;
 814
 815         val = intel_de_read(i915, PLANE_NV12_BUF_CFG(pipe, plane_id));
 816         skl_ddb_entry_init_from_hw(ddb_y, val);
 817 }
 818
 819 static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
 820                                       struct skl_ddb_entry *ddb,
 821                                       struct skl_ddb_entry *ddb_y)
 822 {
 823         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 824         enum intel_display_power_domain power_domain;
 825         enum pipe pipe = crtc->pipe;
 826         intel_wakeref_t wakeref;
 827         enum plane_id plane_id;
 828
 829         power_domain = POWER_DOMAIN_PIPE(pipe);
 830         wakeref = intel_display_power_get_if_enabled(i915, power_domain);
 831         if (!wakeref)
 832                 return;
 833
 834         for_each_plane_id_on_crtc(crtc, plane_id)
 835                 skl_ddb_get_hw_plane_state(i915, pipe,
 836                                            plane_id,
 837                                            &ddb[plane_id],
 838                                            &ddb_y[plane_id]);
 839
 840         intel_display_power_put(i915, power_domain, wakeref);
 841 }
 842
 843 struct dbuf_slice_conf_entry {
 844         u8 active_pipes;
 845         u8 dbuf_mask[I915_MAX_PIPES];
 846         bool join_mbus;
 847 };
 848
 849 /*
 850  * Table taken from Bspec 12716
 851  * Pipes do have some preferred DBuf slice affinity,
 852  * plus there are some hardcoded requirements on how
 853  * those should be distributed for multipipe scenarios.
 854  * For more DBuf slices algorithm can get even more messy
 855  * and less readable, so decided to use a table almost
 856  * as is from BSpec itself - that way it is at least easier
 857  * to compare, change and check.
 858  */
 859 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
 860 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
 861 {
 862         {
 863                 .active_pipes = BIT(PIPE_A),
 864                 .dbuf_mask = {
 865                         [PIPE_A] = BIT(DBUF_S1),
 866                 },
 867         },
 868         {
 869                 .active_pipes = BIT(PIPE_B),
 870                 .dbuf_mask = {
 871                         [PIPE_B] = BIT(DBUF_S1),
 872                 },
 873         },
 874         {
 875                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
 876                 .dbuf_mask = {
 877                         [PIPE_A] = BIT(DBUF_S1),
 878                         [PIPE_B] = BIT(DBUF_S2),
 879                 },
 880         },
 881         {
 882                 .active_pipes = BIT(PIPE_C),
 883                 .dbuf_mask = {
 884                         [PIPE_C] = BIT(DBUF_S2),
 885                 },
 886         },
 887         {
 888                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
 889                 .dbuf_mask = {
 890                         [PIPE_A] = BIT(DBUF_S1),
 891                         [PIPE_C] = BIT(DBUF_S2),
 892                 },
 893         },
 894         {
 895                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
 896                 .dbuf_mask = {
 897                         [PIPE_B] = BIT(DBUF_S1),
 898                         [PIPE_C] = BIT(DBUF_S2),
 899                 },
 900         },
 901         {
 902                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
 903                 .dbuf_mask = {
 904                         [PIPE_A] = BIT(DBUF_S1),
 905                         [PIPE_B] = BIT(DBUF_S1),
 906                         [PIPE_C] = BIT(DBUF_S2),
 907                 },
 908         },
 909         {}
 910 };
 911
 912 /*
 913  * Table taken from Bspec 49255
 914  * Pipes do have some preferred DBuf slice affinity,
 915  * plus there are some hardcoded requirements on how
 916  * those should be distributed for multipipe scenarios.
 917  * For more DBuf slices algorithm can get even more messy
 918  * and less readable, so decided to use a table almost
 919  * as is from BSpec itself - that way it is at least easier
 920  * to compare, change and check.
 921  */
 922 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
 923 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
 924 {
 925         {
 926                 .active_pipes = BIT(PIPE_A),
 927                 .dbuf_mask = {
 928                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
 929                 },
 930         },
 931         {
 932                 .active_pipes = BIT(PIPE_B),
 933                 .dbuf_mask = {
 934                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
 935                 },
 936         },
 937         {
 938                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
 939                 .dbuf_mask = {
 940                         [PIPE_A] = BIT(DBUF_S2),
 941                         [PIPE_B] = BIT(DBUF_S1),
 942                 },
 943         },
 944         {
 945                 .active_pipes = BIT(PIPE_C),
 946                 .dbuf_mask = {
 947                         [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
 948                 },
 949         },
 950         {
 951                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
 952                 .dbuf_mask = {
 953                         [PIPE_A] = BIT(DBUF_S1),
 954                         [PIPE_C] = BIT(DBUF_S2),
 955                 },
 956         },
 957         {
 958                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
 959                 .dbuf_mask = {
 960                         [PIPE_B] = BIT(DBUF_S1),
 961                         [PIPE_C] = BIT(DBUF_S2),
 962                 },
 963         },
 964         {
 965                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
 966                 .dbuf_mask = {
 967                         [PIPE_A] = BIT(DBUF_S1),
 968                         [PIPE_B] = BIT(DBUF_S1),
 969                         [PIPE_C] = BIT(DBUF_S2),
 970                 },
 971         },
 972         {
 973                 .active_pipes = BIT(PIPE_D),
 974                 .dbuf_mask = {
 975                         [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
 976                 },
 977         },
 978         {
 979                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
 980                 .dbuf_mask = {
 981                         [PIPE_A] = BIT(DBUF_S1),
 982                         [PIPE_D] = BIT(DBUF_S2),
 983                 },
 984         },
 985         {
 986                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
 987                 .dbuf_mask = {
 988                         [PIPE_B] = BIT(DBUF_S1),
 989                         [PIPE_D] = BIT(DBUF_S2),
 990                 },
 991         },
 992         {
 993                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
 994                 .dbuf_mask = {
 995                         [PIPE_A] = BIT(DBUF_S1),
 996                         [PIPE_B] = BIT(DBUF_S1),
 997                         [PIPE_D] = BIT(DBUF_S2),
 998                 },
 999         },
1000         {
1001                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1002                 .dbuf_mask = {
1003                         [PIPE_C] = BIT(DBUF_S1),
1004                         [PIPE_D] = BIT(DBUF_S2),
1005                 },
1006         },
1007         {
1008                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1009                 .dbuf_mask = {
1010                         [PIPE_A] = BIT(DBUF_S1),
1011                         [PIPE_C] = BIT(DBUF_S2),
1012                         [PIPE_D] = BIT(DBUF_S2),
1013                 },
1014         },
1015         {
1016                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1017                 .dbuf_mask = {
1018                         [PIPE_B] = BIT(DBUF_S1),
1019                         [PIPE_C] = BIT(DBUF_S2),
1020                         [PIPE_D] = BIT(DBUF_S2),
1021                 },
1022         },
1023         {
1024                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1025                 .dbuf_mask = {
1026                         [PIPE_A] = BIT(DBUF_S1),
1027                         [PIPE_B] = BIT(DBUF_S1),
1028                         [PIPE_C] = BIT(DBUF_S2),
1029                         [PIPE_D] = BIT(DBUF_S2),
1030                 },
1031         },
1032         {}
1033 };
1034
1035 static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
1036         {
1037                 .active_pipes = BIT(PIPE_A),
1038                 .dbuf_mask = {
1039                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1040                 },
1041         },
1042         {
1043                 .active_pipes = BIT(PIPE_B),
1044                 .dbuf_mask = {
1045                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1046                 },
1047         },
1048         {
1049                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1050                 .dbuf_mask = {
1051                         [PIPE_A] = BIT(DBUF_S1),
1052                         [PIPE_B] = BIT(DBUF_S2),
1053                 },
1054         },
1055         {
1056                 .active_pipes = BIT(PIPE_C),
1057                 .dbuf_mask = {
1058                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1059                 },
1060         },
1061         {
1062                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1063                 .dbuf_mask = {
1064                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1065                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1066                 },
1067         },
1068         {
1069                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1070                 .dbuf_mask = {
1071                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1072                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1073                 },
1074         },
1075         {
1076                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1077                 .dbuf_mask = {
1078                         [PIPE_A] = BIT(DBUF_S1),
1079                         [PIPE_B] = BIT(DBUF_S2),
1080                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1081                 },
1082         },
1083         {
1084                 .active_pipes = BIT(PIPE_D),
1085                 .dbuf_mask = {
1086                         [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1087                 },
1088         },
1089         {
1090                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1091                 .dbuf_mask = {
1092                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1093                         [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1094                 },
1095         },
1096         {
1097                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1098                 .dbuf_mask = {
1099                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1100                         [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1101                 },
1102         },
1103         {
1104                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1105                 .dbuf_mask = {
1106                         [PIPE_A] = BIT(DBUF_S1),
1107                         [PIPE_B] = BIT(DBUF_S2),
1108                         [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1109                 },
1110         },
1111         {
1112                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1113                 .dbuf_mask = {
1114                         [PIPE_C] = BIT(DBUF_S3),
1115                         [PIPE_D] = BIT(DBUF_S4),
1116                 },
1117         },
1118         {
1119                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1120                 .dbuf_mask = {
1121                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1122                         [PIPE_C] = BIT(DBUF_S3),
1123                         [PIPE_D] = BIT(DBUF_S4),
1124                 },
1125         },
1126         {
1127                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1128                 .dbuf_mask = {
1129                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1130                         [PIPE_C] = BIT(DBUF_S3),
1131                         [PIPE_D] = BIT(DBUF_S4),
1132                 },
1133         },
1134         {
1135                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1136                 .dbuf_mask = {
1137                         [PIPE_A] = BIT(DBUF_S1),
1138                         [PIPE_B] = BIT(DBUF_S2),
1139                         [PIPE_C] = BIT(DBUF_S3),
1140                         [PIPE_D] = BIT(DBUF_S4),
1141                 },
1142         },
1143         {}
1144 };
1145
1146 static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
1147         /*
1148          * Keep the join_mbus cases first so check_mbus_joined()
1149          * will prefer them over the !join_mbus cases.
1150          */
1151         {
1152                 .active_pipes = BIT(PIPE_A),
1153                 .dbuf_mask = {
1154                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1155                 },
1156                 .join_mbus = true,
1157         },
1158         {
1159                 .active_pipes = BIT(PIPE_B),
1160                 .dbuf_mask = {
1161                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1162                 },
1163                 .join_mbus = true,
1164         },
1165         {
1166                 .active_pipes = BIT(PIPE_A),
1167                 .dbuf_mask = {
1168                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1169                 },
1170                 .join_mbus = false,
1171         },
1172         {
1173                 .active_pipes = BIT(PIPE_B),
1174                 .dbuf_mask = {
1175                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1176                 },
1177                 .join_mbus = false,
1178         },
1179         {
1180                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1181                 .dbuf_mask = {
1182                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1183                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1184                 },
1185         },
1186         {
1187                 .active_pipes = BIT(PIPE_C),
1188                 .dbuf_mask = {
1189                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1190                 },
1191         },
1192         {
1193                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1194                 .dbuf_mask = {
1195                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1196                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1197                 },
1198         },
1199         {
1200                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1201                 .dbuf_mask = {
1202                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1203                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1204                 },
1205         },
1206         {
1207                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1208                 .dbuf_mask = {
1209                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1210                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1211                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1212                 },
1213         },
1214         {
1215                 .active_pipes = BIT(PIPE_D),
1216                 .dbuf_mask = {
1217                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1218                 },
1219         },
1220         {
1221                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1222                 .dbuf_mask = {
1223                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1224                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1225                 },
1226         },
1227         {
1228                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1229                 .dbuf_mask = {
1230                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1231                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1232                 },
1233         },
1234         {
1235                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1236                 .dbuf_mask = {
1237                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1238                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1239                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1240                 },
1241         },
1242         {
1243                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1244                 .dbuf_mask = {
1245                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1246                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1247                 },
1248         },
1249         {
1250                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1251                 .dbuf_mask = {
1252                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1253                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1254                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1255                 },
1256         },
1257         {
1258                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1259                 .dbuf_mask = {
1260                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1261                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1262                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1263                 },
1264         },
1265         {
1266                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1267                 .dbuf_mask = {
1268                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1269                         [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1270                         [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1271                         [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1272                 },
1273         },
1274         {}
1275
1276 };
1277
1278 static bool check_mbus_joined(u8 active_pipes,
1279                               const struct dbuf_slice_conf_entry *dbuf_slices)
1280 {
1281         int i;
1282
1283         for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1284                 if (dbuf_slices[i].active_pipes == active_pipes)
1285                         return dbuf_slices[i].join_mbus;
1286         }
1287         return false;
1288 }
1289
1290 static bool adlp_check_mbus_joined(u8 active_pipes)
1291 {
1292         return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
1293 }
1294
1295 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
1296                               const struct dbuf_slice_conf_entry *dbuf_slices)
1297 {
1298         int i;
1299
1300         for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1301                 if (dbuf_slices[i].active_pipes == active_pipes &&
1302                     dbuf_slices[i].join_mbus == join_mbus)
1303                         return dbuf_slices[i].dbuf_mask[pipe];
1304         }
1305         return 0;
1306 }
1307
1308 /*
1309  * This function finds an entry with same enabled pipe configuration and
1310  * returns correspondent DBuf slice mask as stated in BSpec for particular
1311  * platform.
1312  */
1313 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1314 {
1315         /*
1316          * FIXME: For ICL this is still a bit unclear as prev BSpec revision
1317          * required calculating "pipe ratio" in order to determine
1318          * if one or two slices can be used for single pipe configurations
1319          * as additional constraint to the existing table.
1320          * However based on recent info, it should be not "pipe ratio"
1321          * but rather ratio between pixel_rate and cdclk with additional
1322          * constants, so for now we are using only table until this is
1323          * clarified. Also this is the reason why crtc_state param is
1324          * still here - we will need it once those additional constraints
1325          * pop up.
1326          */
1327         return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1328                                    icl_allowed_dbufs);
1329 }
1330
1331 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1332 {
1333         return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1334                                    tgl_allowed_dbufs);
1335 }
1336
1337 static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1338 {
1339         return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1340                                    adlp_allowed_dbufs);
1341 }
1342
1343 static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1344 {
1345         return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1346                                    dg2_allowed_dbufs);
1347 }
1348
1349 static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
1350 {
1351         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1352         enum pipe pipe = crtc->pipe;
1353
1354         if (IS_DG2(i915))
1355                 return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1356         else if (DISPLAY_VER(i915) >= 13)
1357                 return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1358         else if (DISPLAY_VER(i915) == 12)
1359                 return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1360         else if (DISPLAY_VER(i915) == 11)
1361                 return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1362         /*
1363          * For anything else just return one slice yet.
1364          * Should be extended for other platforms.
1365          */
1366         return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
1367 }
1368
1369 static bool
1370 use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
1371                      struct intel_plane *plane)
1372 {
1373         struct drm_i915_private *i915 = to_i915(plane->base.dev);
1374
1375         return DISPLAY_VER(i915) >= 13 &&
1376                crtc_state->uapi.async_flip &&
1377                plane->async_flip;
1378 }
1379
1380 static u64
1381 skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state)
1382 {
1383         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1384         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1385         enum plane_id plane_id;
1386         u64 data_rate = 0;
1387
1388         for_each_plane_id_on_crtc(crtc, plane_id) {
1389                 if (plane_id == PLANE_CURSOR)
1390                         continue;
1391
1392                 data_rate += crtc_state->rel_data_rate[plane_id];
1393
1394                 if (DISPLAY_VER(i915) < 11)
1395                         data_rate += crtc_state->rel_data_rate_y[plane_id];
1396         }
1397
1398         return data_rate;
1399 }
1400
1401 const struct skl_wm_level *
1402 skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
1403                    enum plane_id plane_id,
1404                    int level)
1405 {
1406         const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1407
1408         if (level == 0 && pipe_wm->use_sagv_wm)
1409                 return &wm->sagv.wm0;
1410
1411         return &wm->wm[level];
1412 }
1413
1414 const struct skl_wm_level *
1415 skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
1416                    enum plane_id plane_id)
1417 {
1418         const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1419
1420         if (pipe_wm->use_sagv_wm)
1421                 return &wm->sagv.trans_wm;
1422
1423         return &wm->trans_wm;
1424 }
1425
1426 /*
1427  * We only disable the watermarks for each plane if
1428  * they exceed the ddb allocation of said plane. This
1429  * is done so that we don't end up touching cursor
1430  * watermarks needlessly when some other plane reduces
1431  * our max possible watermark level.
1432  *
1433  * Bspec has this to say about the PLANE_WM enable bit:
1434  * "All the watermarks at this level for all enabled
1435  *  planes must be enabled before the level will be used."
1436  * So this is actually safe to do.
1437  */
1438 static void
1439 skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb)
1440 {
1441         if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb))
1442                 memset(wm, 0, sizeof(*wm));
1443 }
1444
1445 static void
1446 skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
1447                         const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb)
1448 {
1449         if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb_y) ||
1450             uv_wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1451                 memset(wm, 0, sizeof(*wm));
1452                 memset(uv_wm, 0, sizeof(*uv_wm));
1453         }
1454 }
1455
1456 static bool skl_need_wm_copy_wa(struct drm_i915_private *i915, int level,
1457                                 const struct skl_plane_wm *wm)
1458 {
1459         /*
1460          * Wa_1408961008:icl, ehl
1461          * Wa_14012656716:tgl, adl
1462          * Wa_14017887344:icl
1463          * Wa_14017868169:adl, tgl
1464          * Due to some power saving optimizations, different subsystems
1465          * like PSR, might still use even disabled wm level registers,
1466          * for "reference", so lets keep at least the values sane.
1467          * Considering amount of WA requiring us to do similar things, was
1468          * decided to simply do it for all of the platforms, as those wm
1469          * levels are disabled, this isn't going to do harm anyway.
1470          */
1471         return level > 0 && !wm->wm[level].enable;
1472 }
1473
1474 struct skl_plane_ddb_iter {
1475         u64 data_rate;
1476         u16 start, size;
1477 };
1478
1479 static void
1480 skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
1481                        struct skl_ddb_entry *ddb,
1482                        const struct skl_wm_level *wm,
1483                        u64 data_rate)
1484 {
1485         u16 size, extra = 0;
1486
1487         if (data_rate) {
1488                 extra = min_t(u16, iter->size,
1489                               DIV64_U64_ROUND_UP(iter->size * data_rate,
1490                                                  iter->data_rate));
1491                 iter->size -= extra;
1492                 iter->data_rate -= data_rate;
1493         }
1494
1495         /*
1496          * Keep ddb entry of all disabled planes explicitly zeroed
1497          * to avoid skl_ddb_add_affected_planes() adding them to
1498          * the state when other planes change their allocations.
1499          */
1500         size = wm->min_ddb_alloc + extra;
1501         if (size)
1502                 iter->start = skl_ddb_entry_init(ddb, iter->start,
1503                                                  iter->start + size);
1504 }
1505
1506 static int
1507 skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
1508                             struct intel_crtc *crtc)
1509 {
1510         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1511         struct intel_crtc_state *crtc_state =
1512                 intel_atomic_get_new_crtc_state(state, crtc);
1513         const struct intel_dbuf_state *dbuf_state =
1514                 intel_atomic_get_new_dbuf_state(state);
1515         const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
1516         int num_active = hweight8(dbuf_state->active_pipes);
1517         struct skl_plane_ddb_iter iter;
1518         enum plane_id plane_id;
1519         u16 cursor_size;
1520         u32 blocks;
1521         int level;
1522
1523         /* Clear the partitioning for disabled planes. */
1524         memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb));
1525         memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
1526
1527         if (!crtc_state->hw.active)
1528                 return 0;
1529
1530         iter.start = alloc->start;
1531         iter.size = skl_ddb_entry_size(alloc);
1532         if (iter.size == 0)
1533                 return 0;
1534
1535         /* Allocate fixed number of blocks for cursor. */
1536         cursor_size = skl_cursor_allocation(crtc_state, num_active);
1537         iter.size -= cursor_size;
1538         skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[PLANE_CURSOR],
1539                            alloc->end - cursor_size, alloc->end);
1540
1541         iter.data_rate = skl_total_relative_data_rate(crtc_state);
1542
1543         /*
1544          * Find the highest watermark level for which we can satisfy the block
1545          * requirement of active planes.
1546          */
1547         for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
1548                 blocks = 0;
1549                 for_each_plane_id_on_crtc(crtc, plane_id) {
1550                         const struct skl_plane_wm *wm =
1551                                 &crtc_state->wm.skl.optimal.planes[plane_id];
1552
1553                         if (plane_id == PLANE_CURSOR) {
1554                                 const struct skl_ddb_entry *ddb =
1555                                         &crtc_state->wm.skl.plane_ddb[plane_id];
1556
1557                                 if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1558                                         drm_WARN_ON(&i915->drm,
1559                                                     wm->wm[level].min_ddb_alloc != U16_MAX);
1560                                         blocks = U32_MAX;
1561                                         break;
1562                                 }
1563                                 continue;
1564                         }
1565
1566                         blocks += wm->wm[level].min_ddb_alloc;
1567                         blocks += wm->uv_wm[level].min_ddb_alloc;
1568                 }
1569
1570                 if (blocks <= iter.size) {
1571                         iter.size -= blocks;
1572                         break;
1573                 }
1574         }
1575
1576         if (level < 0) {
1577                 drm_dbg_kms(&i915->drm,
1578                             "Requested display configuration exceeds system DDB limitations");
1579                 drm_dbg_kms(&i915->drm, "minimum required %d/%d\n",
1580                             blocks, iter.size);
1581                 return -EINVAL;
1582         }
1583
1584         /* avoid the WARN later when we don't allocate any extra DDB */
1585         if (iter.data_rate == 0)
1586                 iter.size = 0;
1587
1588         /*
1589          * Grant each plane the blocks it requires at the highest achievable
1590          * watermark level, plus an extra share of the leftover blocks
1591          * proportional to its relative data rate.
1592          */
1593         for_each_plane_id_on_crtc(crtc, plane_id) {
1594                 struct skl_ddb_entry *ddb =
1595                         &crtc_state->wm.skl.plane_ddb[plane_id];
1596                 struct skl_ddb_entry *ddb_y =
1597                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
1598                 const struct skl_plane_wm *wm =
1599                         &crtc_state->wm.skl.optimal.planes[plane_id];
1600
1601                 if (plane_id == PLANE_CURSOR)
1602                         continue;
1603
1604                 if (DISPLAY_VER(i915) < 11 &&
1605                     crtc_state->nv12_planes & BIT(plane_id)) {
1606                         skl_allocate_plane_ddb(&iter, ddb_y, &wm->wm[level],
1607                                                crtc_state->rel_data_rate_y[plane_id]);
1608                         skl_allocate_plane_ddb(&iter, ddb, &wm->uv_wm[level],
1609                                                crtc_state->rel_data_rate[plane_id]);
1610                 } else {
1611                         skl_allocate_plane_ddb(&iter, ddb, &wm->wm[level],
1612                                                crtc_state->rel_data_rate[plane_id]);
1613                 }
1614         }
1615         drm_WARN_ON(&i915->drm, iter.size != 0 || iter.data_rate != 0);
1616
1617         /*
1618          * When we calculated watermark values we didn't know how high
1619          * of a level we'd actually be able to hit, so we just marked
1620          * all levels as "enabled."  Go back now and disable the ones
1621          * that aren't actually possible.
1622          */
1623         for (level++; level < i915->display.wm.num_levels; level++) {
1624                 for_each_plane_id_on_crtc(crtc, plane_id) {
1625                         const struct skl_ddb_entry *ddb =
1626                                 &crtc_state->wm.skl.plane_ddb[plane_id];
1627                         const struct skl_ddb_entry *ddb_y =
1628                                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
1629                         struct skl_plane_wm *wm =
1630                                 &crtc_state->wm.skl.optimal.planes[plane_id];
1631
1632                         if (DISPLAY_VER(i915) < 11 &&
1633                             crtc_state->nv12_planes & BIT(plane_id))
1634                                 skl_check_nv12_wm_level(&wm->wm[level],
1635                                                         &wm->uv_wm[level],
1636                                                         ddb_y, ddb);
1637                         else
1638                                 skl_check_wm_level(&wm->wm[level], ddb);
1639
1640                         if (skl_need_wm_copy_wa(i915, level, wm)) {
1641                                 wm->wm[level].blocks = wm->wm[level - 1].blocks;
1642                                 wm->wm[level].lines = wm->wm[level - 1].lines;
1643                                 wm->wm[level].ignore_lines = wm->wm[level - 1].ignore_lines;
1644                         }
1645                 }
1646         }
1647
1648         /*
1649          * Go back and disable the transition and SAGV watermarks
1650          * if it turns out we don't have enough DDB blocks for them.
1651          */
1652         for_each_plane_id_on_crtc(crtc, plane_id) {
1653                 const struct skl_ddb_entry *ddb =
1654                         &crtc_state->wm.skl.plane_ddb[plane_id];
1655                 const struct skl_ddb_entry *ddb_y =
1656                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
1657                 struct skl_plane_wm *wm =
1658                         &crtc_state->wm.skl.optimal.planes[plane_id];
1659
1660                 if (DISPLAY_VER(i915) < 11 &&
1661                     crtc_state->nv12_planes & BIT(plane_id)) {
1662                         skl_check_wm_level(&wm->trans_wm, ddb_y);
1663                 } else {
1664                         WARN_ON(skl_ddb_entry_size(ddb_y));
1665
1666                         skl_check_wm_level(&wm->trans_wm, ddb);
1667                 }
1668
1669                 skl_check_wm_level(&wm->sagv.wm0, ddb);
1670                 skl_check_wm_level(&wm->sagv.trans_wm, ddb);
1671         }
1672
1673         return 0;
1674 }
1675
1676 /*
1677  * The max latency should be 257 (max the punit can code is 255 and we add 2us
1678  * for the read latency) and cpp should always be <= 8, so that
1679  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
1680  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
1681  */
1682 static uint_fixed_16_16_t
1683 skl_wm_method1(const struct drm_i915_private *i915, u32 pixel_rate,
1684                u8 cpp, u32 latency, u32 dbuf_block_size)
1685 {
1686         u32 wm_intermediate_val;
1687         uint_fixed_16_16_t ret;
1688
1689         if (latency == 0)
1690                 return FP_16_16_MAX;
1691
1692         wm_intermediate_val = latency * pixel_rate * cpp;
1693         ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
1694
1695         if (DISPLAY_VER(i915) >= 10)
1696                 ret = add_fixed16_u32(ret, 1);
1697
1698         return ret;
1699 }
1700
1701 static uint_fixed_16_16_t
1702 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
1703                uint_fixed_16_16_t plane_blocks_per_line)
1704 {
1705         u32 wm_intermediate_val;
1706         uint_fixed_16_16_t ret;
1707
1708         if (latency == 0)
1709                 return FP_16_16_MAX;
1710
1711         wm_intermediate_val = latency * pixel_rate;
1712         wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
1713                                            pipe_htotal * 1000);
1714         ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
1715         return ret;
1716 }
1717
1718 static uint_fixed_16_16_t
1719 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
1720 {
1721         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1722         u32 pixel_rate;
1723         u32 crtc_htotal;
1724         uint_fixed_16_16_t linetime_us;
1725
1726         if (!crtc_state->hw.active)
1727                 return u32_to_fixed16(0);
1728
1729         pixel_rate = crtc_state->pixel_rate;
1730
1731         if (drm_WARN_ON(&i915->drm, pixel_rate == 0))
1732                 return u32_to_fixed16(0);
1733
1734         crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
1735         linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
1736
1737         return linetime_us;
1738 }
1739
1740 static int
1741 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
1742                       int width, const struct drm_format_info *format,
1743                       u64 modifier, unsigned int rotation,
1744                       u32 plane_pixel_rate, struct skl_wm_params *wp,
1745                       int color_plane, unsigned int pan_x)
1746 {
1747         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1748         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1749         u32 interm_pbpl;
1750
1751         /* only planar format has two planes */
1752         if (color_plane == 1 &&
1753             !intel_format_info_is_yuv_semiplanar(format, modifier)) {
1754                 drm_dbg_kms(&i915->drm,
1755                             "Non planar format have single plane\n");
1756                 return -EINVAL;
1757         }
1758
1759         wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
1760         wp->y_tiled = modifier != I915_FORMAT_MOD_X_TILED &&
1761                 intel_fb_is_tiled_modifier(modifier);
1762         wp->rc_surface = intel_fb_is_ccs_modifier(modifier);
1763         wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
1764
1765         wp->width = width;
1766         if (color_plane == 1 && wp->is_planar)
1767                 wp->width /= 2;
1768
1769         wp->cpp = format->cpp[color_plane];
1770         wp->plane_pixel_rate = plane_pixel_rate;
1771
1772         if (DISPLAY_VER(i915) >= 11 &&
1773             modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
1774                 wp->dbuf_block_size = 256;
1775         else
1776                 wp->dbuf_block_size = 512;
1777
1778         if (drm_rotation_90_or_270(rotation)) {
1779                 switch (wp->cpp) {
1780                 case 1:
1781                         wp->y_min_scanlines = 16;
1782                         break;
1783                 case 2:
1784                         wp->y_min_scanlines = 8;
1785                         break;
1786                 case 4:
1787                         wp->y_min_scanlines = 4;
1788                         break;
1789                 default:
1790                         MISSING_CASE(wp->cpp);
1791                         return -EINVAL;
1792                 }
1793         } else {
1794                 wp->y_min_scanlines = 4;
1795         }
1796
1797         if (skl_needs_memory_bw_wa(i915))
1798                 wp->y_min_scanlines *= 2;
1799
1800         wp->plane_bytes_per_line = wp->width * wp->cpp;
1801         if (wp->y_tiled) {
1802                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
1803                                            wp->y_min_scanlines,
1804                                            wp->dbuf_block_size);
1805
1806                 if (DISPLAY_VER(i915) >= 30)
1807                         interm_pbpl += (pan_x != 0);
1808                 else if (DISPLAY_VER(i915) >= 10)
1809                         interm_pbpl++;
1810
1811                 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
1812                                                         wp->y_min_scanlines);
1813         } else {
1814                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
1815                                            wp->dbuf_block_size);
1816
1817                 if (!wp->x_tiled || DISPLAY_VER(i915) >= 10)
1818                         interm_pbpl++;
1819
1820                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
1821         }
1822
1823         wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
1824                                              wp->plane_blocks_per_line);
1825
1826         wp->linetime_us = fixed16_to_u32_round_up(intel_get_linetime_us(crtc_state));
1827
1828         return 0;
1829 }
1830
1831 static int
1832 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
1833                             const struct intel_plane_state *plane_state,
1834                             struct skl_wm_params *wp, int color_plane)
1835 {
1836         const struct drm_framebuffer *fb = plane_state->hw.fb;
1837         int width;
1838
1839         /*
1840          * Src coordinates are already rotated by 270 degrees for
1841          * the 90/270 degree plane rotation cases (to match the
1842          * GTT mapping), hence no need to account for rotation here.
1843          */
1844         width = drm_rect_width(&plane_state->uapi.src) >> 16;
1845
1846         return skl_compute_wm_params(crtc_state, width,
1847                                      fb->format, fb->modifier,
1848                                      plane_state->hw.rotation,
1849                                      intel_plane_pixel_rate(crtc_state, plane_state),
1850                                      wp, color_plane,
1851                                      plane_state->uapi.src.x1);
1852 }
1853
1854 static bool skl_wm_has_lines(struct drm_i915_private *i915, int level)
1855 {
1856         if (DISPLAY_VER(i915) >= 10)
1857                 return true;
1858
1859         /* The number of lines are ignored for the level 0 watermark. */
1860         return level > 0;
1861 }
1862
1863 static int skl_wm_max_lines(struct drm_i915_private *i915)
1864 {
1865         if (DISPLAY_VER(i915) >= 13)
1866                 return 255;
1867         else
1868                 return 31;
1869 }
1870
1871 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
1872                                  struct intel_plane *plane,
1873                                  int level,
1874                                  unsigned int latency,
1875                                  const struct skl_wm_params *wp,
1876                                  const struct skl_wm_level *result_prev,
1877                                  struct skl_wm_level *result /* out */)
1878 {
1879         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1880         uint_fixed_16_16_t method1, method2;
1881         uint_fixed_16_16_t selected_result;
1882         u32 blocks, lines, min_ddb_alloc = 0;
1883
1884         if (latency == 0 ||
1885             (use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
1886                 /* reject it */
1887                 result->min_ddb_alloc = U16_MAX;
1888                 return;
1889         }
1890
1891         method1 = skl_wm_method1(i915, wp->plane_pixel_rate,
1892                                  wp->cpp, latency, wp->dbuf_block_size);
1893         method2 = skl_wm_method2(wp->plane_pixel_rate,
1894                                  crtc_state->hw.pipe_mode.crtc_htotal,
1895                                  latency,
1896                                  wp->plane_blocks_per_line);
1897
1898         if (wp->y_tiled) {
1899                 selected_result = max_fixed16(method2, wp->y_tile_minimum);
1900         } else {
1901                 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
1902                      wp->dbuf_block_size < 1) &&
1903                      (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
1904                         selected_result = method2;
1905                 } else if (latency >= wp->linetime_us) {
1906                         if (DISPLAY_VER(i915) == 9)
1907                                 selected_result = min_fixed16(method1, method2);
1908                         else
1909                                 selected_result = method2;
1910                 } else {
1911                         selected_result = method1;
1912                 }
1913         }
1914
1915         blocks = fixed16_to_u32_round_up(selected_result);
1916         if (DISPLAY_VER(i915) < 30)
1917                 blocks++;
1918
1919         /*
1920          * Lets have blocks at minimum equivalent to plane_blocks_per_line
1921          * as there will be at minimum one line for lines configuration. This
1922          * is a work around for FIFO underruns observed with resolutions like
1923          * 4k 60 Hz in single channel DRAM configurations.
1924          *
1925          * As per the Bspec 49325, if the ddb allocation can hold at least
1926          * one plane_blocks_per_line, we should have selected method2 in
1927          * the above logic. Assuming that modern versions have enough dbuf
1928          * and method2 guarantees blocks equivalent to at least 1 line,
1929          * select the blocks as plane_blocks_per_line.
1930          *
1931          * TODO: Revisit the logic when we have better understanding on DRAM
1932          * channels' impact on the level 0 memory latency and the relevant
1933          * wm calculations.
1934          */
1935         if (skl_wm_has_lines(i915, level))
1936                 blocks = max(blocks,
1937                              fixed16_to_u32_round_up(wp->plane_blocks_per_line));
1938         lines = div_round_up_fixed16(selected_result,
1939                                      wp->plane_blocks_per_line);
1940
1941         if (DISPLAY_VER(i915) == 9) {
1942                 /* Display WA #1125: skl,bxt,kbl */
1943                 if (level == 0 && wp->rc_surface)
1944                         blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
1945
1946                 /* Display WA #1126: skl,bxt,kbl */
1947                 if (level >= 1 && level <= 7) {
1948                         if (wp->y_tiled) {
1949                                 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
1950                                 lines += wp->y_min_scanlines;
1951                         } else {
1952                                 blocks++;
1953                         }
1954
1955                         /*
1956                          * Make sure result blocks for higher latency levels are
1957                          * at least as high as level below the current level.
1958                          * Assumption in DDB algorithm optimization for special
1959                          * cases. Also covers Display WA #1125 for RC.
1960                          */
1961                         if (result_prev->blocks > blocks)
1962                                 blocks = result_prev->blocks;
1963                 }
1964         }
1965
1966         if (DISPLAY_VER(i915) >= 11) {
1967                 if (wp->y_tiled) {
1968                         int extra_lines;
1969
1970                         if (lines % wp->y_min_scanlines == 0)
1971                                 extra_lines = wp->y_min_scanlines;
1972                         else
1973                                 extra_lines = wp->y_min_scanlines * 2 -
1974                                         lines % wp->y_min_scanlines;
1975
1976                         min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
1977                                                                  wp->plane_blocks_per_line);
1978                 } else {
1979                         min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
1980                 }
1981         }
1982
1983         if (!skl_wm_has_lines(i915, level))
1984                 lines = 0;
1985
1986         if (lines > skl_wm_max_lines(i915)) {
1987                 /* reject it */
1988                 result->min_ddb_alloc = U16_MAX;
1989                 return;
1990         }
1991
1992         /*
1993          * If lines is valid, assume we can use this watermark level
1994          * for now.  We'll come back and disable it after we calculate the
1995          * DDB allocation if it turns out we don't actually have enough
1996          * blocks to satisfy it.
1997          */
1998         result->blocks = blocks;
1999         result->lines = lines;
2000         /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
2001         result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
2002         result->enable = true;
2003
2004         if (DISPLAY_VER(i915) < 12 && i915->display.sagv.block_time_us)
2005                 result->can_sagv = latency >= i915->display.sagv.block_time_us;
2006 }
2007
2008 static void
2009 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
2010                       struct intel_plane *plane,
2011                       const struct skl_wm_params *wm_params,
2012                       struct skl_wm_level *levels)
2013 {
2014         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
2015         struct skl_wm_level *result_prev = &levels[0];
2016         int level;
2017
2018         for (level = 0; level < i915->display.wm.num_levels; level++) {
2019                 struct skl_wm_level *result = &levels[level];
2020                 unsigned int latency = skl_wm_latency(i915, level, wm_params);
2021
2022                 skl_compute_plane_wm(crtc_state, plane, level, latency,
2023                                      wm_params, result_prev, result);
2024
2025                 result_prev = result;
2026         }
2027 }
2028
2029 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
2030                                 struct intel_plane *plane,
2031                                 const struct skl_wm_params *wm_params,
2032                                 struct skl_plane_wm *plane_wm)
2033 {
2034         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
2035         struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
2036         struct skl_wm_level *levels = plane_wm->wm;
2037         unsigned int latency = 0;
2038
2039         if (i915->display.sagv.block_time_us)
2040                 latency = i915->display.sagv.block_time_us +
2041                         skl_wm_latency(i915, 0, wm_params);
2042
2043         skl_compute_plane_wm(crtc_state, plane, 0, latency,
2044                              wm_params, &levels[0],
2045                              sagv_wm);
2046 }
2047
2048 static void skl_compute_transition_wm(struct drm_i915_private *i915,
2049                                       struct skl_wm_level *trans_wm,
2050                                       const struct skl_wm_level *wm0,
2051                                       const struct skl_wm_params *wp)
2052 {
2053         u16 trans_min, trans_amount, trans_y_tile_min;
2054         u16 wm0_blocks, trans_offset, blocks;
2055
2056         /* Transition WM don't make any sense if ipc is disabled */
2057         if (!skl_watermark_ipc_enabled(i915))
2058                 return;
2059
2060         /*
2061          * WaDisableTWM:skl,kbl,cfl,bxt
2062          * Transition WM are not recommended by HW team for GEN9
2063          */
2064         if (DISPLAY_VER(i915) == 9)
2065                 return;
2066
2067         if (DISPLAY_VER(i915) >= 11)
2068                 trans_min = 4;
2069         else
2070                 trans_min = 14;
2071
2072         /* Display WA #1140: glk,cnl */
2073         if (DISPLAY_VER(i915) == 10)
2074                 trans_amount = 0;
2075         else
2076                 trans_amount = 10; /* This is configurable amount */
2077
2078         trans_offset = trans_min + trans_amount;
2079
2080         /*
2081          * The spec asks for Selected Result Blocks for wm0 (the real value),
2082          * not Result Blocks (the integer value). Pay attention to the capital
2083          * letters. The value wm_l0->blocks is actually Result Blocks, but
2084          * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
2085          * and since we later will have to get the ceiling of the sum in the
2086          * transition watermarks calculation, we can just pretend Selected
2087          * Result Blocks is Result Blocks minus 1 and it should work for the
2088          * current platforms.
2089          */
2090         wm0_blocks = wm0->blocks - 1;
2091
2092         if (wp->y_tiled) {
2093                 trans_y_tile_min =
2094                         (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
2095                 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
2096         } else {
2097                 blocks = wm0_blocks + trans_offset;
2098         }
2099         blocks++;
2100
2101         /*
2102          * Just assume we can enable the transition watermark.  After
2103          * computing the DDB we'll come back and disable it if that
2104          * assumption turns out to be false.
2105          */
2106         trans_wm->blocks = blocks;
2107         trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
2108         trans_wm->enable = true;
2109 }
2110
2111 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
2112                                      const struct intel_plane_state *plane_state,
2113                                      struct intel_plane *plane, int color_plane)
2114 {
2115         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2116         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2117         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2118         struct skl_wm_params wm_params;
2119         int ret;
2120
2121         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2122                                           &wm_params, color_plane);
2123         if (ret)
2124                 return ret;
2125
2126         skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm);
2127
2128         skl_compute_transition_wm(i915, &wm->trans_wm,
2129                                   &wm->wm[0], &wm_params);
2130
2131         if (DISPLAY_VER(i915) >= 12) {
2132                 tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm);
2133
2134                 skl_compute_transition_wm(i915, &wm->sagv.trans_wm,
2135                                           &wm->sagv.wm0, &wm_params);
2136         }
2137
2138         return 0;
2139 }
2140
2141 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
2142                                  const struct intel_plane_state *plane_state,
2143                                  struct intel_plane *plane)
2144 {
2145         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2146         struct skl_wm_params wm_params;
2147         int ret;
2148
2149         wm->is_planar = true;
2150
2151         /* uv plane watermarks must also be validated for NV12/Planar */
2152         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2153                                           &wm_params, 1);
2154         if (ret)
2155                 return ret;
2156
2157         skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm);
2158
2159         return 0;
2160 }
2161
2162 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
2163                               const struct intel_plane_state *plane_state)
2164 {
2165         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2166         enum plane_id plane_id = plane->id;
2167         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2168         const struct drm_framebuffer *fb = plane_state->hw.fb;
2169         int ret;
2170
2171         memset(wm, 0, sizeof(*wm));
2172
2173         if (!intel_wm_plane_visible(crtc_state, plane_state))
2174                 return 0;
2175
2176         ret = skl_build_plane_wm_single(crtc_state, plane_state,
2177                                         plane, 0);
2178         if (ret)
2179                 return ret;
2180
2181         if (fb->format->is_yuv && fb->format->num_planes > 1) {
2182                 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
2183                                             plane);
2184                 if (ret)
2185                         return ret;
2186         }
2187
2188         return 0;
2189 }
2190
2191 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
2192                               const struct intel_plane_state *plane_state)
2193 {
2194         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2195         struct drm_i915_private *i915 = to_i915(plane->base.dev);
2196         enum plane_id plane_id = plane->id;
2197         struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2198         int ret;
2199
2200         /* Watermarks calculated in master */
2201         if (plane_state->planar_slave)
2202                 return 0;
2203
2204         memset(wm, 0, sizeof(*wm));
2205
2206         if (plane_state->planar_linked_plane) {
2207                 const struct drm_framebuffer *fb = plane_state->hw.fb;
2208
2209                 drm_WARN_ON(&i915->drm,
2210                             !intel_wm_plane_visible(crtc_state, plane_state));
2211                 drm_WARN_ON(&i915->drm, !fb->format->is_yuv ||
2212                             fb->format->num_planes == 1);
2213
2214                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2215                                                 plane_state->planar_linked_plane, 0);
2216                 if (ret)
2217                         return ret;
2218
2219                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2220                                                 plane, 1);
2221                 if (ret)
2222                         return ret;
2223         } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
2224                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2225                                                 plane, 0);
2226                 if (ret)
2227                         return ret;
2228         }
2229
2230         return 0;
2231 }
2232
2233 static bool
2234 skl_is_vblank_too_short(const struct intel_crtc_state *crtc_state,
2235                         int wm0_lines, int latency)
2236 {
2237         const struct drm_display_mode *adjusted_mode =
2238                 &crtc_state->hw.adjusted_mode;
2239
2240         /* FIXME missing scaler and DSC pre-fill time */
2241         return crtc_state->framestart_delay +
2242                 intel_usecs_to_scanlines(adjusted_mode, latency) +
2243                 wm0_lines >
2244                 adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vblank_start;
2245 }
2246
2247 static int skl_max_wm0_lines(const struct intel_crtc_state *crtc_state)
2248 {
2249         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2250         enum plane_id plane_id;
2251         int wm0_lines = 0;
2252
2253         for_each_plane_id_on_crtc(crtc, plane_id) {
2254                 const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
2255
2256                 /* FIXME what about !skl_wm_has_lines() platforms? */
2257                 wm0_lines = max_t(int, wm0_lines, wm->wm[0].lines);
2258         }
2259
2260         return wm0_lines;
2261 }
2262
2263 static int skl_max_wm_level_for_vblank(struct intel_crtc_state *crtc_state,
2264                                        int wm0_lines)
2265 {
2266         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2267         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2268         int level;
2269
2270         for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
2271                 int latency;
2272
2273                 /* FIXME should we care about the latency w/a's? */
2274                 latency = skl_wm_latency(i915, level, NULL);
2275                 if (latency == 0)
2276                         continue;
2277
2278                 /* FIXME is it correct to use 0 latency for wm0 here? */
2279                 if (level == 0)
2280                         latency = 0;
2281
2282                 if (!skl_is_vblank_too_short(crtc_state, wm0_lines, latency))
2283                         return level;
2284         }
2285
2286         return -EINVAL;
2287 }
2288
2289 static int skl_wm_check_vblank(struct intel_crtc_state *crtc_state)
2290 {
2291         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2292         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2293         int wm0_lines, level;
2294
2295         if (!crtc_state->hw.active)
2296                 return 0;
2297
2298         wm0_lines = skl_max_wm0_lines(crtc_state);
2299
2300         level = skl_max_wm_level_for_vblank(crtc_state, wm0_lines);
2301         if (level < 0)
2302                 return level;
2303
2304         /*
2305          * PSR needs to toggle LATENCY_REPORTING_REMOVED_PIPE_*
2306          * based on whether we're limited by the vblank duration.
2307          */
2308         crtc_state->wm_level_disabled = level < i915->display.wm.num_levels - 1;
2309
2310         for (level++; level < i915->display.wm.num_levels; level++) {
2311                 enum plane_id plane_id;
2312
2313                 for_each_plane_id_on_crtc(crtc, plane_id) {
2314                         struct skl_plane_wm *wm =
2315                                 &crtc_state->wm.skl.optimal.planes[plane_id];
2316
2317                         /*
2318                          * FIXME just clear enable or flag the entire
2319                          * thing as bad via min_ddb_alloc=U16_MAX?
2320                          */
2321                         wm->wm[level].enable = false;
2322                         wm->uv_wm[level].enable = false;
2323                 }
2324         }
2325
2326         if (DISPLAY_VER(i915) >= 12 &&
2327             i915->display.sagv.block_time_us &&
2328             skl_is_vblank_too_short(crtc_state, wm0_lines,
2329                                     i915->display.sagv.block_time_us)) {
2330                 enum plane_id plane_id;
2331
2332                 for_each_plane_id_on_crtc(crtc, plane_id) {
2333                         struct skl_plane_wm *wm =
2334                                 &crtc_state->wm.skl.optimal.planes[plane_id];
2335
2336                         wm->sagv.wm0.enable = false;
2337                         wm->sagv.trans_wm.enable = false;
2338                 }
2339         }
2340
2341         return 0;
2342 }
2343
2344 static int skl_build_pipe_wm(struct intel_atomic_state *state,
2345                              struct intel_crtc *crtc)
2346 {
2347         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2348         struct intel_crtc_state *crtc_state =
2349                 intel_atomic_get_new_crtc_state(state, crtc);
2350         const struct intel_plane_state *plane_state;
2351         struct intel_plane *plane;
2352         int ret, i;
2353
2354         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2355                 /*
2356                  * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
2357                  * instead but we don't populate that correctly for NV12 Y
2358                  * planes so for now hack this.
2359                  */
2360                 if (plane->pipe != crtc->pipe)
2361                         continue;
2362
2363                 if (DISPLAY_VER(i915) >= 11)
2364                         ret = icl_build_plane_wm(crtc_state, plane_state);
2365                 else
2366                         ret = skl_build_plane_wm(crtc_state, plane_state);
2367                 if (ret)
2368                         return ret;
2369         }
2370
2371         crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
2372
2373         return skl_wm_check_vblank(crtc_state);
2374 }
2375
2376 static bool skl_wm_level_equals(const struct skl_wm_level *l1,
2377                                 const struct skl_wm_level *l2)
2378 {
2379         return l1->enable == l2->enable &&
2380                 l1->ignore_lines == l2->ignore_lines &&
2381                 l1->lines == l2->lines &&
2382                 l1->blocks == l2->blocks;
2383 }
2384
2385 static bool skl_plane_wm_equals(struct drm_i915_private *i915,
2386                                 const struct skl_plane_wm *wm1,
2387                                 const struct skl_plane_wm *wm2)
2388 {
2389         int level;
2390
2391         for (level = 0; level < i915->display.wm.num_levels; level++) {
2392                 /*
2393                  * We don't check uv_wm as the hardware doesn't actually
2394                  * use it. It only gets used for calculating the required
2395                  * ddb allocation.
2396                  */
2397                 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
2398                         return false;
2399         }
2400
2401         return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
2402                 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
2403                 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
2404 }
2405
2406 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
2407                                     const struct skl_ddb_entry *b)
2408 {
2409         return a->start < b->end && b->start < a->end;
2410 }
2411
2412 static void skl_ddb_entry_union(struct skl_ddb_entry *a,
2413                                 const struct skl_ddb_entry *b)
2414 {
2415         if (a->end && b->end) {
2416                 a->start = min(a->start, b->start);
2417                 a->end = max(a->end, b->end);
2418         } else if (b->end) {
2419                 a->start = b->start;
2420                 a->end = b->end;
2421         }
2422 }
2423
2424 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
2425                                  const struct skl_ddb_entry *entries,
2426                                  int num_entries, int ignore_idx)
2427 {
2428         int i;
2429
2430         for (i = 0; i < num_entries; i++) {
2431                 if (i != ignore_idx &&
2432                     skl_ddb_entries_overlap(ddb, &entries[i]))
2433                         return true;
2434         }
2435
2436         return false;
2437 }
2438
2439 static int
2440 skl_ddb_add_affected_planes(struct intel_atomic_state *state,
2441                             struct intel_crtc *crtc)
2442 {
2443         struct drm_i915_private *i915 = to_i915(state->base.dev);
2444         const struct intel_crtc_state *old_crtc_state =
2445                 intel_atomic_get_old_crtc_state(state, crtc);
2446         struct intel_crtc_state *new_crtc_state =
2447                 intel_atomic_get_new_crtc_state(state, crtc);
2448         struct intel_plane *plane;
2449
2450         for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2451                 struct intel_plane_state *plane_state;
2452                 enum plane_id plane_id = plane->id;
2453
2454                 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb[plane_id],
2455                                         &new_crtc_state->wm.skl.plane_ddb[plane_id]) &&
2456                     skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
2457                                         &new_crtc_state->wm.skl.plane_ddb_y[plane_id]))
2458                         continue;
2459
2460                 if (new_crtc_state->do_async_flip) {
2461                         drm_dbg_kms(&i915->drm, "[PLANE:%d:%s] Can't change DDB during async flip\n",
2462                                     plane->base.base.id, plane->base.name);
2463                         return -EINVAL;
2464                 }
2465
2466                 plane_state = intel_atomic_get_plane_state(state, plane);
2467                 if (IS_ERR(plane_state))
2468                         return PTR_ERR(plane_state);
2469
2470                 new_crtc_state->update_planes |= BIT(plane_id);
2471                 new_crtc_state->async_flip_planes = 0;
2472                 new_crtc_state->do_async_flip = false;
2473         }
2474
2475         return 0;
2476 }
2477
2478 static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
2479 {
2480         struct drm_i915_private *i915 = to_i915(dbuf_state->base.state->base.dev);
2481         u8 enabled_slices;
2482         enum pipe pipe;
2483
2484         /*
2485          * FIXME: For now we always enable slice S1 as per
2486          * the Bspec display initialization sequence.
2487          */
2488         enabled_slices = BIT(DBUF_S1);
2489
2490         for_each_pipe(i915, pipe)
2491                 enabled_slices |= dbuf_state->slices[pipe];
2492
2493         return enabled_slices;
2494 }
2495
2496 static int
2497 skl_compute_ddb(struct intel_atomic_state *state)
2498 {
2499         struct drm_i915_private *i915 = to_i915(state->base.dev);
2500         const struct intel_dbuf_state *old_dbuf_state;
2501         struct intel_dbuf_state *new_dbuf_state = NULL;
2502         struct intel_crtc_state *new_crtc_state;
2503         struct intel_crtc *crtc;
2504         int ret, i;
2505
2506         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2507                 new_dbuf_state = intel_atomic_get_dbuf_state(state);
2508                 if (IS_ERR(new_dbuf_state))
2509                         return PTR_ERR(new_dbuf_state);
2510
2511                 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
2512                 break;
2513         }
2514
2515         if (!new_dbuf_state)
2516                 return 0;
2517
2518         new_dbuf_state->active_pipes =
2519                 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
2520
2521         if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
2522                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2523                 if (ret)
2524                         return ret;
2525         }
2526
2527         if (HAS_MBUS_JOINING(i915)) {
2528                 new_dbuf_state->joined_mbus =
2529                         adlp_check_mbus_joined(new_dbuf_state->active_pipes);
2530
2531                 if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2532                         ret = intel_cdclk_state_set_joined_mbus(state, new_dbuf_state->joined_mbus);
2533                         if (ret)
2534                                 return ret;
2535                 }
2536         }
2537
2538         for_each_intel_crtc(&i915->drm, crtc) {
2539                 enum pipe pipe = crtc->pipe;
2540
2541                 new_dbuf_state->slices[pipe] =
2542                         skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes,
2543                                                 new_dbuf_state->joined_mbus);
2544
2545                 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
2546                         continue;
2547
2548                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2549                 if (ret)
2550                         return ret;
2551         }
2552
2553         new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
2554
2555         if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
2556             old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2557                 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
2558                 if (ret)
2559                         return ret;
2560
2561                 drm_dbg_kms(&i915->drm,
2562                             "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
2563                             old_dbuf_state->enabled_slices,
2564                             new_dbuf_state->enabled_slices,
2565                             DISPLAY_INFO(i915)->dbuf.slice_mask,
2566                             str_yes_no(old_dbuf_state->joined_mbus),
2567                             str_yes_no(new_dbuf_state->joined_mbus));
2568         }
2569
2570         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2571                 enum pipe pipe = crtc->pipe;
2572
2573                 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
2574
2575                 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
2576                         continue;
2577
2578                 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2579                 if (ret)
2580                         return ret;
2581         }
2582
2583         for_each_intel_crtc(&i915->drm, crtc) {
2584                 ret = skl_crtc_allocate_ddb(state, crtc);
2585                 if (ret)
2586                         return ret;
2587         }
2588
2589         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2590                 ret = skl_crtc_allocate_plane_ddb(state, crtc);
2591                 if (ret)
2592                         return ret;
2593
2594                 ret = skl_ddb_add_affected_planes(state, crtc);
2595                 if (ret)
2596                         return ret;
2597         }
2598
2599         return 0;
2600 }
2601
2602 static char enast(bool enable)
2603 {
2604         return enable ? '*' : ' ';
2605 }
2606
2607 static void
2608 skl_print_wm_changes(struct intel_atomic_state *state)
2609 {
2610         struct drm_i915_private *i915 = to_i915(state->base.dev);
2611         const struct intel_crtc_state *old_crtc_state;
2612         const struct intel_crtc_state *new_crtc_state;
2613         struct intel_plane *plane;
2614         struct intel_crtc *crtc;
2615         int i;
2616
2617         if (!drm_debug_enabled(DRM_UT_KMS))
2618                 return;
2619
2620         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2621                                             new_crtc_state, i) {
2622                 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
2623
2624                 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
2625                 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
2626
2627                 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2628                         enum plane_id plane_id = plane->id;
2629                         const struct skl_ddb_entry *old, *new;
2630
2631                         old = &old_crtc_state->wm.skl.plane_ddb[plane_id];
2632                         new = &new_crtc_state->wm.skl.plane_ddb[plane_id];
2633
2634                         if (skl_ddb_entry_equal(old, new))
2635                                 continue;
2636
2637                         drm_dbg_kms(&i915->drm,
2638                                     "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
2639                                     plane->base.base.id, plane->base.name,
2640                                     old->start, old->end, new->start, new->end,
2641                                     skl_ddb_entry_size(old), skl_ddb_entry_size(new));
2642                 }
2643
2644                 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2645                         enum plane_id plane_id = plane->id;
2646                         const struct skl_plane_wm *old_wm, *new_wm;
2647
2648                         old_wm = &old_pipe_wm->planes[plane_id];
2649                         new_wm = &new_pipe_wm->planes[plane_id];
2650
2651                         if (skl_plane_wm_equals(i915, old_wm, new_wm))
2652                                 continue;
2653
2654                         drm_dbg_kms(&i915->drm,
2655                                     "[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
2656                                     " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
2657                                     plane->base.base.id, plane->base.name,
2658                                     enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
2659                                     enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
2660                                     enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
2661                                     enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
2662                                     enast(old_wm->trans_wm.enable),
2663                                     enast(old_wm->sagv.wm0.enable),
2664                                     enast(old_wm->sagv.trans_wm.enable),
2665                                     enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
2666                                     enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
2667                                     enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
2668                                     enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
2669                                     enast(new_wm->trans_wm.enable),
2670                                     enast(new_wm->sagv.wm0.enable),
2671                                     enast(new_wm->sagv.trans_wm.enable));
2672
2673                         drm_dbg_kms(&i915->drm,
2674                                     "[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
2675                                       " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
2676                                     plane->base.base.id, plane->base.name,
2677                                     enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
2678                                     enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
2679                                     enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
2680                                     enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
2681                                     enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
2682                                     enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
2683                                     enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
2684                                     enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
2685                                     enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
2686                                     enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
2687                                     enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
2688                                     enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
2689                                     enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
2690                                     enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
2691                                     enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
2692                                     enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
2693                                     enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
2694                                     enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
2695                                     enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
2696                                     enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
2697                                     enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
2698                                     enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
2699
2700                         drm_dbg_kms(&i915->drm,
2701                                     "[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2702                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2703                                     plane->base.base.id, plane->base.name,
2704                                     old_wm->wm[0].blocks, old_wm->wm[1].blocks,
2705                                     old_wm->wm[2].blocks, old_wm->wm[3].blocks,
2706                                     old_wm->wm[4].blocks, old_wm->wm[5].blocks,
2707                                     old_wm->wm[6].blocks, old_wm->wm[7].blocks,
2708                                     old_wm->trans_wm.blocks,
2709                                     old_wm->sagv.wm0.blocks,
2710                                     old_wm->sagv.trans_wm.blocks,
2711                                     new_wm->wm[0].blocks, new_wm->wm[1].blocks,
2712                                     new_wm->wm[2].blocks, new_wm->wm[3].blocks,
2713                                     new_wm->wm[4].blocks, new_wm->wm[5].blocks,
2714                                     new_wm->wm[6].blocks, new_wm->wm[7].blocks,
2715                                     new_wm->trans_wm.blocks,
2716                                     new_wm->sagv.wm0.blocks,
2717                                     new_wm->sagv.trans_wm.blocks);
2718
2719                         drm_dbg_kms(&i915->drm,
2720                                     "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2721                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2722                                     plane->base.base.id, plane->base.name,
2723                                     old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
2724                                     old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
2725                                     old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
2726                                     old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
2727                                     old_wm->trans_wm.min_ddb_alloc,
2728                                     old_wm->sagv.wm0.min_ddb_alloc,
2729                                     old_wm->sagv.trans_wm.min_ddb_alloc,
2730                                     new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
2731                                     new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
2732                                     new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
2733                                     new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
2734                                     new_wm->trans_wm.min_ddb_alloc,
2735                                     new_wm->sagv.wm0.min_ddb_alloc,
2736                                     new_wm->sagv.trans_wm.min_ddb_alloc);
2737                 }
2738         }
2739 }
2740
2741 static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
2742                                          const struct skl_pipe_wm *old_pipe_wm,
2743                                          const struct skl_pipe_wm *new_pipe_wm)
2744 {
2745         struct drm_i915_private *i915 = to_i915(plane->base.dev);
2746         int level;
2747
2748         for (level = 0; level < i915->display.wm.num_levels; level++) {
2749                 /*
2750                  * We don't check uv_wm as the hardware doesn't actually
2751                  * use it. It only gets used for calculating the required
2752                  * ddb allocation.
2753                  */
2754                 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
2755                                          skl_plane_wm_level(new_pipe_wm, plane->id, level)))
2756                         return false;
2757         }
2758
2759         if (HAS_HW_SAGV_WM(i915)) {
2760                 const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
2761                 const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
2762
2763                 if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
2764                     !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
2765                         return false;
2766         }
2767
2768         return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
2769                                    skl_plane_trans_wm(new_pipe_wm, plane->id));
2770 }
2771
2772 /*
2773  * To make sure the cursor watermark registers are always consistent
2774  * with our computed state the following scenario needs special
2775  * treatment:
2776  *
2777  * 1. enable cursor
2778  * 2. move cursor entirely offscreen
2779  * 3. disable cursor
2780  *
2781  * Step 2. does call .disable_plane() but does not zero the watermarks
2782  * (since we consider an offscreen cursor still active for the purposes
2783  * of watermarks). Step 3. would not normally call .disable_plane()
2784  * because the actual plane visibility isn't changing, and we don't
2785  * deallocate the cursor ddb until the pipe gets disabled. So we must
2786  * force step 3. to call .disable_plane() to update the watermark
2787  * registers properly.
2788  *
2789  * Other planes do not suffer from this issues as their watermarks are
2790  * calculated based on the actual plane visibility. The only time this
2791  * can trigger for the other planes is during the initial readout as the
2792  * default value of the watermarks registers is not zero.
2793  */
2794 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
2795                                       struct intel_crtc *crtc)
2796 {
2797         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2798         const struct intel_crtc_state *old_crtc_state =
2799                 intel_atomic_get_old_crtc_state(state, crtc);
2800         struct intel_crtc_state *new_crtc_state =
2801                 intel_atomic_get_new_crtc_state(state, crtc);
2802         struct intel_plane *plane;
2803
2804         for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2805                 struct intel_plane_state *plane_state;
2806                 enum plane_id plane_id = plane->id;
2807
2808                 /*
2809                  * Force a full wm update for every plane on modeset.
2810                  * Required because the reset value of the wm registers
2811                  * is non-zero, whereas we want all disabled planes to
2812                  * have zero watermarks. So if we turn off the relevant
2813                  * power well the hardware state will go out of sync
2814                  * with the software state.
2815                  */
2816                 if (!intel_crtc_needs_modeset(new_crtc_state) &&
2817                     skl_plane_selected_wm_equals(plane,
2818                                                  &old_crtc_state->wm.skl.optimal,
2819                                                  &new_crtc_state->wm.skl.optimal))
2820                         continue;
2821
2822                 if (new_crtc_state->do_async_flip) {
2823                         drm_dbg_kms(&i915->drm, "[PLANE:%d:%s] Can't change watermarks during async flip\n",
2824                                     plane->base.base.id, plane->base.name);
2825                         return -EINVAL;
2826                 }
2827
2828                 plane_state = intel_atomic_get_plane_state(state, plane);
2829                 if (IS_ERR(plane_state))
2830                         return PTR_ERR(plane_state);
2831
2832                 new_crtc_state->update_planes |= BIT(plane_id);
2833                 new_crtc_state->async_flip_planes = 0;
2834                 new_crtc_state->do_async_flip = false;
2835         }
2836
2837         return 0;
2838 }
2839
2840 /*
2841  * If Fixed Refresh Rate or For VRR case Vmin = Vmax = Flipline:
2842  * Program DEEP PKG_C_LATENCY Pkg C with highest valid latency from
2843  * watermark level1 and up and above. If watermark level 1 is
2844  * invalid program it with all 1's.
2845  * Program PKG_C_LATENCY Added Wake Time = DSB execution time
2846  * If Variable Refresh Rate where Vmin != Vmax != Flipline:
2847  * Program DEEP PKG_C_LATENCY Pkg C with all 1's.
2848  * Program PKG_C_LATENCY Added Wake Time = 0
2849  */
2850 static void
2851 skl_program_dpkgc_latency(struct drm_i915_private *i915, bool enable_dpkgc)
2852 {
2853         u32 max_latency = 0;
2854         u32 clear = 0, val = 0;
2855         u32 added_wake_time = 0;
2856
2857         if (DISPLAY_VER(i915) < 20)
2858                 return;
2859
2860         if (enable_dpkgc) {
2861                 max_latency = skl_watermark_max_latency(i915, 1);
2862                 if (max_latency == 0)
2863                         max_latency = LNL_PKG_C_LATENCY_MASK;
2864                 added_wake_time = DSB_EXE_TIME +
2865                         i915->display.sagv.block_time_us;
2866         } else {
2867                 max_latency = LNL_PKG_C_LATENCY_MASK;
2868                 added_wake_time = 0;
2869         }
2870
2871         clear |= LNL_ADDED_WAKE_TIME_MASK | LNL_PKG_C_LATENCY_MASK;
2872         val |= REG_FIELD_PREP(LNL_PKG_C_LATENCY_MASK, max_latency);
2873         val |= REG_FIELD_PREP(LNL_ADDED_WAKE_TIME_MASK, added_wake_time);
2874
2875         intel_uncore_rmw(&i915->uncore, LNL_PKG_C_LATENCY, clear, val);
2876 }
2877
2878 static int
2879 skl_compute_wm(struct intel_atomic_state *state)
2880 {
2881         struct intel_crtc *crtc;
2882         struct intel_crtc_state __maybe_unused *new_crtc_state;
2883         int ret, i;
2884         bool enable_dpkgc = false;
2885
2886         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2887                 ret = skl_build_pipe_wm(state, crtc);
2888                 if (ret)
2889                         return ret;
2890         }
2891
2892         ret = skl_compute_ddb(state);
2893         if (ret)
2894                 return ret;
2895
2896         ret = intel_compute_sagv_mask(state);
2897         if (ret)
2898                 return ret;
2899
2900         /*
2901          * skl_compute_ddb() will have adjusted the final watermarks
2902          * based on how much ddb is available. Now we can actually
2903          * check if the final watermarks changed.
2904          */
2905         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2906                 ret = skl_wm_add_affected_planes(state, crtc);
2907                 if (ret)
2908                         return ret;
2909
2910                 if ((new_crtc_state->vrr.vmin == new_crtc_state->vrr.vmax &&
2911                      new_crtc_state->vrr.vmin == new_crtc_state->vrr.flipline) ||
2912                     !new_crtc_state->vrr.enable)
2913                         enable_dpkgc = true;
2914         }
2915
2916         skl_program_dpkgc_latency(to_i915(state->base.dev), enable_dpkgc);
2917
2918         skl_print_wm_changes(state);
2919
2920         return 0;
2921 }
2922
2923 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
2924 {
2925         level->enable = val & PLANE_WM_EN;
2926         level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
2927         level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val);
2928         level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
2929 }
2930
2931 static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
2932                                      struct skl_pipe_wm *out)
2933 {
2934         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2935         enum pipe pipe = crtc->pipe;
2936         enum plane_id plane_id;
2937         int level;
2938         u32 val;
2939
2940         for_each_plane_id_on_crtc(crtc, plane_id) {
2941                 struct skl_plane_wm *wm = &out->planes[plane_id];
2942
2943                 for (level = 0; level < i915->display.wm.num_levels; level++) {
2944                         if (plane_id != PLANE_CURSOR)
2945                                 val = intel_de_read(i915, PLANE_WM(pipe, plane_id, level));
2946                         else
2947                                 val = intel_de_read(i915, CUR_WM(pipe, level));
2948
2949                         skl_wm_level_from_reg_val(val, &wm->wm[level]);
2950                 }
2951
2952                 if (plane_id != PLANE_CURSOR)
2953                         val = intel_de_read(i915, PLANE_WM_TRANS(pipe, plane_id));
2954                 else
2955                         val = intel_de_read(i915, CUR_WM_TRANS(pipe));
2956
2957                 skl_wm_level_from_reg_val(val, &wm->trans_wm);
2958
2959                 if (HAS_HW_SAGV_WM(i915)) {
2960                         if (plane_id != PLANE_CURSOR)
2961                                 val = intel_de_read(i915, PLANE_WM_SAGV(pipe, plane_id));
2962                         else
2963                                 val = intel_de_read(i915, CUR_WM_SAGV(pipe));
2964
2965                         skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
2966
2967                         if (plane_id != PLANE_CURSOR)
2968                                 val = intel_de_read(i915, PLANE_WM_SAGV_TRANS(pipe, plane_id));
2969                         else
2970                                 val = intel_de_read(i915, CUR_WM_SAGV_TRANS(pipe));
2971
2972                         skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
2973                 } else if (DISPLAY_VER(i915) >= 12) {
2974                         wm->sagv.wm0 = wm->wm[0];
2975                         wm->sagv.trans_wm = wm->trans_wm;
2976                 }
2977         }
2978 }
2979
2980 static void skl_wm_get_hw_state(struct drm_i915_private *i915)
2981 {
2982         struct intel_display *display = &i915->display;
2983         struct intel_dbuf_state *dbuf_state =
2984                 to_intel_dbuf_state(i915->display.dbuf.obj.state);
2985         struct intel_crtc *crtc;
2986
2987         if (HAS_MBUS_JOINING(i915))
2988                 dbuf_state->joined_mbus = intel_de_read(i915, MBUS_CTL) & MBUS_JOIN;
2989
2990         dbuf_state->mdclk_cdclk_ratio = intel_mdclk_cdclk_ratio(display, &display->cdclk.hw);
2991
2992         for_each_intel_crtc(&i915->drm, crtc) {
2993                 struct intel_crtc_state *crtc_state =
2994                         to_intel_crtc_state(crtc->base.state);
2995                 enum pipe pipe = crtc->pipe;
2996                 unsigned int mbus_offset;
2997                 enum plane_id plane_id;
2998                 u8 slices;
2999
3000                 memset(&crtc_state->wm.skl.optimal, 0,
3001                        sizeof(crtc_state->wm.skl.optimal));
3002                 if (crtc_state->hw.active)
3003                         skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
3004                 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
3005
3006                 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
3007
3008                 for_each_plane_id_on_crtc(crtc, plane_id) {
3009                         struct skl_ddb_entry *ddb =
3010                                 &crtc_state->wm.skl.plane_ddb[plane_id];
3011                         struct skl_ddb_entry *ddb_y =
3012                                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
3013
3014                         if (!crtc_state->hw.active)
3015                                 continue;
3016
3017                         skl_ddb_get_hw_plane_state(i915, crtc->pipe,
3018                                                    plane_id, ddb, ddb_y);
3019
3020                         skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb);
3021                         skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
3022                 }
3023
3024                 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
3025
3026                 /*
3027                  * Used for checking overlaps, so we need absolute
3028                  * offsets instead of MBUS relative offsets.
3029                  */
3030                 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
3031                                                  dbuf_state->joined_mbus);
3032                 mbus_offset = mbus_ddb_offset(i915, slices);
3033                 crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
3034                 crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
3035
3036                 /* The slices actually used by the planes on the pipe */
3037                 dbuf_state->slices[pipe] =
3038                         skl_ddb_dbuf_slice_mask(i915, &crtc_state->wm.skl.ddb);
3039
3040                 drm_dbg_kms(&i915->drm,
3041                             "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
3042                             crtc->base.base.id, crtc->base.name,
3043                             dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
3044                             dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
3045                             str_yes_no(dbuf_state->joined_mbus));
3046         }
3047
3048         dbuf_state->enabled_slices = i915->display.dbuf.enabled_slices;
3049 }
3050
3051 static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
3052 {
3053         const struct intel_dbuf_state *dbuf_state =
3054                 to_intel_dbuf_state(i915->display.dbuf.obj.state);
3055         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
3056         struct intel_crtc *crtc;
3057
3058         for_each_intel_crtc(&i915->drm, crtc) {
3059                 const struct intel_crtc_state *crtc_state =
3060                         to_intel_crtc_state(crtc->base.state);
3061
3062                 entries[crtc->pipe] = crtc_state->wm.skl.ddb;
3063         }
3064
3065         for_each_intel_crtc(&i915->drm, crtc) {
3066                 const struct intel_crtc_state *crtc_state =
3067                         to_intel_crtc_state(crtc->base.state);
3068                 u8 slices;
3069
3070                 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
3071                                                  dbuf_state->joined_mbus);
3072                 if (dbuf_state->slices[crtc->pipe] & ~slices)
3073                         return true;
3074
3075                 if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries,
3076                                                 I915_MAX_PIPES, crtc->pipe))
3077                         return true;
3078         }
3079
3080         return false;
3081 }
3082
3083 static void skl_wm_sanitize(struct drm_i915_private *i915)
3084 {
3085         struct intel_crtc *crtc;
3086
3087         /*
3088          * On TGL/RKL (at least) the BIOS likes to assign the planes
3089          * to the wrong DBUF slices. This will cause an infinite loop
3090          * in skl_commit_modeset_enables() as it can't find a way to
3091          * transition between the old bogus DBUF layout to the new
3092          * proper DBUF layout without DBUF allocation overlaps between
3093          * the planes (which cannot be allowed or else the hardware
3094          * may hang). If we detect a bogus DBUF layout just turn off
3095          * all the planes so that skl_commit_modeset_enables() can
3096          * simply ignore them.
3097          */
3098         if (!skl_dbuf_is_misconfigured(i915))
3099                 return;
3100
3101         drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
3102
3103         for_each_intel_crtc(&i915->drm, crtc) {
3104                 struct intel_plane *plane = to_intel_plane(crtc->base.primary);
3105                 const struct intel_plane_state *plane_state =
3106                         to_intel_plane_state(plane->base.state);
3107                 struct intel_crtc_state *crtc_state =
3108                         to_intel_crtc_state(crtc->base.state);
3109
3110                 if (plane_state->uapi.visible)
3111                         intel_plane_disable_noatomic(crtc, plane);
3112
3113                 drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
3114
3115                 memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
3116         }
3117 }
3118
3119 static void skl_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3120 {
3121         skl_wm_get_hw_state(i915);
3122         skl_wm_sanitize(i915);
3123 }
3124
3125 void intel_wm_state_verify(struct intel_atomic_state *state,
3126                            struct intel_crtc *crtc)
3127 {
3128         struct drm_i915_private *i915 = to_i915(state->base.dev);
3129         const struct intel_crtc_state *new_crtc_state =
3130                 intel_atomic_get_new_crtc_state(state, crtc);
3131         struct skl_hw_state {
3132                 struct skl_ddb_entry ddb[I915_MAX_PLANES];
3133                 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
3134                 struct skl_pipe_wm wm;
3135         } *hw;
3136         const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
3137         struct intel_plane *plane;
3138         u8 hw_enabled_slices;
3139         int level;
3140
3141         if (DISPLAY_VER(i915) < 9 || !new_crtc_state->hw.active)
3142                 return;
3143
3144         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
3145         if (!hw)
3146                 return;
3147
3148         skl_pipe_wm_get_hw_state(crtc, &hw->wm);
3149
3150         skl_pipe_ddb_get_hw_state(crtc, hw->ddb, hw->ddb_y);
3151
3152         hw_enabled_slices = intel_enabled_dbuf_slices_mask(i915);
3153
3154         if (DISPLAY_VER(i915) >= 11 &&
3155             hw_enabled_slices != i915->display.dbuf.enabled_slices)
3156                 drm_err(&i915->drm,
3157                         "mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
3158                         i915->display.dbuf.enabled_slices,
3159                         hw_enabled_slices);
3160
3161         for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
3162                 const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
3163                 const struct skl_wm_level *hw_wm_level, *sw_wm_level;
3164
3165                 /* Watermarks */
3166                 for (level = 0; level < i915->display.wm.num_levels; level++) {
3167                         hw_wm_level = &hw->wm.planes[plane->id].wm[level];
3168                         sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level);
3169
3170                         if (skl_wm_level_equals(hw_wm_level, sw_wm_level))
3171                                 continue;
3172
3173                         drm_err(&i915->drm,
3174                                 "[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3175                                 plane->base.base.id, plane->base.name, level,
3176                                 sw_wm_level->enable,
3177                                 sw_wm_level->blocks,
3178                                 sw_wm_level->lines,
3179                                 hw_wm_level->enable,
3180                                 hw_wm_level->blocks,
3181                                 hw_wm_level->lines);
3182                 }
3183
3184                 hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
3185                 sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id);
3186
3187                 if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3188                         drm_err(&i915->drm,
3189                                 "[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3190                                 plane->base.base.id, plane->base.name,
3191                                 sw_wm_level->enable,
3192                                 sw_wm_level->blocks,
3193                                 sw_wm_level->lines,
3194                                 hw_wm_level->enable,
3195                                 hw_wm_level->blocks,
3196                                 hw_wm_level->lines);
3197                 }
3198
3199                 hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
3200                 sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
3201
3202                 if (HAS_HW_SAGV_WM(i915) &&
3203                     !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3204                         drm_err(&i915->drm,
3205                                 "[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3206                                 plane->base.base.id, plane->base.name,
3207                                 sw_wm_level->enable,
3208                                 sw_wm_level->blocks,
3209                                 sw_wm_level->lines,
3210                                 hw_wm_level->enable,
3211                                 hw_wm_level->blocks,
3212                                 hw_wm_level->lines);
3213                 }
3214
3215                 hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
3216                 sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
3217
3218                 if (HAS_HW_SAGV_WM(i915) &&
3219                     !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3220                         drm_err(&i915->drm,
3221                                 "[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3222                                 plane->base.base.id, plane->base.name,
3223                                 sw_wm_level->enable,
3224                                 sw_wm_level->blocks,
3225                                 sw_wm_level->lines,
3226                                 hw_wm_level->enable,
3227                                 hw_wm_level->blocks,
3228                                 hw_wm_level->lines);
3229                 }
3230
3231                 /* DDB */
3232                 hw_ddb_entry = &hw->ddb[PLANE_CURSOR];
3233                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR];
3234
3235                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
3236                         drm_err(&i915->drm,
3237                                 "[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
3238                                 plane->base.base.id, plane->base.name,
3239                                 sw_ddb_entry->start, sw_ddb_entry->end,
3240                                 hw_ddb_entry->start, hw_ddb_entry->end);
3241                 }
3242         }
3243
3244         kfree(hw);
3245 }
3246
3247 bool skl_watermark_ipc_enabled(struct drm_i915_private *i915)
3248 {
3249         return i915->display.wm.ipc_enabled;
3250 }
3251
3252 void skl_watermark_ipc_update(struct drm_i915_private *i915)
3253 {
3254         if (!HAS_IPC(i915))
3255                 return;
3256
3257         intel_de_rmw(i915, DISP_ARB_CTL2, DISP_IPC_ENABLE,
3258                      skl_watermark_ipc_enabled(i915) ? DISP_IPC_ENABLE : 0);
3259 }
3260
3261 static bool skl_watermark_ipc_can_enable(struct drm_i915_private *i915)
3262 {
3263         /* Display WA #0477 WaDisableIPC: skl */
3264         if (IS_SKYLAKE(i915))
3265                 return false;
3266
3267         /* Display WA #1141: SKL:all KBL:all CFL */
3268         if (IS_KABYLAKE(i915) ||
3269             IS_COFFEELAKE(i915) ||
3270             IS_COMETLAKE(i915))
3271                 return i915->dram_info.symmetric_memory;
3272
3273         return true;
3274 }
3275
3276 void skl_watermark_ipc_init(struct drm_i915_private *i915)
3277 {
3278         if (!HAS_IPC(i915))
3279                 return;
3280
3281         i915->display.wm.ipc_enabled = skl_watermark_ipc_can_enable(i915);
3282
3283         skl_watermark_ipc_update(i915);
3284 }
3285
3286 static void
3287 adjust_wm_latency(struct drm_i915_private *i915,
3288                   u16 wm[], int num_levels, int read_latency)
3289 {
3290         bool wm_lv_0_adjust_needed = i915->dram_info.wm_lv_0_adjust_needed;
3291         int i, level;
3292
3293         /*
3294          * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
3295          * need to be disabled. We make sure to sanitize the values out
3296          * of the punit to satisfy this requirement.
3297          */
3298         for (level = 1; level < num_levels; level++) {
3299                 if (wm[level] == 0) {
3300                         for (i = level + 1; i < num_levels; i++)
3301                                 wm[i] = 0;
3302
3303                         num_levels = level;
3304                         break;
3305                 }
3306         }
3307
3308         /*
3309          * WaWmMemoryReadLatency
3310          *
3311          * punit doesn't take into account the read latency so we need
3312          * to add proper adjustement to each valid level we retrieve
3313          * from the punit when level 0 response data is 0us.
3314          */
3315         if (wm[0] == 0) {
3316                 for (level = 0; level < num_levels; level++)
3317                         wm[level] += read_latency;
3318         }
3319
3320         /*
3321          * WA Level-0 adjustment for 16GB DIMMs: SKL+
3322          * If we could not get dimm info enable this WA to prevent from
3323          * any underrun. If not able to get Dimm info assume 16GB dimm
3324          * to avoid any underrun.
3325          */
3326         if (wm_lv_0_adjust_needed)
3327                 wm[0] += 1;
3328 }
3329
3330 static void mtl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3331 {
3332         int num_levels = i915->display.wm.num_levels;
3333         u32 val;
3334
3335         val = intel_de_read(i915, MTL_LATENCY_LP0_LP1);
3336         wm[0] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3337         wm[1] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3338
3339         val = intel_de_read(i915, MTL_LATENCY_LP2_LP3);
3340         wm[2] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3341         wm[3] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3342
3343         val = intel_de_read(i915, MTL_LATENCY_LP4_LP5);
3344         wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3345         wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3346
3347         adjust_wm_latency(i915, wm, num_levels, 6);
3348 }
3349
3350 static void skl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3351 {
3352         int num_levels = i915->display.wm.num_levels;
3353         int read_latency = DISPLAY_VER(i915) >= 12 ? 3 : 2;
3354         int mult = IS_DG2(i915) ? 2 : 1;
3355         u32 val;
3356         int ret;
3357
3358         /* read the first set of memory latencies[0:3] */
3359         val = 0; /* data0 to be programmed to 0 for first set */
3360         ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3361         if (ret) {
3362                 drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3363                 return;
3364         }
3365
3366         wm[0] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3367         wm[1] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3368         wm[2] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3369         wm[3] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3370
3371         /* read the second set of memory latencies[4:7] */
3372         val = 1; /* data0 to be programmed to 1 for second set */
3373         ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3374         if (ret) {
3375                 drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3376                 return;
3377         }
3378
3379         wm[4] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3380         wm[5] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3381         wm[6] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3382         wm[7] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3383
3384         adjust_wm_latency(i915, wm, num_levels, read_latency);
3385 }
3386
3387 static void skl_setup_wm_latency(struct drm_i915_private *i915)
3388 {
3389         if (HAS_HW_SAGV_WM(i915))
3390                 i915->display.wm.num_levels = 6;
3391         else
3392                 i915->display.wm.num_levels = 8;
3393
3394         if (DISPLAY_VER(i915) >= 14)
3395                 mtl_read_wm_latency(i915, i915->display.wm.skl_latency);
3396         else
3397                 skl_read_wm_latency(i915, i915->display.wm.skl_latency);
3398
3399         intel_print_wm_latency(i915, "Gen9 Plane", i915->display.wm.skl_latency);
3400 }
3401
3402 static const struct intel_wm_funcs skl_wm_funcs = {
3403         .compute_global_watermarks = skl_compute_wm,
3404         .get_hw_state = skl_wm_get_hw_state_and_sanitize,
3405 };
3406
3407 void skl_wm_init(struct drm_i915_private *i915)
3408 {
3409         intel_sagv_init(i915);
3410
3411         skl_setup_wm_latency(i915);
3412
3413         i915->display.funcs.wm = &skl_wm_funcs;
3414 }
3415
3416 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
3417 {
3418         struct intel_dbuf_state *dbuf_state;
3419
3420         dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
3421         if (!dbuf_state)
3422                 return NULL;
3423
3424         return &dbuf_state->base;
3425 }
3426
3427 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
3428                                      struct intel_global_state *state)
3429 {
3430         kfree(state);
3431 }
3432
3433 static const struct intel_global_state_funcs intel_dbuf_funcs = {
3434         .atomic_duplicate_state = intel_dbuf_duplicate_state,
3435         .atomic_destroy_state = intel_dbuf_destroy_state,
3436 };
3437
3438 struct intel_dbuf_state *
3439 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
3440 {
3441         struct drm_i915_private *i915 = to_i915(state->base.dev);
3442         struct intel_global_state *dbuf_state;
3443
3444         dbuf_state = intel_atomic_get_global_obj_state(state, &i915->display.dbuf.obj);
3445         if (IS_ERR(dbuf_state))
3446                 return ERR_CAST(dbuf_state);
3447
3448         return to_intel_dbuf_state(dbuf_state);
3449 }
3450
3451 int intel_dbuf_init(struct drm_i915_private *i915)
3452 {
3453         struct intel_dbuf_state *dbuf_state;
3454
3455         dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
3456         if (!dbuf_state)
3457                 return -ENOMEM;
3458
3459         intel_atomic_global_obj_init(i915, &i915->display.dbuf.obj,
3460                                      &dbuf_state->base, &intel_dbuf_funcs);
3461
3462         return 0;
3463 }
3464
3465 static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes)
3466 {
3467         switch (pipe) {
3468         case PIPE_A:
3469                 return !(active_pipes & BIT(PIPE_D));
3470         case PIPE_D:
3471                 return !(active_pipes & BIT(PIPE_A));
3472         case PIPE_B:
3473                 return !(active_pipes & BIT(PIPE_C));
3474         case PIPE_C:
3475                 return !(active_pipes & BIT(PIPE_B));
3476         default: /* to suppress compiler warning */
3477                 MISSING_CASE(pipe);
3478                 break;
3479         }
3480
3481         return false;
3482 }
3483
3484 static void intel_mbus_dbox_update(struct intel_atomic_state *state)
3485 {
3486         struct drm_i915_private *i915 = to_i915(state->base.dev);
3487         const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
3488         const struct intel_crtc *crtc;
3489         u32 val = 0;
3490
3491         if (DISPLAY_VER(i915) < 11)
3492                 return;
3493
3494         new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
3495         old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
3496         if (!new_dbuf_state ||
3497             (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus &&
3498              new_dbuf_state->active_pipes == old_dbuf_state->active_pipes))
3499                 return;
3500
3501         if (DISPLAY_VER(i915) >= 14)
3502                 val |= MBUS_DBOX_I_CREDIT(2);
3503
3504         if (DISPLAY_VER(i915) >= 12) {
3505                 val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16);
3506                 val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1);
3507                 val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN;
3508         }
3509
3510         if (DISPLAY_VER(i915) >= 14)
3511                 val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(12) :
3512                                                      MBUS_DBOX_A_CREDIT(8);
3513         else if (IS_ALDERLAKE_P(i915))
3514                 /* Wa_22010947358:adl-p */
3515                 val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(6) :
3516                                                      MBUS_DBOX_A_CREDIT(4);
3517         else
3518                 val |= MBUS_DBOX_A_CREDIT(2);
3519
3520         if (DISPLAY_VER(i915) >= 14) {
3521                 val |= MBUS_DBOX_B_CREDIT(0xA);
3522         } else if (IS_ALDERLAKE_P(i915)) {
3523                 val |= MBUS_DBOX_BW_CREDIT(2);
3524                 val |= MBUS_DBOX_B_CREDIT(8);
3525         } else if (DISPLAY_VER(i915) >= 12) {
3526                 val |= MBUS_DBOX_BW_CREDIT(2);
3527                 val |= MBUS_DBOX_B_CREDIT(12);
3528         } else {
3529                 val |= MBUS_DBOX_BW_CREDIT(1);
3530                 val |= MBUS_DBOX_B_CREDIT(8);
3531         }
3532
3533         for_each_intel_crtc_in_pipe_mask(&i915->drm, crtc, new_dbuf_state->active_pipes) {
3534                 u32 pipe_val = val;
3535
3536                 if (DISPLAY_VERx100(i915) == 1400) {
3537                         if (xelpdp_is_only_pipe_per_dbuf_bank(crtc->pipe,
3538                                                               new_dbuf_state->active_pipes))
3539                                 pipe_val |= MBUS_DBOX_BW_8CREDITS_MTL;
3540                         else
3541                                 pipe_val |= MBUS_DBOX_BW_4CREDITS_MTL;
3542                 }
3543
3544                 intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe), pipe_val);
3545         }
3546 }
3547
3548 int intel_dbuf_state_set_mdclk_cdclk_ratio(struct intel_atomic_state *state,
3549                                            int ratio)
3550 {
3551         struct intel_dbuf_state *dbuf_state;
3552
3553         dbuf_state = intel_atomic_get_dbuf_state(state);
3554         if (IS_ERR(dbuf_state))
3555                 return PTR_ERR(dbuf_state);
3556
3557         dbuf_state->mdclk_cdclk_ratio = ratio;
3558
3559         return intel_atomic_lock_global_state(&dbuf_state->base);
3560 }
3561
3562 void intel_dbuf_mdclk_cdclk_ratio_update(struct drm_i915_private *i915,
3563                                          int ratio, bool joined_mbus)
3564 {
3565         enum dbuf_slice slice;
3566
3567         if (!HAS_MBUS_JOINING(i915))
3568                 return;
3569
3570         if (DISPLAY_VER(i915) >= 20)
3571                 intel_de_rmw(i915, MBUS_CTL, MBUS_TRANSLATION_THROTTLE_MIN_MASK,
3572                              MBUS_TRANSLATION_THROTTLE_MIN(ratio - 1));
3573
3574         if (joined_mbus)
3575                 ratio *= 2;
3576
3577         drm_dbg_kms(&i915->drm, "Updating dbuf ratio to %d (mbus joined: %s)\n",
3578                     ratio, str_yes_no(joined_mbus));
3579
3580         for_each_dbuf_slice(i915, slice)
3581                 intel_de_rmw(i915, DBUF_CTL_S(slice),
3582                              DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
3583                              DBUF_MIN_TRACKER_STATE_SERVICE(ratio - 1));
3584 }
3585
3586 static void intel_dbuf_mdclk_min_tracker_update(struct intel_atomic_state *state)
3587 {
3588         struct drm_i915_private *i915 = to_i915(state->base.dev);
3589         const struct intel_dbuf_state *old_dbuf_state =
3590                 intel_atomic_get_old_dbuf_state(state);
3591         const struct intel_dbuf_state *new_dbuf_state =
3592                 intel_atomic_get_new_dbuf_state(state);
3593         int mdclk_cdclk_ratio;
3594
3595         if (intel_cdclk_is_decreasing_later(state)) {
3596                 /* cdclk/mdclk will be changed later by intel_set_cdclk_post_plane_update() */
3597                 mdclk_cdclk_ratio = old_dbuf_state->mdclk_cdclk_ratio;
3598         } else {
3599                 /* cdclk/mdclk already changed by intel_set_cdclk_pre_plane_update() */
3600                 mdclk_cdclk_ratio = new_dbuf_state->mdclk_cdclk_ratio;
3601         }
3602
3603         intel_dbuf_mdclk_cdclk_ratio_update(i915, mdclk_cdclk_ratio,
3604                                             new_dbuf_state->joined_mbus);
3605 }
3606
3607 static enum pipe intel_mbus_joined_pipe(struct intel_atomic_state *state,
3608                                         const struct intel_dbuf_state *dbuf_state)
3609 {
3610         struct intel_display *display = to_intel_display(state);
3611         struct drm_i915_private *i915 = to_i915(state->base.dev);
3612         enum pipe pipe = ffs(dbuf_state->active_pipes) - 1;
3613         const struct intel_crtc_state *new_crtc_state;
3614         struct intel_crtc *crtc;
3615
3616         drm_WARN_ON(&i915->drm, !dbuf_state->joined_mbus);
3617         drm_WARN_ON(&i915->drm, !is_power_of_2(dbuf_state->active_pipes));
3618
3619         crtc = intel_crtc_for_pipe(display, pipe);
3620         new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
3621
3622         if (new_crtc_state && !intel_crtc_needs_modeset(new_crtc_state))
3623                 return pipe;
3624         else
3625                 return INVALID_PIPE;
3626 }
3627
3628 static void intel_dbuf_mbus_join_update(struct intel_atomic_state *state,
3629                                         enum pipe pipe)
3630 {
3631         struct drm_i915_private *i915 = to_i915(state->base.dev);
3632         const struct intel_dbuf_state *old_dbuf_state =
3633                 intel_atomic_get_old_dbuf_state(state);
3634         const struct intel_dbuf_state *new_dbuf_state =
3635                 intel_atomic_get_new_dbuf_state(state);
3636         u32 mbus_ctl;
3637
3638         drm_dbg_kms(&i915->drm, "Changing mbus joined: %s -> %s (pipe: %c)\n",
3639                     str_yes_no(old_dbuf_state->joined_mbus),
3640                     str_yes_no(new_dbuf_state->joined_mbus),
3641                     pipe != INVALID_PIPE ? pipe_name(pipe) : '*');
3642
3643         if (new_dbuf_state->joined_mbus)
3644                 mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN;
3645         else
3646                 mbus_ctl = MBUS_HASHING_MODE_2x2;
3647
3648         if (pipe != INVALID_PIPE)
3649                 mbus_ctl |= MBUS_JOIN_PIPE_SELECT(pipe);
3650         else
3651                 mbus_ctl |= MBUS_JOIN_PIPE_SELECT_NONE;
3652
3653         intel_de_rmw(i915, MBUS_CTL,
3654                      MBUS_HASHING_MODE_MASK | MBUS_JOIN |
3655                      MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
3656 }
3657
3658 void intel_dbuf_mbus_pre_ddb_update(struct intel_atomic_state *state)
3659 {
3660         const struct intel_dbuf_state *new_dbuf_state =
3661                 intel_atomic_get_new_dbuf_state(state);
3662         const struct intel_dbuf_state *old_dbuf_state =
3663                 intel_atomic_get_old_dbuf_state(state);
3664
3665         if (!new_dbuf_state)
3666                 return;
3667
3668         if (!old_dbuf_state->joined_mbus && new_dbuf_state->joined_mbus) {
3669                 enum pipe pipe = intel_mbus_joined_pipe(state, new_dbuf_state);
3670
3671                 WARN_ON(!new_dbuf_state->base.changed);
3672
3673                 intel_dbuf_mbus_join_update(state, pipe);
3674                 intel_mbus_dbox_update(state);
3675                 intel_dbuf_mdclk_min_tracker_update(state);
3676         }
3677 }
3678
3679 void intel_dbuf_mbus_post_ddb_update(struct intel_atomic_state *state)
3680 {
3681         struct intel_display *display = to_intel_display(state);
3682         const struct intel_dbuf_state *new_dbuf_state =
3683                 intel_atomic_get_new_dbuf_state(state);
3684         const struct intel_dbuf_state *old_dbuf_state =
3685                 intel_atomic_get_old_dbuf_state(state);
3686
3687         if (!new_dbuf_state)
3688                 return;
3689
3690         if (old_dbuf_state->joined_mbus && !new_dbuf_state->joined_mbus) {
3691                 enum pipe pipe = intel_mbus_joined_pipe(state, old_dbuf_state);
3692
3693                 WARN_ON(!new_dbuf_state->base.changed);
3694
3695                 intel_dbuf_mdclk_min_tracker_update(state);
3696                 intel_mbus_dbox_update(state);
3697                 intel_dbuf_mbus_join_update(state, pipe);
3698
3699                 if (pipe != INVALID_PIPE) {
3700                         struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
3701
3702                         intel_crtc_wait_for_next_vblank(crtc);
3703                 }
3704         } else if (old_dbuf_state->joined_mbus == new_dbuf_state->joined_mbus &&
3705                    old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
3706                 WARN_ON(!new_dbuf_state->base.changed);
3707
3708                 intel_dbuf_mdclk_min_tracker_update(state);
3709                 intel_mbus_dbox_update(state);
3710         }
3711
3712 }
3713
3714 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
3715 {
3716         struct drm_i915_private *i915 = to_i915(state->base.dev);
3717         const struct intel_dbuf_state *new_dbuf_state =
3718                 intel_atomic_get_new_dbuf_state(state);
3719         const struct intel_dbuf_state *old_dbuf_state =
3720                 intel_atomic_get_old_dbuf_state(state);
3721         u8 old_slices, new_slices;
3722
3723         if (!new_dbuf_state)
3724                 return;
3725
3726         old_slices = old_dbuf_state->enabled_slices;
3727         new_slices = old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices;
3728
3729         if (old_slices == new_slices)
3730                 return;
3731
3732         WARN_ON(!new_dbuf_state->base.changed);
3733
3734         gen9_dbuf_slices_update(i915, new_slices);
3735 }
3736
3737 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
3738 {
3739         struct drm_i915_private *i915 = to_i915(state->base.dev);
3740         const struct intel_dbuf_state *new_dbuf_state =
3741                 intel_atomic_get_new_dbuf_state(state);
3742         const struct intel_dbuf_state *old_dbuf_state =
3743                 intel_atomic_get_old_dbuf_state(state);
3744         u8 old_slices, new_slices;
3745
3746         if (!new_dbuf_state)
3747                 return;
3748
3749         old_slices = old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices;
3750         new_slices = new_dbuf_state->enabled_slices;
3751
3752         if (old_slices == new_slices)
3753                 return;
3754
3755         WARN_ON(!new_dbuf_state->base.changed);
3756
3757         gen9_dbuf_slices_update(i915, new_slices);
3758 }
3759
3760 static int skl_watermark_ipc_status_show(struct seq_file *m, void *data)
3761 {
3762         struct drm_i915_private *i915 = m->private;
3763
3764         seq_printf(m, "Isochronous Priority Control: %s\n",
3765                    str_yes_no(skl_watermark_ipc_enabled(i915)));
3766         return 0;
3767 }
3768
3769 static int skl_watermark_ipc_status_open(struct inode *inode, struct file *file)
3770 {
3771         struct drm_i915_private *i915 = inode->i_private;
3772
3773         return single_open(file, skl_watermark_ipc_status_show, i915);
3774 }
3775
3776 static ssize_t skl_watermark_ipc_status_write(struct file *file,
3777                                               const char __user *ubuf,
3778                                               size_t len, loff_t *offp)
3779 {
3780         struct seq_file *m = file->private_data;
3781         struct drm_i915_private *i915 = m->private;
3782         intel_wakeref_t wakeref;
3783         bool enable;
3784         int ret;
3785
3786         ret = kstrtobool_from_user(ubuf, len, &enable);
3787         if (ret < 0)
3788                 return ret;
3789
3790         with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
3791                 if (!skl_watermark_ipc_enabled(i915) && enable)
3792                         drm_info(&i915->drm,
3793                                  "Enabling IPC: WM will be proper only after next commit\n");
3794                 i915->display.wm.ipc_enabled = enable;
3795                 skl_watermark_ipc_update(i915);
3796         }
3797
3798         return len;
3799 }
3800
3801 static const struct file_operations skl_watermark_ipc_status_fops = {
3802         .owner = THIS_MODULE,
3803         .open = skl_watermark_ipc_status_open,
3804         .read = seq_read,
3805         .llseek = seq_lseek,
3806         .release = single_release,
3807         .write = skl_watermark_ipc_status_write
3808 };
3809
3810 static int intel_sagv_status_show(struct seq_file *m, void *unused)
3811 {
3812         struct drm_i915_private *i915 = m->private;
3813         static const char * const sagv_status[] = {
3814                 [I915_SAGV_UNKNOWN] = "unknown",
3815                 [I915_SAGV_DISABLED] = "disabled",
3816                 [I915_SAGV_ENABLED] = "enabled",
3817                 [I915_SAGV_NOT_CONTROLLED] = "not controlled",
3818         };
3819
3820         seq_printf(m, "SAGV available: %s\n", str_yes_no(intel_has_sagv(i915)));
3821         seq_printf(m, "SAGV modparam: %s\n",
3822                    str_enabled_disabled(i915->display.params.enable_sagv));
3823         seq_printf(m, "SAGV status: %s\n", sagv_status[i915->display.sagv.status]);
3824         seq_printf(m, "SAGV block time: %d usec\n", i915->display.sagv.block_time_us);
3825
3826         return 0;
3827 }
3828
3829 DEFINE_SHOW_ATTRIBUTE(intel_sagv_status);
3830
3831 void skl_watermark_debugfs_register(struct drm_i915_private *i915)
3832 {
3833         struct drm_minor *minor = i915->drm.primary;
3834
3835         if (HAS_IPC(i915))
3836                 debugfs_create_file("i915_ipc_status", 0644, minor->debugfs_root, i915,
3837                                     &skl_watermark_ipc_status_fops);
3838
3839         if (HAS_SAGV(i915))
3840                 debugfs_create_file("i915_sagv_status", 0444, minor->debugfs_root, i915,
3841                                     &intel_sagv_status_fops);
3842 }
3843
3844 unsigned int skl_watermark_max_latency(struct drm_i915_private *i915, int initial_wm_level)
3845 {
3846         int level;
3847
3848         for (level = i915->display.wm.num_levels - 1; level >= initial_wm_level; level--) {
3849                 unsigned int latency = skl_wm_latency(i915, level, NULL);
3850
3851                 if (latency)
3852                         return latency;
3853         }
3854
3855         return 0;
3856 }