drivers/gpu/drm/i915/display/intel_dmc_wl.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright (C) 2024 Intel Corporation
   4  */
   5
   6 #include <linux/kernel.h>
   7
   8 #include <drm/drm_print.h>
   9
  10 #include "i915_drv.h"
  11 #include "i915_reg.h"
  12 #include "intel_de.h"
  13 #include "intel_dmc.h"
  14 #include "intel_dmc_regs.h"
  15 #include "intel_dmc_wl.h"
  16
  17 /**
  18  * DOC: DMC wakelock support
  19  *
  20  * Wake lock is the mechanism to cause display engine to exit DC
  21  * states to allow programming to registers that are powered down in
  22  * those states. Previous projects exited DC states automatically when
  23  * detecting programming. Now software controls the exit by
  24  * programming the wake lock. This improves system performance and
  25  * system interactions and better fits the flip queue style of
  26  * programming. Wake lock is only required when DC5, DC6, or DC6v have
  27  * been enabled in DC_STATE_EN and the wake lock mode of operation has
  28  * been enabled.
  29  *
  30  * The wakelock mechanism in DMC allows the display engine to exit DC
  31  * states explicitly before programming registers that may be powered
  32  * down.  In earlier hardware, this was done automatically and
  33  * implicitly when the display engine accessed a register.  With the
  34  * wakelock implementation, the driver asserts a wakelock in DMC,
  35  * which forces it to exit the DC state until the wakelock is
  36  * deasserted.
  37  *
  38  * The mechanism can be enabled and disabled by writing to the
  39  * DMC_WAKELOCK_CFG register.  There are also 13 control registers
  40  * that can be used to hold and release different wakelocks.  In the
  41  * current implementation, we only need one wakelock, so only
  42  * DMC_WAKELOCK1_CTL is used.  The other definitions are here for
  43  * potential future use.
  44  */
  45
  46 /*
  47  * Define DMC_WAKELOCK_CTL_TIMEOUT_US in microseconds because we use the
  48  * atomic variant of waiting MMIO.
  49  */
  50 #define DMC_WAKELOCK_CTL_TIMEOUT_US 5000
  51 #define DMC_WAKELOCK_HOLD_TIME 50
  52
  53 struct intel_dmc_wl_range {
  54         u32 start;
  55         u32 end;
  56 };
  57
  58 static const struct intel_dmc_wl_range powered_off_ranges[] = {
  59         { .start = 0x60000, .end = 0x7ffff },
  60         {},
  61 };
  62
  63 static const struct intel_dmc_wl_range xe3lpd_dc5_dc6_dmc_ranges[] = {
  64         { .start = 0x45500 }, /* DC_STATE_SEL */
  65         { .start = 0x457a0, .end = 0x457b0 }, /* DC*_RESIDENCY_COUNTER */
  66         { .start = 0x45504 }, /* DC_STATE_EN */
  67         { .start = 0x45400, .end = 0x4540c }, /* PWR_WELL_CTL_* */
  68         { .start = 0x454f0 }, /* RETENTION_CTRL */
  69
  70         /* DBUF_CTL_* */
  71         { .start = 0x44300 },
  72         { .start = 0x44304 },
  73         { .start = 0x44f00 },
  74         { .start = 0x44f04 },
  75         { .start = 0x44fe8 },
  76         { .start = 0x45008 },
  77
  78         { .start = 0x46070 }, /* CDCLK_PLL_ENABLE */
  79         { .start = 0x46000 }, /* CDCLK_CTL */
  80         { .start = 0x46008 }, /* CDCLK_SQUASH_CTL */
  81
  82         /* TRANS_CMTG_CTL_* */
  83         { .start = 0x6fa88 },
  84         { .start = 0x6fb88 },
  85
  86         { .start = 0x46430 }, /* CHICKEN_DCPR_1 */
  87         { .start = 0x46434 }, /* CHICKEN_DCPR_2 */
  88         { .start = 0x454a0 }, /* CHICKEN_DCPR_4 */
  89         { .start = 0x42084 }, /* CHICKEN_MISC_2 */
  90         { .start = 0x42088 }, /* CHICKEN_MISC_3 */
  91         { .start = 0x46160 }, /* CMTG_CLK_SEL */
  92         { .start = 0x8f000, .end = 0x8ffff }, /* Main DMC registers */
  93
  94         {},
  95 };
  96
  97 static const struct intel_dmc_wl_range xe3lpd_dc3co_dmc_ranges[] = {
  98         { .start = 0x454a0 }, /* CHICKEN_DCPR_4 */
  99
 100         { .start = 0x45504 }, /* DC_STATE_EN */
 101
 102         /* DBUF_CTL_* */
 103         { .start = 0x44300 },
 104         { .start = 0x44304 },
 105         { .start = 0x44f00 },
 106         { .start = 0x44f04 },
 107         { .start = 0x44fe8 },
 108         { .start = 0x45008 },
 109
 110         { .start = 0x46070 }, /* CDCLK_PLL_ENABLE */
 111         { .start = 0x46000 }, /* CDCLK_CTL */
 112         { .start = 0x46008 }, /* CDCLK_SQUASH_CTL */
 113         { .start = 0x8f000, .end = 0x8ffff }, /* Main DMC registers */
 114
 115         /* Scanline registers */
 116         { .start = 0x70000 },
 117         { .start = 0x70004 },
 118         { .start = 0x70014 },
 119         { .start = 0x70018 },
 120         { .start = 0x71000 },
 121         { .start = 0x71004 },
 122         { .start = 0x71014 },
 123         { .start = 0x71018 },
 124         { .start = 0x72000 },
 125         { .start = 0x72004 },
 126         { .start = 0x72014 },
 127         { .start = 0x72018 },
 128         { .start = 0x73000 },
 129         { .start = 0x73004 },
 130         { .start = 0x73014 },
 131         { .start = 0x73018 },
 132         { .start = 0x7b000 },
 133         { .start = 0x7b004 },
 134         { .start = 0x7b014 },
 135         { .start = 0x7b018 },
 136         { .start = 0x7c000 },
 137         { .start = 0x7c004 },
 138         { .start = 0x7c014 },
 139         { .start = 0x7c018 },
 140
 141         {},
 142 };
 143
 144 static void __intel_dmc_wl_release(struct intel_display *display)
 145 {
 146         struct drm_i915_private *i915 = to_i915(display->drm);
 147         struct intel_dmc_wl *wl = &display->wl;
 148
 149         WARN_ON(refcount_read(&wl->refcount));
 150
 151         queue_delayed_work(i915->unordered_wq, &wl->work,
 152                            msecs_to_jiffies(DMC_WAKELOCK_HOLD_TIME));
 153 }
 154
 155 static void intel_dmc_wl_work(struct work_struct *work)
 156 {
 157         struct intel_dmc_wl *wl =
 158                 container_of(work, struct intel_dmc_wl, work.work);
 159         struct intel_display *display =
 160                 container_of(wl, struct intel_display, wl);
 161         unsigned long flags;
 162
 163         spin_lock_irqsave(&wl->lock, flags);
 164
 165         /*
 166          * Bail out if refcount became non-zero while waiting for the spinlock,
 167          * meaning that the lock is now taken again.
 168          */
 169         if (refcount_read(&wl->refcount))
 170                 goto out_unlock;
 171
 172         __intel_de_rmw_nowl(display, DMC_WAKELOCK1_CTL, DMC_WAKELOCK_CTL_REQ, 0);
 173
 174         if (__intel_de_wait_for_register_atomic_nowl(display, DMC_WAKELOCK1_CTL,
 175                                                      DMC_WAKELOCK_CTL_ACK, 0,
 176                                                      DMC_WAKELOCK_CTL_TIMEOUT_US)) {
 177                 WARN_RATELIMIT(1, "DMC wakelock release timed out");
 178                 goto out_unlock;
 179         }
 180
 181         wl->taken = false;
 182
 183 out_unlock:
 184         spin_unlock_irqrestore(&wl->lock, flags);
 185 }
 186
 187 static void __intel_dmc_wl_take(struct intel_display *display)
 188 {
 189         struct intel_dmc_wl *wl = &display->wl;
 190
 191         /*
 192          * Only try to take the wakelock if it's not marked as taken
 193          * yet.  It may be already taken at this point if we have
 194          * already released the last reference, but the work has not
 195          * run yet.
 196          */
 197         if (wl->taken)
 198                 return;
 199
 200         __intel_de_rmw_nowl(display, DMC_WAKELOCK1_CTL, 0,
 201                             DMC_WAKELOCK_CTL_REQ);
 202
 203         /*
 204          * We need to use the atomic variant of the waiting routine
 205          * because the DMC wakelock is also taken in atomic context.
 206          */
 207         if (__intel_de_wait_for_register_atomic_nowl(display, DMC_WAKELOCK1_CTL,
 208                                                      DMC_WAKELOCK_CTL_ACK,
 209                                                      DMC_WAKELOCK_CTL_ACK,
 210                                                      DMC_WAKELOCK_CTL_TIMEOUT_US)) {
 211                 WARN_RATELIMIT(1, "DMC wakelock ack timed out");
 212                 return;
 213         }
 214
 215         wl->taken = true;
 216 }
 217
 218 static bool intel_dmc_wl_reg_in_range(i915_reg_t reg,
 219                                       const struct intel_dmc_wl_range ranges[])
 220 {
 221         u32 offset = i915_mmio_reg_offset(reg);
 222
 223         for (int i = 0; ranges[i].start; i++) {
 224                 u32 end = ranges[i].end ?: ranges[i].start;
 225
 226                 if (ranges[i].start <= offset && offset <= end)
 227                         return true;
 228         }
 229
 230         return false;
 231 }
 232
 233 static bool intel_dmc_wl_check_range(i915_reg_t reg, u32 dc_state)
 234 {
 235         const struct intel_dmc_wl_range *ranges;
 236
 237         /*
 238          * Check that the offset is in one of the ranges for which
 239          * registers are powered off during DC states.
 240          */
 241         if (intel_dmc_wl_reg_in_range(reg, powered_off_ranges))
 242                 return true;
 243
 244         /*
 245          * Check that the offset is for a register that is touched by
 246          * the DMC and requires a DC exit for proper access.
 247          */
 248         switch (dc_state) {
 249         case DC_STATE_EN_DC3CO:
 250                 ranges = xe3lpd_dc3co_dmc_ranges;
 251                 break;
 252         case DC_STATE_EN_UPTO_DC5:
 253         case DC_STATE_EN_UPTO_DC6:
 254                 ranges = xe3lpd_dc5_dc6_dmc_ranges;
 255                 break;
 256         default:
 257                 ranges = NULL;
 258         }
 259
 260         if (ranges && intel_dmc_wl_reg_in_range(reg, ranges))
 261                 return true;
 262
 263         return false;
 264 }
 265
 266 static bool __intel_dmc_wl_supported(struct intel_display *display)
 267 {
 268         return display->params.enable_dmc_wl && intel_dmc_has_payload(display);
 269 }
 270
 271 static void intel_dmc_wl_sanitize_param(struct intel_display *display)
 272 {
 273         if (!HAS_DMC_WAKELOCK(display))
 274                 display->params.enable_dmc_wl = 0;
 275         else if (display->params.enable_dmc_wl >= 0)
 276                 display->params.enable_dmc_wl = !!display->params.enable_dmc_wl;
 277         else
 278                 display->params.enable_dmc_wl = DISPLAY_VER(display) >= 30;
 279
 280         drm_dbg_kms(display->drm, "Sanitized enable_dmc_wl value: %d\n",
 281                     display->params.enable_dmc_wl);
 282 }
 283
 284 void intel_dmc_wl_init(struct intel_display *display)
 285 {
 286         struct intel_dmc_wl *wl = &display->wl;
 287
 288         intel_dmc_wl_sanitize_param(display);
 289
 290         if (!display->params.enable_dmc_wl)
 291                 return;
 292
 293         INIT_DELAYED_WORK(&wl->work, intel_dmc_wl_work);
 294         spin_lock_init(&wl->lock);
 295         refcount_set(&wl->refcount, 0);
 296 }
 297
 298 /* Must only be called as part of enabling dynamic DC states. */
 299 void intel_dmc_wl_enable(struct intel_display *display, u32 dc_state)
 300 {
 301         struct intel_dmc_wl *wl = &display->wl;
 302         unsigned long flags;
 303
 304         if (!__intel_dmc_wl_supported(display))
 305                 return;
 306
 307         spin_lock_irqsave(&wl->lock, flags);
 308
 309         wl->dc_state = dc_state;
 310
 311         if (drm_WARN_ON(display->drm, wl->enabled))
 312                 goto out_unlock;
 313
 314         /*
 315          * Enable wakelock in DMC.  We shouldn't try to take the
 316          * wakelock, because we're just enabling it, so call the
 317          * non-locking version directly here.
 318          */
 319         __intel_de_rmw_nowl(display, DMC_WAKELOCK_CFG, 0, DMC_WAKELOCK_CFG_ENABLE);
 320
 321         wl->enabled = true;
 322
 323         /*
 324          * This would be racy in the following scenario:
 325          *
 326          *   1. Function A calls intel_dmc_wl_get();
 327          *   2. Some function calls intel_dmc_wl_disable();
 328          *   3. Some function calls intel_dmc_wl_enable();
 329          *   4. Concurrently with (3), function A performs the MMIO in between
 330          *      setting DMC_WAKELOCK_CFG_ENABLE and asserting the lock with
 331          *      __intel_dmc_wl_take().
 332          *
 333          * TODO: Check with the hardware team whether it is safe to assert the
 334          * hardware lock before enabling to avoid such a scenario. Otherwise, we
 335          * would need to deal with it via software synchronization.
 336          */
 337         if (refcount_read(&wl->refcount))
 338                 __intel_dmc_wl_take(display);
 339
 340 out_unlock:
 341         spin_unlock_irqrestore(&wl->lock, flags);
 342 }
 343
 344 /* Must only be called as part of disabling dynamic DC states. */
 345 void intel_dmc_wl_disable(struct intel_display *display)
 346 {
 347         struct intel_dmc_wl *wl = &display->wl;
 348         unsigned long flags;
 349
 350         if (!__intel_dmc_wl_supported(display))
 351                 return;
 352
 353         intel_dmc_wl_flush_release_work(display);
 354
 355         spin_lock_irqsave(&wl->lock, flags);
 356
 357         if (drm_WARN_ON(display->drm, !wl->enabled))
 358                 goto out_unlock;
 359
 360         /* Disable wakelock in DMC */
 361         __intel_de_rmw_nowl(display, DMC_WAKELOCK_CFG, DMC_WAKELOCK_CFG_ENABLE, 0);
 362
 363         wl->enabled = false;
 364
 365         /*
 366          * The spec is not explicit about the expectation of existing
 367          * lock users at the moment of disabling, but it does say that we must
 368          * clear DMC_WAKELOCK_CTL_REQ, which gives us a clue that it is okay to
 369          * disable with existing lock users.
 370          *
 371          * TODO: Get the correct expectation from the hardware team.
 372          */
 373         __intel_de_rmw_nowl(display, DMC_WAKELOCK1_CTL, DMC_WAKELOCK_CTL_REQ, 0);
 374
 375         wl->taken = false;
 376
 377 out_unlock:
 378         spin_unlock_irqrestore(&wl->lock, flags);
 379 }
 380
 381 void intel_dmc_wl_flush_release_work(struct intel_display *display)
 382 {
 383         struct intel_dmc_wl *wl = &display->wl;
 384
 385         if (!__intel_dmc_wl_supported(display))
 386                 return;
 387
 388         flush_delayed_work(&wl->work);
 389 }
 390
 391 void intel_dmc_wl_get(struct intel_display *display, i915_reg_t reg)
 392 {
 393         struct intel_dmc_wl *wl = &display->wl;
 394         unsigned long flags;
 395
 396         if (!__intel_dmc_wl_supported(display))
 397                 return;
 398
 399         spin_lock_irqsave(&wl->lock, flags);
 400
 401         if (i915_mmio_reg_valid(reg) && !intel_dmc_wl_check_range(reg, wl->dc_state))
 402                 goto out_unlock;
 403
 404         if (!wl->enabled) {
 405                 if (!refcount_inc_not_zero(&wl->refcount))
 406                         refcount_set(&wl->refcount, 1);
 407                 goto out_unlock;
 408         }
 409
 410         cancel_delayed_work(&wl->work);
 411
 412         if (refcount_inc_not_zero(&wl->refcount))
 413                 goto out_unlock;
 414
 415         refcount_set(&wl->refcount, 1);
 416
 417         __intel_dmc_wl_take(display);
 418
 419 out_unlock:
 420         spin_unlock_irqrestore(&wl->lock, flags);
 421 }
 422
 423 void intel_dmc_wl_put(struct intel_display *display, i915_reg_t reg)
 424 {
 425         struct intel_dmc_wl *wl = &display->wl;
 426         unsigned long flags;
 427
 428         if (!__intel_dmc_wl_supported(display))
 429                 return;
 430
 431         spin_lock_irqsave(&wl->lock, flags);
 432
 433         if (i915_mmio_reg_valid(reg) && !intel_dmc_wl_check_range(reg, wl->dc_state))
 434                 goto out_unlock;
 435
 436         if (WARN_RATELIMIT(!refcount_read(&wl->refcount),
 437                            "Tried to put wakelock with refcount zero\n"))
 438                 goto out_unlock;
 439
 440         if (refcount_dec_and_test(&wl->refcount)) {
 441                 if (!wl->enabled)
 442                         goto out_unlock;
 443
 444                 __intel_dmc_wl_release(display);
 445
 446                 goto out_unlock;
 447         }
 448
 449 out_unlock:
 450         spin_unlock_irqrestore(&wl->lock, flags);
 451 }
 452
 453 void intel_dmc_wl_get_noreg(struct intel_display *display)
 454 {
 455         intel_dmc_wl_get(display, INVALID_MMIO_REG);
 456 }
 457
 458 void intel_dmc_wl_put_noreg(struct intel_display *display)
 459 {
 460         intel_dmc_wl_put(display, INVALID_MMIO_REG);
 461 }