drivers/gpu/drm/i915/gt/intel_gt_pm.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <linux/string_helpers.h>
   7 #include <linux/suspend.h>
   8
   9 #include "i915_drv.h"
  10 #include "i915_irq.h"
  11 #include "i915_params.h"
  12 #include "intel_context.h"
  13 #include "intel_engine_pm.h"
  14 #include "intel_gt.h"
  15 #include "intel_gt_clock_utils.h"
  16 #include "intel_gt_mcr.h"
  17 #include "intel_gt_pm.h"
  18 #include "intel_gt_print.h"
  19 #include "intel_gt_requests.h"
  20 #include "intel_llc.h"
  21 #include "intel_rc6.h"
  22 #include "intel_rps.h"
  23 #include "intel_wakeref.h"
  24 #include "pxp/intel_pxp_pm.h"
  25
  26 #define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2)
  27
  28 static void user_forcewake(struct intel_gt *gt, bool suspend)
  29 {
  30         int count = atomic_read(&gt->user_wakeref);
  31
  32         /* Inside suspend/resume so single threaded, no races to worry about. */
  33         if (likely(!count))
  34                 return;
  35
  36         intel_gt_pm_get(gt);
  37         if (suspend) {
  38                 GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));
  39                 atomic_sub(count, &gt->wakeref.count);
  40         } else {
  41                 atomic_add(count, &gt->wakeref.count);
  42         }
  43         intel_gt_pm_put(gt);
  44 }
  45
  46 static void runtime_begin(struct intel_gt *gt)
  47 {
  48         local_irq_disable();
  49         write_seqcount_begin(&gt->stats.lock);
  50         gt->stats.start = ktime_get();
  51         gt->stats.active = true;
  52         write_seqcount_end(&gt->stats.lock);
  53         local_irq_enable();
  54 }
  55
  56 static void runtime_end(struct intel_gt *gt)
  57 {
  58         local_irq_disable();
  59         write_seqcount_begin(&gt->stats.lock);
  60         gt->stats.active = false;
  61         gt->stats.total =
  62                 ktime_add(gt->stats.total,
  63                           ktime_sub(ktime_get(), gt->stats.start));
  64         write_seqcount_end(&gt->stats.lock);
  65         local_irq_enable();
  66 }
  67
  68 static int __gt_unpark(struct intel_wakeref *wf)
  69 {
  70         struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
  71         struct drm_i915_private *i915 = gt->i915;
  72
  73         GT_TRACE(gt, "\n");
  74
  75         /*
  76          * It seems that the DMC likes to transition between the DC states a lot
  77          * when there are no connected displays (no active power domains) during
  78          * command submission.
  79          *
  80          * This activity has negative impact on the performance of the chip with
  81          * huge latencies observed in the interrupt handler and elsewhere.
  82          *
  83          * Work around it by grabbing a GT IRQ power domain whilst there is any
  84          * GT activity, preventing any DC state transitions.
  85          */
  86         gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
  87         GEM_BUG_ON(!gt->awake);
  88
  89         intel_rc6_unpark(&gt->rc6);
  90         intel_rps_unpark(&gt->rps);
  91         i915_pmu_gt_unparked(gt);
  92         intel_guc_busyness_unpark(gt);
  93
  94         intel_gt_unpark_requests(gt);
  95         runtime_begin(gt);
  96
  97         return 0;
  98 }
  99
 100 static int __gt_park(struct intel_wakeref *wf)
 101 {
 102         struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
 103         intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);
 104         struct drm_i915_private *i915 = gt->i915;
 105
 106         GT_TRACE(gt, "\n");
 107
 108         runtime_end(gt);
 109         intel_gt_park_requests(gt);
 110
 111         intel_guc_busyness_park(gt);
 112         i915_vma_parked(gt);
 113         i915_pmu_gt_parked(gt);
 114         intel_rps_park(&gt->rps);
 115         intel_rc6_park(&gt->rc6);
 116
 117         /* Everything switched off, flush any residual interrupt just in case */
 118         intel_synchronize_irq(i915);
 119
 120         /* Defer dropping the display power well for 100ms, it's slow! */
 121         GEM_BUG_ON(!wakeref);
 122         intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
 123
 124         return 0;
 125 }
 126
 127 static const struct intel_wakeref_ops wf_ops = {
 128         .get = __gt_unpark,
 129         .put = __gt_park,
 130 };
 131
 132 void intel_gt_pm_init_early(struct intel_gt *gt)
 133 {
 134         /*
 135          * We access the runtime_pm structure via gt->i915 here rather than
 136          * gt->uncore as we do elsewhere in the file because gt->uncore is not
 137          * yet initialized for all tiles at this point in the driver startup.
 138          * runtime_pm is per-device rather than per-tile, so this is still the
 139          * correct structure.
 140          */
 141         intel_wakeref_init(&gt->wakeref, gt->i915, &wf_ops);
 142         seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);
 143 }
 144
 145 void intel_gt_pm_init(struct intel_gt *gt)
 146 {
 147         /*
 148          * Enabling power-management should be "self-healing". If we cannot
 149          * enable a feature, simply leave it disabled with a notice to the
 150          * user.
 151          */
 152         intel_rc6_init(&gt->rc6);
 153         intel_rps_init(&gt->rps);
 154 }
 155
 156 static bool reset_engines(struct intel_gt *gt)
 157 {
 158         if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
 159                 return false;
 160
 161         return __intel_gt_reset(gt, ALL_ENGINES) == 0;
 162 }
 163
 164 static void gt_sanitize(struct intel_gt *gt, bool force)
 165 {
 166         struct intel_engine_cs *engine;
 167         enum intel_engine_id id;
 168         intel_wakeref_t wakeref;
 169
 170         GT_TRACE(gt, "force:%s", str_yes_no(force));
 171
 172         /* Use a raw wakeref to avoid calling intel_display_power_get early */
 173         wakeref = intel_runtime_pm_get(gt->uncore->rpm);
 174         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 175
 176         intel_gt_check_clock_frequency(gt);
 177
 178         /*
 179          * As we have just resumed the machine and woken the device up from
 180          * deep PCI sleep (presumably D3_cold), assume the HW has been reset
 181          * back to defaults, recovering from whatever wedged state we left it
 182          * in and so worth trying to use the device once more.
 183          */
 184         if (intel_gt_is_wedged(gt))
 185                 intel_gt_unset_wedged(gt);
 186
 187         /* For GuC mode, ensure submission is disabled before stopping ring */
 188         intel_uc_reset_prepare(&gt->uc);
 189
 190         for_each_engine(engine, gt, id) {
 191                 if (engine->reset.prepare)
 192                         engine->reset.prepare(engine);
 193
 194                 if (engine->sanitize)
 195                         engine->sanitize(engine);
 196         }
 197
 198         if (reset_engines(gt) || force) {
 199                 for_each_engine(engine, gt, id)
 200                         __intel_engine_reset(engine, false);
 201         }
 202
 203         intel_uc_reset(&gt->uc, false);
 204
 205         for_each_engine(engine, gt, id)
 206                 if (engine->reset.finish)
 207                         engine->reset.finish(engine);
 208
 209         intel_rps_sanitize(&gt->rps);
 210
 211         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 212         intel_runtime_pm_put(gt->uncore->rpm, wakeref);
 213 }
 214
 215 void intel_gt_pm_fini(struct intel_gt *gt)
 216 {
 217         intel_rc6_fini(&gt->rc6);
 218 }
 219
 220 void intel_gt_resume_early(struct intel_gt *gt)
 221 {
 222         /*
 223          * Sanitize steer semaphores during driver resume. This is necessary
 224          * to address observed cases of steer semaphores being
 225          * held after a suspend operation. Confirmation from the hardware team
 226          * assures the safety of this operation, as no lock acquisitions
 227          * by other agents occur during driver load/resume process.
 228          */
 229         intel_gt_mcr_lock_sanitize(gt);
 230
 231         intel_uncore_resume_early(gt->uncore);
 232         intel_gt_check_and_clear_faults(gt);
 233 }
 234
 235 int intel_gt_resume(struct intel_gt *gt)
 236 {
 237         struct intel_engine_cs *engine;
 238         enum intel_engine_id id;
 239         int err;
 240
 241         err = intel_gt_has_unrecoverable_error(gt);
 242         if (err)
 243                 return err;
 244
 245         GT_TRACE(gt, "\n");
 246
 247         /*
 248          * After resume, we may need to poke into the pinned kernel
 249          * contexts to paper over any damage caused by the sudden suspend.
 250          * Only the kernel contexts should remain pinned over suspend,
 251          * allowing us to fixup the user contexts on their first pin.
 252          */
 253         gt_sanitize(gt, true);
 254
 255         intel_gt_pm_get(gt);
 256
 257         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 258         intel_rc6_sanitize(&gt->rc6);
 259         if (intel_gt_is_wedged(gt)) {
 260                 err = -EIO;
 261                 goto out_fw;
 262         }
 263
 264         /* Only when the HW is re-initialised, can we replay the requests */
 265         err = intel_gt_init_hw(gt);
 266         if (err) {
 267                 gt_probe_error(gt, "Failed to initialize GPU, declaring it wedged!\n");
 268                 goto err_wedged;
 269         }
 270
 271         intel_uc_reset_finish(&gt->uc);
 272
 273         intel_rps_enable(&gt->rps);
 274         intel_llc_enable(&gt->llc);
 275
 276         for_each_engine(engine, gt, id) {
 277                 intel_engine_pm_get(engine);
 278
 279                 engine->serial++; /* kernel context lost */
 280                 err = intel_engine_resume(engine);
 281
 282                 intel_engine_pm_put(engine);
 283                 if (err) {
 284                         gt_err(gt, "Failed to restart %s (%d)\n",
 285                                engine->name, err);
 286                         goto err_wedged;
 287                 }
 288         }
 289
 290         intel_rc6_enable(&gt->rc6);
 291
 292         intel_uc_resume(&gt->uc);
 293
 294         user_forcewake(gt, false);
 295
 296 out_fw:
 297         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 298         intel_gt_pm_put(gt);
 299         intel_gt_bind_context_set_ready(gt);
 300         return err;
 301
 302 err_wedged:
 303         intel_gt_set_wedged(gt);
 304         goto out_fw;
 305 }
 306
 307 static void wait_for_suspend(struct intel_gt *gt)
 308 {
 309         if (!intel_gt_pm_is_awake(gt))
 310                 return;
 311
 312         if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {
 313                 /*
 314                  * Forcibly cancel outstanding work and leave
 315                  * the gpu quiet.
 316                  */
 317                 intel_gt_set_wedged(gt);
 318                 intel_gt_retire_requests(gt);
 319         }
 320
 321         intel_gt_pm_wait_for_idle(gt);
 322 }
 323
 324 void intel_gt_suspend_prepare(struct intel_gt *gt)
 325 {
 326         intel_gt_bind_context_set_unready(gt);
 327         user_forcewake(gt, true);
 328         wait_for_suspend(gt);
 329 }
 330
 331 static suspend_state_t pm_suspend_target(void)
 332 {
 333 #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
 334         return pm_suspend_target_state;
 335 #else
 336         return PM_SUSPEND_TO_IDLE;
 337 #endif
 338 }
 339
 340 void intel_gt_suspend_late(struct intel_gt *gt)
 341 {
 342         intel_wakeref_t wakeref;
 343
 344         /* We expect to be idle already; but also want to be independent */
 345         wait_for_suspend(gt);
 346
 347         if (is_mock_gt(gt))
 348                 return;
 349
 350         GEM_BUG_ON(gt->awake);
 351
 352         intel_uc_suspend(&gt->uc);
 353
 354         /*
 355          * On disabling the device, we want to turn off HW access to memory
 356          * that we no longer own.
 357          *
 358          * However, not all suspend-states disable the device. S0 (s2idle)
 359          * is effectively runtime-suspend, the device is left powered on
 360          * but needs to be put into a low power state. We need to keep
 361          * powermanagement enabled, but we also retain system state and so
 362          * it remains safe to keep on using our allocated memory.
 363          */
 364         if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
 365                 return;
 366
 367         with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
 368                 intel_rps_disable(&gt->rps);
 369                 intel_rc6_disable(&gt->rc6);
 370                 intel_llc_disable(&gt->llc);
 371         }
 372
 373         gt_sanitize(gt, false);
 374
 375         GT_TRACE(gt, "\n");
 376 }
 377
 378 void intel_gt_runtime_suspend(struct intel_gt *gt)
 379 {
 380         intel_gt_bind_context_set_unready(gt);
 381         intel_uc_runtime_suspend(&gt->uc);
 382
 383         GT_TRACE(gt, "\n");
 384 }
 385
 386 int intel_gt_runtime_resume(struct intel_gt *gt)
 387 {
 388         int ret;
 389
 390         GT_TRACE(gt, "\n");
 391         intel_gt_init_swizzling(gt);
 392         intel_ggtt_restore_fences(gt->ggtt);
 393
 394         ret = intel_uc_runtime_resume(&gt->uc);
 395         if (ret)
 396                 return ret;
 397
 398         intel_gt_bind_context_set_ready(gt);
 399         return 0;
 400 }
 401
 402 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)
 403 {
 404         ktime_t total = gt->stats.total;
 405
 406         if (gt->stats.active)
 407                 total = ktime_add(total,
 408                                   ktime_sub(ktime_get(), gt->stats.start));
 409
 410         return total;
 411 }
 412
 413 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)
 414 {
 415         unsigned int seq;
 416         ktime_t total;
 417
 418         do {
 419                 seq = read_seqcount_begin(&gt->stats.lock);
 420                 total = __intel_gt_get_awake_time(gt);
 421         } while (read_seqcount_retry(&gt->stats.lock, seq));
 422
 423         return total;
 424 }
 425
 426 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 427 #include "selftest_gt_pm.c"
 428 #endif