drivers/gpu/drm/i915/gt/intel_gt_pm.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <linux/string_helpers.h>
   7 #include <linux/suspend.h>
   8
   9 #include "i915_drv.h"
  10 #include "i915_params.h"
  11 #include "intel_context.h"
  12 #include "intel_engine_pm.h"
  13 #include "intel_gt.h"
  14 #include "intel_gt_clock_utils.h"
  15 #include "intel_gt_pm.h"
  16 #include "intel_gt_requests.h"
  17 #include "intel_llc.h"
  18 #include "intel_pm.h"
  19 #include "intel_rc6.h"
  20 #include "intel_rps.h"
  21 #include "intel_wakeref.h"
  22 #include "pxp/intel_pxp_pm.h"
  23
  24 #define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2)
  25
  26 static void user_forcewake(struct intel_gt *gt, bool suspend)
  27 {
  28         int count = atomic_read(&gt->user_wakeref);
  29
  30         /* Inside suspend/resume so single threaded, no races to worry about. */
  31         if (likely(!count))
  32                 return;
  33
  34         intel_gt_pm_get(gt);
  35         if (suspend) {
  36                 GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));
  37                 atomic_sub(count, &gt->wakeref.count);
  38         } else {
  39                 atomic_add(count, &gt->wakeref.count);
  40         }
  41         intel_gt_pm_put(gt);
  42 }
  43
  44 static void runtime_begin(struct intel_gt *gt)
  45 {
  46         local_irq_disable();
  47         write_seqcount_begin(&gt->stats.lock);
  48         gt->stats.start = ktime_get();
  49         gt->stats.active = true;
  50         write_seqcount_end(&gt->stats.lock);
  51         local_irq_enable();
  52 }
  53
  54 static void runtime_end(struct intel_gt *gt)
  55 {
  56         local_irq_disable();
  57         write_seqcount_begin(&gt->stats.lock);
  58         gt->stats.active = false;
  59         gt->stats.total =
  60                 ktime_add(gt->stats.total,
  61                           ktime_sub(ktime_get(), gt->stats.start));
  62         write_seqcount_end(&gt->stats.lock);
  63         local_irq_enable();
  64 }
  65
  66 static int __gt_unpark(struct intel_wakeref *wf)
  67 {
  68         struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
  69         struct drm_i915_private *i915 = gt->i915;
  70
  71         GT_TRACE(gt, "\n");
  72
  73         /*
  74          * It seems that the DMC likes to transition between the DC states a lot
  75          * when there are no connected displays (no active power domains) during
  76          * command submission.
  77          *
  78          * This activity has negative impact on the performance of the chip with
  79          * huge latencies observed in the interrupt handler and elsewhere.
  80          *
  81          * Work around it by grabbing a GT IRQ power domain whilst there is any
  82          * GT activity, preventing any DC state transitions.
  83          */
  84         gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
  85         GEM_BUG_ON(!gt->awake);
  86
  87         intel_rc6_unpark(&gt->rc6);
  88         intel_rps_unpark(&gt->rps);
  89         i915_pmu_gt_unparked(i915);
  90         intel_guc_busyness_unpark(gt);
  91
  92         intel_gt_unpark_requests(gt);
  93         runtime_begin(gt);
  94
  95         return 0;
  96 }
  97
  98 static int __gt_park(struct intel_wakeref *wf)
  99 {
 100         struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
 101         intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);
 102         struct drm_i915_private *i915 = gt->i915;
 103
 104         GT_TRACE(gt, "\n");
 105
 106         runtime_end(gt);
 107         intel_gt_park_requests(gt);
 108
 109         intel_guc_busyness_park(gt);
 110         i915_vma_parked(gt);
 111         i915_pmu_gt_parked(i915);
 112         intel_rps_park(&gt->rps);
 113         intel_rc6_park(&gt->rc6);
 114
 115         /* Everything switched off, flush any residual interrupt just in case */
 116         intel_synchronize_irq(i915);
 117
 118         /* Defer dropping the display power well for 100ms, it's slow! */
 119         GEM_BUG_ON(!wakeref);
 120         intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
 121
 122         return 0;
 123 }
 124
 125 static const struct intel_wakeref_ops wf_ops = {
 126         .get = __gt_unpark,
 127         .put = __gt_park,
 128 };
 129
 130 void intel_gt_pm_init_early(struct intel_gt *gt)
 131 {
 132         /*
 133          * We access the runtime_pm structure via gt->i915 here rather than
 134          * gt->uncore as we do elsewhere in the file because gt->uncore is not
 135          * yet initialized for all tiles at this point in the driver startup.
 136          * runtime_pm is per-device rather than per-tile, so this is still the
 137          * correct structure.
 138          */
 139         intel_wakeref_init(&gt->wakeref, &gt->i915->runtime_pm, &wf_ops);
 140         seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);
 141 }
 142
 143 void intel_gt_pm_init(struct intel_gt *gt)
 144 {
 145         /*
 146          * Enabling power-management should be "self-healing". If we cannot
 147          * enable a feature, simply leave it disabled with a notice to the
 148          * user.
 149          */
 150         intel_rc6_init(&gt->rc6);
 151         intel_rps_init(&gt->rps);
 152 }
 153
 154 static bool reset_engines(struct intel_gt *gt)
 155 {
 156         if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
 157                 return false;
 158
 159         return __intel_gt_reset(gt, ALL_ENGINES) == 0;
 160 }
 161
 162 static void gt_sanitize(struct intel_gt *gt, bool force)
 163 {
 164         struct intel_engine_cs *engine;
 165         enum intel_engine_id id;
 166         intel_wakeref_t wakeref;
 167
 168         GT_TRACE(gt, "force:%s", str_yes_no(force));
 169
 170         /* Use a raw wakeref to avoid calling intel_display_power_get early */
 171         wakeref = intel_runtime_pm_get(gt->uncore->rpm);
 172         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 173
 174         intel_gt_check_clock_frequency(gt);
 175
 176         /*
 177          * As we have just resumed the machine and woken the device up from
 178          * deep PCI sleep (presumably D3_cold), assume the HW has been reset
 179          * back to defaults, recovering from whatever wedged state we left it
 180          * in and so worth trying to use the device once more.
 181          */
 182         if (intel_gt_is_wedged(gt))
 183                 intel_gt_unset_wedged(gt);
 184
 185         /* For GuC mode, ensure submission is disabled before stopping ring */
 186         intel_uc_reset_prepare(&gt->uc);
 187
 188         for_each_engine(engine, gt, id) {
 189                 if (engine->reset.prepare)
 190                         engine->reset.prepare(engine);
 191
 192                 if (engine->sanitize)
 193                         engine->sanitize(engine);
 194         }
 195
 196         if (reset_engines(gt) || force) {
 197                 for_each_engine(engine, gt, id)
 198                         __intel_engine_reset(engine, false);
 199         }
 200
 201         intel_uc_reset(&gt->uc, false);
 202
 203         for_each_engine(engine, gt, id)
 204                 if (engine->reset.finish)
 205                         engine->reset.finish(engine);
 206
 207         intel_rps_sanitize(&gt->rps);
 208
 209         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 210         intel_runtime_pm_put(gt->uncore->rpm, wakeref);
 211 }
 212
 213 void intel_gt_pm_fini(struct intel_gt *gt)
 214 {
 215         intel_rc6_fini(&gt->rc6);
 216 }
 217
 218 int intel_gt_resume(struct intel_gt *gt)
 219 {
 220         struct intel_engine_cs *engine;
 221         enum intel_engine_id id;
 222         int err;
 223
 224         err = intel_gt_has_unrecoverable_error(gt);
 225         if (err)
 226                 return err;
 227
 228         GT_TRACE(gt, "\n");
 229
 230         /*
 231          * After resume, we may need to poke into the pinned kernel
 232          * contexts to paper over any damage caused by the sudden suspend.
 233          * Only the kernel contexts should remain pinned over suspend,
 234          * allowing us to fixup the user contexts on their first pin.
 235          */
 236         gt_sanitize(gt, true);
 237
 238         intel_gt_pm_get(gt);
 239
 240         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 241         intel_rc6_sanitize(&gt->rc6);
 242         if (intel_gt_is_wedged(gt)) {
 243                 err = -EIO;
 244                 goto out_fw;
 245         }
 246
 247         /* Only when the HW is re-initialised, can we replay the requests */
 248         err = intel_gt_init_hw(gt);
 249         if (err) {
 250                 i915_probe_error(gt->i915,
 251                                  "Failed to initialize GPU, declaring it wedged!\n");
 252                 goto err_wedged;
 253         }
 254
 255         intel_uc_reset_finish(&gt->uc);
 256
 257         intel_rps_enable(&gt->rps);
 258         intel_llc_enable(&gt->llc);
 259
 260         for_each_engine(engine, gt, id) {
 261                 intel_engine_pm_get(engine);
 262
 263                 engine->serial++; /* kernel context lost */
 264                 err = intel_engine_resume(engine);
 265
 266                 intel_engine_pm_put(engine);
 267                 if (err) {
 268                         drm_err(&gt->i915->drm,
 269                                 "Failed to restart %s (%d)\n",
 270                                 engine->name, err);
 271                         goto err_wedged;
 272                 }
 273         }
 274
 275         intel_rc6_enable(&gt->rc6);
 276
 277         intel_uc_resume(&gt->uc);
 278
 279         intel_pxp_resume(&gt->pxp);
 280
 281         user_forcewake(gt, false);
 282
 283 out_fw:
 284         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 285         intel_gt_pm_put(gt);
 286         return err;
 287
 288 err_wedged:
 289         intel_gt_set_wedged(gt);
 290         goto out_fw;
 291 }
 292
 293 static void wait_for_suspend(struct intel_gt *gt)
 294 {
 295         if (!intel_gt_pm_is_awake(gt))
 296                 return;
 297
 298         if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {
 299                 /*
 300                  * Forcibly cancel outstanding work and leave
 301                  * the gpu quiet.
 302                  */
 303                 intel_gt_set_wedged(gt);
 304                 intel_gt_retire_requests(gt);
 305         }
 306
 307         intel_gt_pm_wait_for_idle(gt);
 308 }
 309
 310 void intel_gt_suspend_prepare(struct intel_gt *gt)
 311 {
 312         user_forcewake(gt, true);
 313         wait_for_suspend(gt);
 314
 315         intel_pxp_suspend_prepare(&gt->pxp);
 316 }
 317
 318 static suspend_state_t pm_suspend_target(void)
 319 {
 320 #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
 321         return pm_suspend_target_state;
 322 #else
 323         return PM_SUSPEND_TO_IDLE;
 324 #endif
 325 }
 326
 327 void intel_gt_suspend_late(struct intel_gt *gt)
 328 {
 329         intel_wakeref_t wakeref;
 330
 331         /* We expect to be idle already; but also want to be independent */
 332         wait_for_suspend(gt);
 333
 334         if (is_mock_gt(gt))
 335                 return;
 336
 337         GEM_BUG_ON(gt->awake);
 338
 339         intel_uc_suspend(&gt->uc);
 340         intel_pxp_suspend(&gt->pxp);
 341
 342         /*
 343          * On disabling the device, we want to turn off HW access to memory
 344          * that we no longer own.
 345          *
 346          * However, not all suspend-states disable the device. S0 (s2idle)
 347          * is effectively runtime-suspend, the device is left powered on
 348          * but needs to be put into a low power state. We need to keep
 349          * powermanagement enabled, but we also retain system state and so
 350          * it remains safe to keep on using our allocated memory.
 351          */
 352         if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
 353                 return;
 354
 355         with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
 356                 intel_rps_disable(&gt->rps);
 357                 intel_rc6_disable(&gt->rc6);
 358                 intel_llc_disable(&gt->llc);
 359         }
 360
 361         gt_sanitize(gt, false);
 362
 363         GT_TRACE(gt, "\n");
 364 }
 365
 366 void intel_gt_runtime_suspend(struct intel_gt *gt)
 367 {
 368         intel_pxp_runtime_suspend(&gt->pxp);
 369         intel_uc_runtime_suspend(&gt->uc);
 370
 371         GT_TRACE(gt, "\n");
 372 }
 373
 374 int intel_gt_runtime_resume(struct intel_gt *gt)
 375 {
 376         int ret;
 377
 378         GT_TRACE(gt, "\n");
 379         intel_gt_init_swizzling(gt);
 380         intel_ggtt_restore_fences(gt->ggtt);
 381
 382         ret = intel_uc_runtime_resume(&gt->uc);
 383         if (ret)
 384                 return ret;
 385
 386         intel_pxp_runtime_resume(&gt->pxp);
 387
 388         return 0;
 389 }
 390
 391 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)
 392 {
 393         ktime_t total = gt->stats.total;
 394
 395         if (gt->stats.active)
 396                 total = ktime_add(total,
 397                                   ktime_sub(ktime_get(), gt->stats.start));
 398
 399         return total;
 400 }
 401
 402 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)
 403 {
 404         unsigned int seq;
 405         ktime_t total;
 406
 407         do {
 408                 seq = read_seqcount_begin(&gt->stats.lock);
 409                 total = __intel_gt_get_awake_time(gt);
 410         } while (read_seqcount_retry(&gt->stats.lock, seq));
 411
 412         return total;
 413 }
 414
 415 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 416 #include "selftest_gt_pm.c"
 417 #endif