drivers/gpu/drm/i915/gt/intel_gt_pm.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <linux/string_helpers.h>
   7 #include <linux/suspend.h>
   8
   9 #include "i915_drv.h"
  10 #include "i915_irq.h"
  11 #include "i915_params.h"
  12 #include "intel_context.h"
  13 #include "intel_engine_pm.h"
  14 #include "intel_gt.h"
  15 #include "intel_gt_clock_utils.h"
  16 #include "intel_gt_pm.h"
  17 #include "intel_gt_print.h"
  18 #include "intel_gt_requests.h"
  19 #include "intel_llc.h"
  20 #include "intel_rc6.h"
  21 #include "intel_rps.h"
  22 #include "intel_wakeref.h"
  23 #include "pxp/intel_pxp_pm.h"
  24
  25 #define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2)
  26
  27 static void user_forcewake(struct intel_gt *gt, bool suspend)
  28 {
  29         int count = atomic_read(&gt->user_wakeref);
  30
  31         /* Inside suspend/resume so single threaded, no races to worry about. */
  32         if (likely(!count))
  33                 return;
  34
  35         intel_gt_pm_get(gt);
  36         if (suspend) {
  37                 GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));
  38                 atomic_sub(count, &gt->wakeref.count);
  39         } else {
  40                 atomic_add(count, &gt->wakeref.count);
  41         }
  42         intel_gt_pm_put(gt);
  43 }
  44
  45 static void runtime_begin(struct intel_gt *gt)
  46 {
  47         local_irq_disable();
  48         write_seqcount_begin(&gt->stats.lock);
  49         gt->stats.start = ktime_get();
  50         gt->stats.active = true;
  51         write_seqcount_end(&gt->stats.lock);
  52         local_irq_enable();
  53 }
  54
  55 static void runtime_end(struct intel_gt *gt)
  56 {
  57         local_irq_disable();
  58         write_seqcount_begin(&gt->stats.lock);
  59         gt->stats.active = false;
  60         gt->stats.total =
  61                 ktime_add(gt->stats.total,
  62                           ktime_sub(ktime_get(), gt->stats.start));
  63         write_seqcount_end(&gt->stats.lock);
  64         local_irq_enable();
  65 }
  66
  67 static int __gt_unpark(struct intel_wakeref *wf)
  68 {
  69         struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
  70         struct drm_i915_private *i915 = gt->i915;
  71
  72         GT_TRACE(gt, "\n");
  73
  74         /*
  75          * It seems that the DMC likes to transition between the DC states a lot
  76          * when there are no connected displays (no active power domains) during
  77          * command submission.
  78          *
  79          * This activity has negative impact on the performance of the chip with
  80          * huge latencies observed in the interrupt handler and elsewhere.
  81          *
  82          * Work around it by grabbing a GT IRQ power domain whilst there is any
  83          * GT activity, preventing any DC state transitions.
  84          */
  85         gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
  86         GEM_BUG_ON(!gt->awake);
  87
  88         intel_rc6_unpark(&gt->rc6);
  89         intel_rps_unpark(&gt->rps);
  90         i915_pmu_gt_unparked(gt);
  91         intel_guc_busyness_unpark(gt);
  92
  93         intel_gt_unpark_requests(gt);
  94         runtime_begin(gt);
  95
  96         return 0;
  97 }
  98
  99 static int __gt_park(struct intel_wakeref *wf)
 100 {
 101         struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
 102         intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);
 103         struct drm_i915_private *i915 = gt->i915;
 104
 105         GT_TRACE(gt, "\n");
 106
 107         runtime_end(gt);
 108         intel_gt_park_requests(gt);
 109
 110         intel_guc_busyness_park(gt);
 111         i915_vma_parked(gt);
 112         i915_pmu_gt_parked(gt);
 113         intel_rps_park(&gt->rps);
 114         intel_rc6_park(&gt->rc6);
 115
 116         /* Everything switched off, flush any residual interrupt just in case */
 117         intel_synchronize_irq(i915);
 118
 119         /* Defer dropping the display power well for 100ms, it's slow! */
 120         GEM_BUG_ON(!wakeref);
 121         intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
 122
 123         return 0;
 124 }
 125
 126 static const struct intel_wakeref_ops wf_ops = {
 127         .get = __gt_unpark,
 128         .put = __gt_park,
 129 };
 130
 131 void intel_gt_pm_init_early(struct intel_gt *gt)
 132 {
 133         /*
 134          * We access the runtime_pm structure via gt->i915 here rather than
 135          * gt->uncore as we do elsewhere in the file because gt->uncore is not
 136          * yet initialized for all tiles at this point in the driver startup.
 137          * runtime_pm is per-device rather than per-tile, so this is still the
 138          * correct structure.
 139          */
 140         intel_wakeref_init(&gt->wakeref, &gt->i915->runtime_pm, &wf_ops);
 141         seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);
 142 }
 143
 144 void intel_gt_pm_init(struct intel_gt *gt)
 145 {
 146         /*
 147          * Enabling power-management should be "self-healing". If we cannot
 148          * enable a feature, simply leave it disabled with a notice to the
 149          * user.
 150          */
 151         intel_rc6_init(&gt->rc6);
 152         intel_rps_init(&gt->rps);
 153 }
 154
 155 static bool reset_engines(struct intel_gt *gt)
 156 {
 157         if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
 158                 return false;
 159
 160         return __intel_gt_reset(gt, ALL_ENGINES) == 0;
 161 }
 162
 163 static void gt_sanitize(struct intel_gt *gt, bool force)
 164 {
 165         struct intel_engine_cs *engine;
 166         enum intel_engine_id id;
 167         intel_wakeref_t wakeref;
 168
 169         GT_TRACE(gt, "force:%s", str_yes_no(force));
 170
 171         /* Use a raw wakeref to avoid calling intel_display_power_get early */
 172         wakeref = intel_runtime_pm_get(gt->uncore->rpm);
 173         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 174
 175         intel_gt_check_clock_frequency(gt);
 176
 177         /*
 178          * As we have just resumed the machine and woken the device up from
 179          * deep PCI sleep (presumably D3_cold), assume the HW has been reset
 180          * back to defaults, recovering from whatever wedged state we left it
 181          * in and so worth trying to use the device once more.
 182          */
 183         if (intel_gt_is_wedged(gt))
 184                 intel_gt_unset_wedged(gt);
 185
 186         /* For GuC mode, ensure submission is disabled before stopping ring */
 187         intel_uc_reset_prepare(&gt->uc);
 188
 189         for_each_engine(engine, gt, id) {
 190                 if (engine->reset.prepare)
 191                         engine->reset.prepare(engine);
 192
 193                 if (engine->sanitize)
 194                         engine->sanitize(engine);
 195         }
 196
 197         if (reset_engines(gt) || force) {
 198                 for_each_engine(engine, gt, id)
 199                         __intel_engine_reset(engine, false);
 200         }
 201
 202         intel_uc_reset(&gt->uc, false);
 203
 204         for_each_engine(engine, gt, id)
 205                 if (engine->reset.finish)
 206                         engine->reset.finish(engine);
 207
 208         intel_rps_sanitize(&gt->rps);
 209
 210         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 211         intel_runtime_pm_put(gt->uncore->rpm, wakeref);
 212 }
 213
 214 void intel_gt_pm_fini(struct intel_gt *gt)
 215 {
 216         intel_rc6_fini(&gt->rc6);
 217 }
 218
 219 int intel_gt_resume(struct intel_gt *gt)
 220 {
 221         struct intel_engine_cs *engine;
 222         enum intel_engine_id id;
 223         int err;
 224
 225         err = intel_gt_has_unrecoverable_error(gt);
 226         if (err)
 227                 return err;
 228
 229         GT_TRACE(gt, "\n");
 230
 231         /*
 232          * After resume, we may need to poke into the pinned kernel
 233          * contexts to paper over any damage caused by the sudden suspend.
 234          * Only the kernel contexts should remain pinned over suspend,
 235          * allowing us to fixup the user contexts on their first pin.
 236          */
 237         gt_sanitize(gt, true);
 238
 239         intel_gt_pm_get(gt);
 240
 241         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 242         intel_rc6_sanitize(&gt->rc6);
 243         if (intel_gt_is_wedged(gt)) {
 244                 err = -EIO;
 245                 goto out_fw;
 246         }
 247
 248         /* Only when the HW is re-initialised, can we replay the requests */
 249         err = intel_gt_init_hw(gt);
 250         if (err) {
 251                 gt_probe_error(gt, "Failed to initialize GPU, declaring it wedged!\n");
 252                 goto err_wedged;
 253         }
 254
 255         intel_uc_reset_finish(&gt->uc);
 256
 257         intel_rps_enable(&gt->rps);
 258         intel_llc_enable(&gt->llc);
 259
 260         for_each_engine(engine, gt, id) {
 261                 intel_engine_pm_get(engine);
 262
 263                 engine->serial++; /* kernel context lost */
 264                 err = intel_engine_resume(engine);
 265
 266                 intel_engine_pm_put(engine);
 267                 if (err) {
 268                         gt_err(gt, "Failed to restart %s (%d)\n",
 269                                engine->name, err);
 270                         goto err_wedged;
 271                 }
 272         }
 273
 274         intel_rc6_enable(&gt->rc6);
 275
 276         intel_uc_resume(&gt->uc);
 277
 278         user_forcewake(gt, false);
 279
 280 out_fw:
 281         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 282         intel_gt_pm_put(gt);
 283         return err;
 284
 285 err_wedged:
 286         intel_gt_set_wedged(gt);
 287         goto out_fw;
 288 }
 289
 290 static void wait_for_suspend(struct intel_gt *gt)
 291 {
 292         if (!intel_gt_pm_is_awake(gt))
 293                 return;
 294
 295         if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {
 296                 /*
 297                  * Forcibly cancel outstanding work and leave
 298                  * the gpu quiet.
 299                  */
 300                 intel_gt_set_wedged(gt);
 301                 intel_gt_retire_requests(gt);
 302         }
 303
 304         intel_gt_pm_wait_for_idle(gt);
 305 }
 306
 307 void intel_gt_suspend_prepare(struct intel_gt *gt)
 308 {
 309         user_forcewake(gt, true);
 310         wait_for_suspend(gt);
 311 }
 312
 313 static suspend_state_t pm_suspend_target(void)
 314 {
 315 #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
 316         return pm_suspend_target_state;
 317 #else
 318         return PM_SUSPEND_TO_IDLE;
 319 #endif
 320 }
 321
 322 void intel_gt_suspend_late(struct intel_gt *gt)
 323 {
 324         intel_wakeref_t wakeref;
 325
 326         /* We expect to be idle already; but also want to be independent */
 327         wait_for_suspend(gt);
 328
 329         if (is_mock_gt(gt))
 330                 return;
 331
 332         GEM_BUG_ON(gt->awake);
 333
 334         intel_uc_suspend(&gt->uc);
 335
 336         /*
 337          * On disabling the device, we want to turn off HW access to memory
 338          * that we no longer own.
 339          *
 340          * However, not all suspend-states disable the device. S0 (s2idle)
 341          * is effectively runtime-suspend, the device is left powered on
 342          * but needs to be put into a low power state. We need to keep
 343          * powermanagement enabled, but we also retain system state and so
 344          * it remains safe to keep on using our allocated memory.
 345          */
 346         if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
 347                 return;
 348
 349         with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
 350                 intel_rps_disable(&gt->rps);
 351                 intel_rc6_disable(&gt->rc6);
 352                 intel_llc_disable(&gt->llc);
 353         }
 354
 355         gt_sanitize(gt, false);
 356
 357         GT_TRACE(gt, "\n");
 358 }
 359
 360 void intel_gt_runtime_suspend(struct intel_gt *gt)
 361 {
 362         intel_uc_runtime_suspend(&gt->uc);
 363
 364         GT_TRACE(gt, "\n");
 365 }
 366
 367 int intel_gt_runtime_resume(struct intel_gt *gt)
 368 {
 369         int ret;
 370
 371         GT_TRACE(gt, "\n");
 372         intel_gt_init_swizzling(gt);
 373         intel_ggtt_restore_fences(gt->ggtt);
 374
 375         ret = intel_uc_runtime_resume(&gt->uc);
 376         if (ret)
 377                 return ret;
 378
 379         return 0;
 380 }
 381
 382 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)
 383 {
 384         ktime_t total = gt->stats.total;
 385
 386         if (gt->stats.active)
 387                 total = ktime_add(total,
 388                                   ktime_sub(ktime_get(), gt->stats.start));
 389
 390         return total;
 391 }
 392
 393 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)
 394 {
 395         unsigned int seq;
 396         ktime_t total;
 397
 398         do {
 399                 seq = read_seqcount_begin(&gt->stats.lock);
 400                 total = __intel_gt_get_awake_time(gt);
 401         } while (read_seqcount_retry(&gt->stats.lock, seq));
 402
 403         return total;
 404 }
 405
 406 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 407 #include "selftest_gt_pm.c"
 408 #endif