]> Git Repo - linux.git/blob - drivers/gpu/drm/i915/gt/intel_gt_pm.c
Merge tag 'cxl-for-6.0' of git://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl
[linux.git] / drivers / gpu / drm / i915 / gt / intel_gt_pm.c
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5
6 #include <linux/string_helpers.h>
7 #include <linux/suspend.h>
8
9 #include "i915_drv.h"
10 #include "i915_params.h"
11 #include "intel_context.h"
12 #include "intel_engine_pm.h"
13 #include "intel_gt.h"
14 #include "intel_gt_clock_utils.h"
15 #include "intel_gt_pm.h"
16 #include "intel_gt_requests.h"
17 #include "intel_llc.h"
18 #include "intel_pm.h"
19 #include "intel_rc6.h"
20 #include "intel_rps.h"
21 #include "intel_wakeref.h"
22 #include "pxp/intel_pxp_pm.h"
23
24 #define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2)
25
26 static void user_forcewake(struct intel_gt *gt, bool suspend)
27 {
28         int count = atomic_read(&gt->user_wakeref);
29
30         /* Inside suspend/resume so single threaded, no races to worry about. */
31         if (likely(!count))
32                 return;
33
34         intel_gt_pm_get(gt);
35         if (suspend) {
36                 GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));
37                 atomic_sub(count, &gt->wakeref.count);
38         } else {
39                 atomic_add(count, &gt->wakeref.count);
40         }
41         intel_gt_pm_put(gt);
42 }
43
44 static void runtime_begin(struct intel_gt *gt)
45 {
46         local_irq_disable();
47         write_seqcount_begin(&gt->stats.lock);
48         gt->stats.start = ktime_get();
49         gt->stats.active = true;
50         write_seqcount_end(&gt->stats.lock);
51         local_irq_enable();
52 }
53
54 static void runtime_end(struct intel_gt *gt)
55 {
56         local_irq_disable();
57         write_seqcount_begin(&gt->stats.lock);
58         gt->stats.active = false;
59         gt->stats.total =
60                 ktime_add(gt->stats.total,
61                           ktime_sub(ktime_get(), gt->stats.start));
62         write_seqcount_end(&gt->stats.lock);
63         local_irq_enable();
64 }
65
66 static int __gt_unpark(struct intel_wakeref *wf)
67 {
68         struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
69         struct drm_i915_private *i915 = gt->i915;
70
71         GT_TRACE(gt, "\n");
72
73         /*
74          * It seems that the DMC likes to transition between the DC states a lot
75          * when there are no connected displays (no active power domains) during
76          * command submission.
77          *
78          * This activity has negative impact on the performance of the chip with
79          * huge latencies observed in the interrupt handler and elsewhere.
80          *
81          * Work around it by grabbing a GT IRQ power domain whilst there is any
82          * GT activity, preventing any DC state transitions.
83          */
84         gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
85         GEM_BUG_ON(!gt->awake);
86
87         intel_rc6_unpark(&gt->rc6);
88         intel_rps_unpark(&gt->rps);
89         i915_pmu_gt_unparked(i915);
90         intel_guc_busyness_unpark(gt);
91
92         intel_gt_unpark_requests(gt);
93         runtime_begin(gt);
94
95         return 0;
96 }
97
98 static int __gt_park(struct intel_wakeref *wf)
99 {
100         struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
101         intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);
102         struct drm_i915_private *i915 = gt->i915;
103
104         GT_TRACE(gt, "\n");
105
106         runtime_end(gt);
107         intel_gt_park_requests(gt);
108
109         intel_guc_busyness_park(gt);
110         i915_vma_parked(gt);
111         i915_pmu_gt_parked(i915);
112         intel_rps_park(&gt->rps);
113         intel_rc6_park(&gt->rc6);
114
115         /* Everything switched off, flush any residual interrupt just in case */
116         intel_synchronize_irq(i915);
117
118         /* Defer dropping the display power well for 100ms, it's slow! */
119         GEM_BUG_ON(!wakeref);
120         intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
121
122         return 0;
123 }
124
125 static const struct intel_wakeref_ops wf_ops = {
126         .get = __gt_unpark,
127         .put = __gt_park,
128 };
129
130 void intel_gt_pm_init_early(struct intel_gt *gt)
131 {
132         /*
133          * We access the runtime_pm structure via gt->i915 here rather than
134          * gt->uncore as we do elsewhere in the file because gt->uncore is not
135          * yet initialized for all tiles at this point in the driver startup.
136          * runtime_pm is per-device rather than per-tile, so this is still the
137          * correct structure.
138          */
139         intel_wakeref_init(&gt->wakeref, &gt->i915->runtime_pm, &wf_ops);
140         seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);
141 }
142
143 void intel_gt_pm_init(struct intel_gt *gt)
144 {
145         /*
146          * Enabling power-management should be "self-healing". If we cannot
147          * enable a feature, simply leave it disabled with a notice to the
148          * user.
149          */
150         intel_rc6_init(&gt->rc6);
151         intel_rps_init(&gt->rps);
152 }
153
154 static bool reset_engines(struct intel_gt *gt)
155 {
156         if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
157                 return false;
158
159         return __intel_gt_reset(gt, ALL_ENGINES) == 0;
160 }
161
162 static void gt_sanitize(struct intel_gt *gt, bool force)
163 {
164         struct intel_engine_cs *engine;
165         enum intel_engine_id id;
166         intel_wakeref_t wakeref;
167
168         GT_TRACE(gt, "force:%s", str_yes_no(force));
169
170         /* Use a raw wakeref to avoid calling intel_display_power_get early */
171         wakeref = intel_runtime_pm_get(gt->uncore->rpm);
172         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
173
174         intel_gt_check_clock_frequency(gt);
175
176         /*
177          * As we have just resumed the machine and woken the device up from
178          * deep PCI sleep (presumably D3_cold), assume the HW has been reset
179          * back to defaults, recovering from whatever wedged state we left it
180          * in and so worth trying to use the device once more.
181          */
182         if (intel_gt_is_wedged(gt))
183                 intel_gt_unset_wedged(gt);
184
185         /* For GuC mode, ensure submission is disabled before stopping ring */
186         intel_uc_reset_prepare(&gt->uc);
187
188         for_each_engine(engine, gt, id) {
189                 if (engine->reset.prepare)
190                         engine->reset.prepare(engine);
191
192                 if (engine->sanitize)
193                         engine->sanitize(engine);
194         }
195
196         if (reset_engines(gt) || force) {
197                 for_each_engine(engine, gt, id)
198                         __intel_engine_reset(engine, false);
199         }
200
201         intel_uc_reset(&gt->uc, false);
202
203         for_each_engine(engine, gt, id)
204                 if (engine->reset.finish)
205                         engine->reset.finish(engine);
206
207         intel_rps_sanitize(&gt->rps);
208
209         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
210         intel_runtime_pm_put(gt->uncore->rpm, wakeref);
211 }
212
213 void intel_gt_pm_fini(struct intel_gt *gt)
214 {
215         intel_rc6_fini(&gt->rc6);
216 }
217
218 int intel_gt_resume(struct intel_gt *gt)
219 {
220         struct intel_engine_cs *engine;
221         enum intel_engine_id id;
222         int err;
223
224         err = intel_gt_has_unrecoverable_error(gt);
225         if (err)
226                 return err;
227
228         GT_TRACE(gt, "\n");
229
230         /*
231          * After resume, we may need to poke into the pinned kernel
232          * contexts to paper over any damage caused by the sudden suspend.
233          * Only the kernel contexts should remain pinned over suspend,
234          * allowing us to fixup the user contexts on their first pin.
235          */
236         gt_sanitize(gt, true);
237
238         intel_gt_pm_get(gt);
239
240         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
241         intel_rc6_sanitize(&gt->rc6);
242         if (intel_gt_is_wedged(gt)) {
243                 err = -EIO;
244                 goto out_fw;
245         }
246
247         /* Only when the HW is re-initialised, can we replay the requests */
248         err = intel_gt_init_hw(gt);
249         if (err) {
250                 i915_probe_error(gt->i915,
251                                  "Failed to initialize GPU, declaring it wedged!\n");
252                 goto err_wedged;
253         }
254
255         intel_uc_reset_finish(&gt->uc);
256
257         intel_rps_enable(&gt->rps);
258         intel_llc_enable(&gt->llc);
259
260         for_each_engine(engine, gt, id) {
261                 intel_engine_pm_get(engine);
262
263                 engine->serial++; /* kernel context lost */
264                 err = intel_engine_resume(engine);
265
266                 intel_engine_pm_put(engine);
267                 if (err) {
268                         drm_err(&gt->i915->drm,
269                                 "Failed to restart %s (%d)\n",
270                                 engine->name, err);
271                         goto err_wedged;
272                 }
273         }
274
275         intel_rc6_enable(&gt->rc6);
276
277         intel_uc_resume(&gt->uc);
278
279         intel_pxp_resume(&gt->pxp);
280
281         user_forcewake(gt, false);
282
283 out_fw:
284         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
285         intel_gt_pm_put(gt);
286         return err;
287
288 err_wedged:
289         intel_gt_set_wedged(gt);
290         goto out_fw;
291 }
292
293 static void wait_for_suspend(struct intel_gt *gt)
294 {
295         if (!intel_gt_pm_is_awake(gt))
296                 return;
297
298         if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {
299                 /*
300                  * Forcibly cancel outstanding work and leave
301                  * the gpu quiet.
302                  */
303                 intel_gt_set_wedged(gt);
304                 intel_gt_retire_requests(gt);
305         }
306
307         intel_gt_pm_wait_for_idle(gt);
308 }
309
310 void intel_gt_suspend_prepare(struct intel_gt *gt)
311 {
312         user_forcewake(gt, true);
313         wait_for_suspend(gt);
314
315         intel_pxp_suspend_prepare(&gt->pxp);
316 }
317
318 static suspend_state_t pm_suspend_target(void)
319 {
320 #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
321         return pm_suspend_target_state;
322 #else
323         return PM_SUSPEND_TO_IDLE;
324 #endif
325 }
326
327 void intel_gt_suspend_late(struct intel_gt *gt)
328 {
329         intel_wakeref_t wakeref;
330
331         /* We expect to be idle already; but also want to be independent */
332         wait_for_suspend(gt);
333
334         if (is_mock_gt(gt))
335                 return;
336
337         GEM_BUG_ON(gt->awake);
338
339         intel_uc_suspend(&gt->uc);
340         intel_pxp_suspend(&gt->pxp);
341
342         /*
343          * On disabling the device, we want to turn off HW access to memory
344          * that we no longer own.
345          *
346          * However, not all suspend-states disable the device. S0 (s2idle)
347          * is effectively runtime-suspend, the device is left powered on
348          * but needs to be put into a low power state. We need to keep
349          * powermanagement enabled, but we also retain system state and so
350          * it remains safe to keep on using our allocated memory.
351          */
352         if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
353                 return;
354
355         with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
356                 intel_rps_disable(&gt->rps);
357                 intel_rc6_disable(&gt->rc6);
358                 intel_llc_disable(&gt->llc);
359         }
360
361         gt_sanitize(gt, false);
362
363         GT_TRACE(gt, "\n");
364 }
365
366 void intel_gt_runtime_suspend(struct intel_gt *gt)
367 {
368         intel_pxp_runtime_suspend(&gt->pxp);
369         intel_uc_runtime_suspend(&gt->uc);
370
371         GT_TRACE(gt, "\n");
372 }
373
374 int intel_gt_runtime_resume(struct intel_gt *gt)
375 {
376         int ret;
377
378         GT_TRACE(gt, "\n");
379         intel_gt_init_swizzling(gt);
380         intel_ggtt_restore_fences(gt->ggtt);
381
382         ret = intel_uc_runtime_resume(&gt->uc);
383         if (ret)
384                 return ret;
385
386         intel_pxp_runtime_resume(&gt->pxp);
387
388         return 0;
389 }
390
391 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)
392 {
393         ktime_t total = gt->stats.total;
394
395         if (gt->stats.active)
396                 total = ktime_add(total,
397                                   ktime_sub(ktime_get(), gt->stats.start));
398
399         return total;
400 }
401
402 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)
403 {
404         unsigned int seq;
405         ktime_t total;
406
407         do {
408                 seq = read_seqcount_begin(&gt->stats.lock);
409                 total = __intel_gt_get_awake_time(gt);
410         } while (read_seqcount_retry(&gt->stats.lock, seq));
411
412         return total;
413 }
414
415 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
416 #include "selftest_gt_pm.c"
417 #endif
This page took 0.059055 seconds and 4 git commands to generate.