2 * SPDX-License-Identifier: MIT
4 * Copyright © 2019 Intel Corporation
7 #include <linux/suspend.h>
10 #include "i915_globals.h"
11 #include "i915_params.h"
12 #include "intel_context.h"
13 #include "intel_engine_pm.h"
15 #include "intel_gt_clock_utils.h"
16 #include "intel_gt_pm.h"
17 #include "intel_gt_requests.h"
18 #include "intel_llc.h"
20 #include "intel_rc6.h"
21 #include "intel_rps.h"
22 #include "intel_wakeref.h"
24 static void user_forcewake(struct intel_gt *gt, bool suspend)
26 int count = atomic_read(>->user_wakeref);
28 /* Inside suspend/resume so single threaded, no races to worry about. */
34 GEM_BUG_ON(count > atomic_read(>->wakeref.count));
35 atomic_sub(count, >->wakeref.count);
37 atomic_add(count, >->wakeref.count);
42 static int __gt_unpark(struct intel_wakeref *wf)
44 struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
45 struct drm_i915_private *i915 = gt->i915;
49 i915_globals_unpark();
52 * It seems that the DMC likes to transition between the DC states a lot
53 * when there are no connected displays (no active power domains) during
56 * This activity has negative impact on the performance of the chip with
57 * huge latencies observed in the interrupt handler and elsewhere.
59 * Work around it by grabbing a GT IRQ power domain whilst there is any
60 * GT activity, preventing any DC state transitions.
62 gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
63 GEM_BUG_ON(!gt->awake);
65 intel_rc6_unpark(>->rc6);
66 intel_rps_unpark(>->rps);
67 i915_pmu_gt_unparked(i915);
69 intel_gt_unpark_requests(gt);
74 static int __gt_park(struct intel_wakeref *wf)
76 struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
77 intel_wakeref_t wakeref = fetch_and_zero(>->awake);
78 struct drm_i915_private *i915 = gt->i915;
82 intel_gt_park_requests(gt);
85 i915_pmu_gt_parked(i915);
86 intel_rps_park(>->rps);
87 intel_rc6_park(>->rc6);
89 /* Everything switched off, flush any residual interrupt just in case */
90 intel_synchronize_irq(i915);
92 /* Defer dropping the display power well for 100ms, it's slow! */
94 intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
101 static const struct intel_wakeref_ops wf_ops = {
106 void intel_gt_pm_init_early(struct intel_gt *gt)
108 intel_wakeref_init(>->wakeref, gt->uncore->rpm, &wf_ops);
111 void intel_gt_pm_init(struct intel_gt *gt)
114 * Enabling power-management should be "self-healing". If we cannot
115 * enable a feature, simply leave it disabled with a notice to the
118 intel_rc6_init(>->rc6);
119 intel_rps_init(>->rps);
122 static bool reset_engines(struct intel_gt *gt)
124 if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
127 return __intel_gt_reset(gt, ALL_ENGINES) == 0;
130 static void gt_sanitize(struct intel_gt *gt, bool force)
132 struct intel_engine_cs *engine;
133 enum intel_engine_id id;
134 intel_wakeref_t wakeref;
136 GT_TRACE(gt, "force:%s", yesno(force));
138 /* Use a raw wakeref to avoid calling intel_display_power_get early */
139 wakeref = intel_runtime_pm_get(gt->uncore->rpm);
140 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
142 intel_gt_check_clock_frequency(gt);
145 * As we have just resumed the machine and woken the device up from
146 * deep PCI sleep (presumably D3_cold), assume the HW has been reset
147 * back to defaults, recovering from whatever wedged state we left it
148 * in and so worth trying to use the device once more.
150 if (intel_gt_is_wedged(gt))
151 intel_gt_unset_wedged(gt);
153 for_each_engine(engine, gt, id)
154 if (engine->sanitize)
155 engine->sanitize(engine);
157 intel_uc_sanitize(>->uc);
159 for_each_engine(engine, gt, id)
160 if (engine->reset.prepare)
161 engine->reset.prepare(engine);
163 intel_uc_reset_prepare(>->uc);
165 if (reset_engines(gt) || force) {
166 for_each_engine(engine, gt, id)
167 __intel_engine_reset(engine, false);
170 for_each_engine(engine, gt, id)
171 if (engine->reset.finish)
172 engine->reset.finish(engine);
174 intel_rps_sanitize(>->rps);
176 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
177 intel_runtime_pm_put(gt->uncore->rpm, wakeref);
180 void intel_gt_pm_fini(struct intel_gt *gt)
182 intel_rc6_fini(>->rc6);
185 int intel_gt_resume(struct intel_gt *gt)
187 struct intel_engine_cs *engine;
188 enum intel_engine_id id;
191 err = intel_gt_has_init_error(gt);
198 * After resume, we may need to poke into the pinned kernel
199 * contexts to paper over any damage caused by the sudden suspend.
200 * Only the kernel contexts should remain pinned over suspend,
201 * allowing us to fixup the user contexts on their first pin.
203 gt_sanitize(gt, true);
207 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
208 intel_rc6_sanitize(>->rc6);
209 if (intel_gt_is_wedged(gt)) {
214 /* Only when the HW is re-initialised, can we replay the requests */
215 err = intel_gt_init_hw(gt);
217 drm_err(>->i915->drm,
218 "Failed to initialize GPU, declaring it wedged!\n");
222 intel_rps_enable(>->rps);
223 intel_llc_enable(>->llc);
225 for_each_engine(engine, gt, id) {
226 intel_engine_pm_get(engine);
228 engine->serial++; /* kernel context lost */
229 err = intel_engine_resume(engine);
231 intel_engine_pm_put(engine);
233 drm_err(>->i915->drm,
234 "Failed to restart %s (%d)\n",
240 intel_rc6_enable(>->rc6);
242 intel_uc_resume(>->uc);
244 user_forcewake(gt, false);
247 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
252 intel_gt_set_wedged(gt);
256 static void wait_for_suspend(struct intel_gt *gt)
258 if (!intel_gt_pm_is_awake(gt))
261 if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) {
263 * Forcibly cancel outstanding work and leave
266 intel_gt_set_wedged(gt);
267 intel_gt_retire_requests(gt);
270 intel_gt_pm_wait_for_idle(gt);
273 void intel_gt_suspend_prepare(struct intel_gt *gt)
275 user_forcewake(gt, true);
276 wait_for_suspend(gt);
278 intel_uc_suspend(>->uc);
281 static suspend_state_t pm_suspend_target(void)
283 #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
284 return pm_suspend_target_state;
286 return PM_SUSPEND_TO_IDLE;
290 void intel_gt_suspend_late(struct intel_gt *gt)
292 intel_wakeref_t wakeref;
294 /* We expect to be idle already; but also want to be independent */
295 wait_for_suspend(gt);
300 GEM_BUG_ON(gt->awake);
303 * On disabling the device, we want to turn off HW access to memory
304 * that we no longer own.
306 * However, not all suspend-states disable the device. S0 (s2idle)
307 * is effectively runtime-suspend, the device is left powered on
308 * but needs to be put into a low power state. We need to keep
309 * powermanagement enabled, but we also retain system state and so
310 * it remains safe to keep on using our allocated memory.
312 if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
315 with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
316 intel_rps_disable(>->rps);
317 intel_rc6_disable(>->rc6);
318 intel_llc_disable(>->llc);
321 gt_sanitize(gt, false);
326 void intel_gt_runtime_suspend(struct intel_gt *gt)
328 intel_uc_runtime_suspend(>->uc);
333 int intel_gt_runtime_resume(struct intel_gt *gt)
336 intel_gt_init_swizzling(gt);
337 intel_ggtt_restore_fences(gt->ggtt);
339 return intel_uc_runtime_resume(>->uc);
342 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
343 #include "selftest_gt_pm.c"