• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include <linux/suspend.h>
7 
8 #include "i915_drv.h"
9 #include "i915_params.h"
10 #include "intel_context.h"
11 #include "intel_engine_pm.h"
12 #include "intel_gt.h"
13 #include "intel_gt_clock_utils.h"
14 #include "intel_gt_pm.h"
15 #include "intel_gt_requests.h"
16 #include "intel_llc.h"
17 #include "intel_pm.h"
18 #include "intel_rc6.h"
19 #include "intel_rps.h"
20 #include "intel_wakeref.h"
21 
user_forcewake(struct intel_gt * gt,bool suspend)22 static void user_forcewake(struct intel_gt *gt, bool suspend)
23 {
24 	int count = atomic_read(&gt->user_wakeref);
25 
26 	/* Inside suspend/resume so single threaded, no races to worry about. */
27 	if (likely(!count))
28 		return;
29 
30 	intel_gt_pm_get(gt);
31 	if (suspend) {
32 		GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));
33 		atomic_sub(count, &gt->wakeref.count);
34 	} else {
35 		atomic_add(count, &gt->wakeref.count);
36 	}
37 	intel_gt_pm_put(gt);
38 }
39 
runtime_begin(struct intel_gt * gt)40 static void runtime_begin(struct intel_gt *gt)
41 {
42 	local_irq_disable();
43 	write_seqcount_begin(&gt->stats.lock);
44 	gt->stats.start = ktime_get();
45 	gt->stats.active = true;
46 	write_seqcount_end(&gt->stats.lock);
47 	local_irq_enable();
48 }
49 
runtime_end(struct intel_gt * gt)50 static void runtime_end(struct intel_gt *gt)
51 {
52 	local_irq_disable();
53 	write_seqcount_begin(&gt->stats.lock);
54 	gt->stats.active = false;
55 	gt->stats.total =
56 		ktime_add(gt->stats.total,
57 			  ktime_sub(ktime_get(), gt->stats.start));
58 	write_seqcount_end(&gt->stats.lock);
59 	local_irq_enable();
60 }
61 
__gt_unpark(struct intel_wakeref * wf)62 static int __gt_unpark(struct intel_wakeref *wf)
63 {
64 	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
65 	struct drm_i915_private *i915 = gt->i915;
66 
67 	GT_TRACE(gt, "\n");
68 
69 	/*
70 	 * It seems that the DMC likes to transition between the DC states a lot
71 	 * when there are no connected displays (no active power domains) during
72 	 * command submission.
73 	 *
74 	 * This activity has negative impact on the performance of the chip with
75 	 * huge latencies observed in the interrupt handler and elsewhere.
76 	 *
77 	 * Work around it by grabbing a GT IRQ power domain whilst there is any
78 	 * GT activity, preventing any DC state transitions.
79 	 */
80 	gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
81 	GEM_BUG_ON(!gt->awake);
82 
83 	intel_rc6_unpark(&gt->rc6);
84 	intel_rps_unpark(&gt->rps);
85 	i915_pmu_gt_unparked(i915);
86 
87 	intel_gt_unpark_requests(gt);
88 	runtime_begin(gt);
89 
90 	return 0;
91 }
92 
__gt_park(struct intel_wakeref * wf)93 static int __gt_park(struct intel_wakeref *wf)
94 {
95 	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
96 	intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);
97 	struct drm_i915_private *i915 = gt->i915;
98 
99 	GT_TRACE(gt, "\n");
100 
101 	runtime_end(gt);
102 	intel_gt_park_requests(gt);
103 
104 	i915_vma_parked(gt);
105 	i915_pmu_gt_parked(i915);
106 	intel_rps_park(&gt->rps);
107 	intel_rc6_park(&gt->rc6);
108 
109 	/* Everything switched off, flush any residual interrupt just in case */
110 	intel_synchronize_irq(i915);
111 
112 	/* Defer dropping the display power well for 100ms, it's slow! */
113 	GEM_BUG_ON(!wakeref);
114 	intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
115 
116 	return 0;
117 }
118 
119 static const struct intel_wakeref_ops wf_ops = {
120 	.get = __gt_unpark,
121 	.put = __gt_park,
122 };
123 
intel_gt_pm_init_early(struct intel_gt * gt)124 void intel_gt_pm_init_early(struct intel_gt *gt)
125 {
126 	intel_wakeref_init(&gt->wakeref, gt->uncore->rpm, &wf_ops);
127 	seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);
128 }
129 
intel_gt_pm_init(struct intel_gt * gt)130 void intel_gt_pm_init(struct intel_gt *gt)
131 {
132 	/*
133 	 * Enabling power-management should be "self-healing". If we cannot
134 	 * enable a feature, simply leave it disabled with a notice to the
135 	 * user.
136 	 */
137 	intel_rc6_init(&gt->rc6);
138 	intel_rps_init(&gt->rps);
139 }
140 
reset_engines(struct intel_gt * gt)141 static bool reset_engines(struct intel_gt *gt)
142 {
143 	if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
144 		return false;
145 
146 	return __intel_gt_reset(gt, ALL_ENGINES) == 0;
147 }
148 
gt_sanitize(struct intel_gt * gt,bool force)149 static void gt_sanitize(struct intel_gt *gt, bool force)
150 {
151 	struct intel_engine_cs *engine;
152 	enum intel_engine_id id;
153 	intel_wakeref_t wakeref;
154 
155 	GT_TRACE(gt, "force:%s", yesno(force));
156 
157 	/* Use a raw wakeref to avoid calling intel_display_power_get early */
158 	wakeref = intel_runtime_pm_get(gt->uncore->rpm);
159 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
160 
161 	intel_gt_check_clock_frequency(gt);
162 
163 	/*
164 	 * As we have just resumed the machine and woken the device up from
165 	 * deep PCI sleep (presumably D3_cold), assume the HW has been reset
166 	 * back to defaults, recovering from whatever wedged state we left it
167 	 * in and so worth trying to use the device once more.
168 	 */
169 	if (intel_gt_is_wedged(gt))
170 		intel_gt_unset_wedged(gt);
171 
172 	for_each_engine(engine, gt, id)
173 		if (engine->reset.prepare)
174 			engine->reset.prepare(engine);
175 
176 	intel_uc_reset_prepare(&gt->uc);
177 
178 	for_each_engine(engine, gt, id)
179 		if (engine->sanitize)
180 			engine->sanitize(engine);
181 
182 	if (reset_engines(gt) || force) {
183 		for_each_engine(engine, gt, id)
184 			__intel_engine_reset(engine, false);
185 	}
186 
187 	intel_uc_reset(&gt->uc, false);
188 
189 	for_each_engine(engine, gt, id)
190 		if (engine->reset.finish)
191 			engine->reset.finish(engine);
192 
193 	intel_rps_sanitize(&gt->rps);
194 
195 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
196 	intel_runtime_pm_put(gt->uncore->rpm, wakeref);
197 }
198 
intel_gt_pm_fini(struct intel_gt * gt)199 void intel_gt_pm_fini(struct intel_gt *gt)
200 {
201 	intel_rc6_fini(&gt->rc6);
202 }
203 
intel_gt_resume(struct intel_gt * gt)204 int intel_gt_resume(struct intel_gt *gt)
205 {
206 	struct intel_engine_cs *engine;
207 	enum intel_engine_id id;
208 	int err;
209 
210 	err = intel_gt_has_unrecoverable_error(gt);
211 	if (err)
212 		return err;
213 
214 	GT_TRACE(gt, "\n");
215 
216 	/*
217 	 * After resume, we may need to poke into the pinned kernel
218 	 * contexts to paper over any damage caused by the sudden suspend.
219 	 * Only the kernel contexts should remain pinned over suspend,
220 	 * allowing us to fixup the user contexts on their first pin.
221 	 */
222 	gt_sanitize(gt, true);
223 
224 	intel_gt_pm_get(gt);
225 
226 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
227 	intel_rc6_sanitize(&gt->rc6);
228 	if (intel_gt_is_wedged(gt)) {
229 		err = -EIO;
230 		goto out_fw;
231 	}
232 
233 	/* Only when the HW is re-initialised, can we replay the requests */
234 	err = intel_gt_init_hw(gt);
235 	if (err) {
236 		i915_probe_error(gt->i915,
237 				 "Failed to initialize GPU, declaring it wedged!\n");
238 		goto err_wedged;
239 	}
240 
241 	intel_uc_reset_finish(&gt->uc);
242 
243 	intel_rps_enable(&gt->rps);
244 	intel_llc_enable(&gt->llc);
245 
246 	for_each_engine(engine, gt, id) {
247 		intel_engine_pm_get(engine);
248 
249 		engine->serial++; /* kernel context lost */
250 		err = intel_engine_resume(engine);
251 
252 		intel_engine_pm_put(engine);
253 		if (err) {
254 			drm_err(&gt->i915->drm,
255 				"Failed to restart %s (%d)\n",
256 				engine->name, err);
257 			goto err_wedged;
258 		}
259 	}
260 
261 	intel_rc6_enable(&gt->rc6);
262 
263 	intel_uc_resume(&gt->uc);
264 
265 	user_forcewake(gt, false);
266 
267 out_fw:
268 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
269 	intel_gt_pm_put(gt);
270 	return err;
271 
272 err_wedged:
273 	intel_gt_set_wedged(gt);
274 	goto out_fw;
275 }
276 
wait_for_suspend(struct intel_gt * gt)277 static void wait_for_suspend(struct intel_gt *gt)
278 {
279 	if (!intel_gt_pm_is_awake(gt))
280 		return;
281 
282 	if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) {
283 		/*
284 		 * Forcibly cancel outstanding work and leave
285 		 * the gpu quiet.
286 		 */
287 		intel_gt_set_wedged(gt);
288 		intel_gt_retire_requests(gt);
289 	}
290 
291 	intel_gt_pm_wait_for_idle(gt);
292 }
293 
intel_gt_suspend_prepare(struct intel_gt * gt)294 void intel_gt_suspend_prepare(struct intel_gt *gt)
295 {
296 	user_forcewake(gt, true);
297 	wait_for_suspend(gt);
298 
299 	intel_uc_suspend(&gt->uc);
300 }
301 
pm_suspend_target(void)302 static suspend_state_t pm_suspend_target(void)
303 {
304 #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
305 	return pm_suspend_target_state;
306 #else
307 	return PM_SUSPEND_TO_IDLE;
308 #endif
309 }
310 
intel_gt_suspend_late(struct intel_gt * gt)311 void intel_gt_suspend_late(struct intel_gt *gt)
312 {
313 	intel_wakeref_t wakeref;
314 
315 	/* We expect to be idle already; but also want to be independent */
316 	wait_for_suspend(gt);
317 
318 	if (is_mock_gt(gt))
319 		return;
320 
321 	GEM_BUG_ON(gt->awake);
322 
323 	/*
324 	 * On disabling the device, we want to turn off HW access to memory
325 	 * that we no longer own.
326 	 *
327 	 * However, not all suspend-states disable the device. S0 (s2idle)
328 	 * is effectively runtime-suspend, the device is left powered on
329 	 * but needs to be put into a low power state. We need to keep
330 	 * powermanagement enabled, but we also retain system state and so
331 	 * it remains safe to keep on using our allocated memory.
332 	 */
333 	if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
334 		return;
335 
336 	with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
337 		intel_rps_disable(&gt->rps);
338 		intel_rc6_disable(&gt->rc6);
339 		intel_llc_disable(&gt->llc);
340 	}
341 
342 	gt_sanitize(gt, false);
343 
344 	GT_TRACE(gt, "\n");
345 }
346 
intel_gt_runtime_suspend(struct intel_gt * gt)347 void intel_gt_runtime_suspend(struct intel_gt *gt)
348 {
349 	intel_uc_runtime_suspend(&gt->uc);
350 
351 	GT_TRACE(gt, "\n");
352 }
353 
intel_gt_runtime_resume(struct intel_gt * gt)354 int intel_gt_runtime_resume(struct intel_gt *gt)
355 {
356 	GT_TRACE(gt, "\n");
357 	intel_gt_init_swizzling(gt);
358 	intel_ggtt_restore_fences(gt->ggtt);
359 
360 	return intel_uc_runtime_resume(&gt->uc);
361 }
362 
__intel_gt_get_awake_time(const struct intel_gt * gt)363 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)
364 {
365 	ktime_t total = gt->stats.total;
366 
367 	if (gt->stats.active)
368 		total = ktime_add(total,
369 				  ktime_sub(ktime_get(), gt->stats.start));
370 
371 	return total;
372 }
373 
intel_gt_get_awake_time(const struct intel_gt * gt)374 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)
375 {
376 	unsigned int seq;
377 	ktime_t total;
378 
379 	do {
380 		seq = read_seqcount_begin(&gt->stats.lock);
381 		total = __intel_gt_get_awake_time(gt);
382 	} while (read_seqcount_retry(&gt->stats.lock, seq));
383 
384 	return total;
385 }
386 
387 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
388 #include "selftest_gt_pm.c"
389 #endif
390