78 static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in init_intel_runtime_pm_wakeref()  argument
80 	spin_lock_init(&rpm->debug.lock);  in init_intel_runtime_pm_wakeref()
84 track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in track_intel_runtime_pm_wakeref()  argument
89 	if (!rpm->available)  in track_intel_runtime_pm_wakeref()
96 	spin_lock_irqsave(&rpm->debug.lock, flags);  in track_intel_runtime_pm_wakeref()
98 	if (!rpm->debug.count)  in track_intel_runtime_pm_wakeref()
99 		rpm->debug.last_acquire = stack;  in track_intel_runtime_pm_wakeref()
101 	stacks = krealloc(rpm->debug.owners,  in track_intel_runtime_pm_wakeref()
102 			  (rpm->debug.count + 1) * sizeof(*stacks),  in track_intel_runtime_pm_wakeref()
105 		stacks[rpm->debug.count++] = stack;  in track_intel_runtime_pm_wakeref()
106 		rpm->debug.owners = stacks;  in track_intel_runtime_pm_wakeref()
111 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in track_intel_runtime_pm_wakeref()
116 static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in untrack_intel_runtime_pm_wakeref()  argument
119 	struct drm_i915_private *i915 = container_of(rpm,  in untrack_intel_runtime_pm_wakeref()
128 	spin_lock_irqsave(&rpm->debug.lock, flags);  in untrack_intel_runtime_pm_wakeref()
129 	for (n = rpm->debug.count; n--; ) {  in untrack_intel_runtime_pm_wakeref()
130 		if (rpm->debug.owners[n] == stack) {  in untrack_intel_runtime_pm_wakeref()
131 			memmove(rpm->debug.owners + n,  in untrack_intel_runtime_pm_wakeref()
132 				rpm->debug.owners + n + 1,  in untrack_intel_runtime_pm_wakeref()
133 				(--rpm->debug.count - n) * sizeof(stack));  in untrack_intel_runtime_pm_wakeref()
138 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in untrack_intel_runtime_pm_wakeref()
142 		     rpm->debug.count, atomic_read(&rpm->wakeref_count))) {  in untrack_intel_runtime_pm_wakeref()
152 		stack = READ_ONCE(rpm->debug.last_release);  in untrack_intel_runtime_pm_wakeref()
237 __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)  in __intel_wakeref_dec_and_check_tracking()  argument
242 	if (!atomic_dec_and_lock_irqsave(&rpm->wakeref_count,  in __intel_wakeref_dec_and_check_tracking()
243 					 &rpm->debug.lock,  in __intel_wakeref_dec_and_check_tracking()
247 	__untrack_all_wakerefs(&rpm->debug, &dbg);  in __intel_wakeref_dec_and_check_tracking()
248 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in __intel_wakeref_dec_and_check_tracking()
254 untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)  in untrack_all_intel_runtime_pm_wakerefs()  argument
259 	spin_lock_irqsave(&rpm->debug.lock, flags);  in untrack_all_intel_runtime_pm_wakerefs()
260 	__untrack_all_wakerefs(&rpm->debug, &dbg);  in untrack_all_intel_runtime_pm_wakerefs()
261 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in untrack_all_intel_runtime_pm_wakerefs()
266 void print_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in print_intel_runtime_pm_wakeref()  argument
275 		spin_lock_irq(&rpm->debug.lock);  in print_intel_runtime_pm_wakeref()
276 		dbg.count = rpm->debug.count;  in print_intel_runtime_pm_wakeref()
279 			       rpm->debug.owners,  in print_intel_runtime_pm_wakeref()
282 		dbg.last_acquire = rpm->debug.last_acquire;  in print_intel_runtime_pm_wakeref()
283 		dbg.last_release = rpm->debug.last_release;  in print_intel_runtime_pm_wakeref()
284 		spin_unlock_irq(&rpm->debug.lock);  in print_intel_runtime_pm_wakeref()
305 static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in init_intel_runtime_pm_wakeref()  argument
310 track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in track_intel_runtime_pm_wakeref()  argument
315 static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in untrack_intel_runtime_pm_wakeref()  argument
321 __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)  in __intel_wakeref_dec_and_check_tracking()  argument
323 	atomic_dec(&rpm->wakeref_count);  in __intel_wakeref_dec_and_check_tracking()
327 untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)  in untrack_all_intel_runtime_pm_wakerefs()  argument
334 intel_runtime_pm_acquire(struct intel_runtime_pm *rpm, bool wakelock)  in intel_runtime_pm_acquire()  argument
337 		atomic_add(1 + INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count);  in intel_runtime_pm_acquire()
338 		assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_acquire()
340 		atomic_inc(&rpm->wakeref_count);  in intel_runtime_pm_acquire()
341 		assert_rpm_raw_wakeref_held(rpm);  in intel_runtime_pm_acquire()
346 intel_runtime_pm_release(struct intel_runtime_pm *rpm, int wakelock)  in intel_runtime_pm_release()  argument
349 		assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_release()
350 		atomic_sub(INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count);  in intel_runtime_pm_release()
352 		assert_rpm_raw_wakeref_held(rpm);  in intel_runtime_pm_release()
355 	__intel_wakeref_dec_and_check_tracking(rpm);  in intel_runtime_pm_release()
358 static intel_wakeref_t __intel_runtime_pm_get(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_get()  argument
361 	struct drm_i915_private *i915 = container_of(rpm,  in __intel_runtime_pm_get()
366 	ret = pm_runtime_get_sync(rpm->kdev);  in __intel_runtime_pm_get()
370 	intel_runtime_pm_acquire(rpm, wakelock);  in __intel_runtime_pm_get()
372 	return track_intel_runtime_pm_wakeref(rpm);  in __intel_runtime_pm_get()
392 intel_wakeref_t intel_runtime_pm_get_raw(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_raw()  argument
394 	return __intel_runtime_pm_get(rpm, false);  in intel_runtime_pm_get_raw()
409 intel_wakeref_t intel_runtime_pm_get(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get()  argument
411 	return __intel_runtime_pm_get(rpm, true);  in intel_runtime_pm_get()
436 static intel_wakeref_t __intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_get_if_active()  argument
446 		if (pm_runtime_get_if_active(rpm->kdev, ignore_usecount) <= 0)  in __intel_runtime_pm_get_if_active()
450 	intel_runtime_pm_acquire(rpm, true);  in __intel_runtime_pm_get_if_active()
452 	return track_intel_runtime_pm_wakeref(rpm);  in __intel_runtime_pm_get_if_active()
455 intel_wakeref_t intel_runtime_pm_get_if_in_use(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_if_in_use()  argument
457 	return __intel_runtime_pm_get_if_active(rpm, false);  in intel_runtime_pm_get_if_in_use()
460 intel_wakeref_t intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_if_active()  argument
462 	return __intel_runtime_pm_get_if_active(rpm, true);  in intel_runtime_pm_get_if_active()
484 intel_wakeref_t intel_runtime_pm_get_noresume(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_noresume()  argument
486 	assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_get_noresume()
487 	pm_runtime_get_noresume(rpm->kdev);  in intel_runtime_pm_get_noresume()
489 	intel_runtime_pm_acquire(rpm, true);  in intel_runtime_pm_get_noresume()
491 	return track_intel_runtime_pm_wakeref(rpm);  in intel_runtime_pm_get_noresume()
494 static void __intel_runtime_pm_put(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_put()  argument
498 	struct device *kdev = rpm->kdev;  in __intel_runtime_pm_put()
500 	untrack_intel_runtime_pm_wakeref(rpm, wref);  in __intel_runtime_pm_put()
502 	intel_runtime_pm_release(rpm, wakelock);  in __intel_runtime_pm_put()
518 intel_runtime_pm_put_raw(struct intel_runtime_pm *rpm, intel_wakeref_t wref)  in intel_runtime_pm_put_raw()  argument
520 	__intel_runtime_pm_put(rpm, wref, false);  in intel_runtime_pm_put_raw()
535 void intel_runtime_pm_put_unchecked(struct intel_runtime_pm *rpm)  in intel_runtime_pm_put_unchecked()  argument
537 	__intel_runtime_pm_put(rpm, -1, true);  in intel_runtime_pm_put_unchecked()
550 void intel_runtime_pm_put(struct intel_runtime_pm *rpm, intel_wakeref_t wref)  in intel_runtime_pm_put()  argument
552 	__intel_runtime_pm_put(rpm, wref, true);  in intel_runtime_pm_put()
566 void intel_runtime_pm_enable(struct intel_runtime_pm *rpm)  in intel_runtime_pm_enable()  argument
568 	struct drm_i915_private *i915 = container_of(rpm,  in intel_runtime_pm_enable()
571 	struct device *kdev = rpm->kdev;  in intel_runtime_pm_enable()
592 	if (!rpm->available) {  in intel_runtime_pm_enable()
611 void intel_runtime_pm_disable(struct intel_runtime_pm *rpm)  in intel_runtime_pm_disable()  argument
613 	struct drm_i915_private *i915 = container_of(rpm,  in intel_runtime_pm_disable()
616 	struct device *kdev = rpm->kdev;  in intel_runtime_pm_disable()
624 	if (!rpm->available)  in intel_runtime_pm_disable()
628 void intel_runtime_pm_driver_release(struct intel_runtime_pm *rpm)  in intel_runtime_pm_driver_release()  argument
630 	struct drm_i915_private *i915 = container_of(rpm,  in intel_runtime_pm_driver_release()
633 	int count = atomic_read(&rpm->wakeref_count);  in intel_runtime_pm_driver_release()
640 	untrack_all_intel_runtime_pm_wakerefs(rpm);  in intel_runtime_pm_driver_release()
643 void intel_runtime_pm_init_early(struct intel_runtime_pm *rpm)  in intel_runtime_pm_init_early()  argument
646 			container_of(rpm, struct drm_i915_private, runtime_pm);  in intel_runtime_pm_init_early()
650 	rpm->kdev = kdev;  in intel_runtime_pm_init_early()
651 	rpm->available = HAS_RUNTIME_PM(i915);  in intel_runtime_pm_init_early()
653 	init_intel_runtime_pm_wakeref(rpm);  in intel_runtime_pm_init_early()