79 static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in init_intel_runtime_pm_wakeref()  argument
81 	spin_lock_init(&rpm->debug.lock);  in init_intel_runtime_pm_wakeref()
85 track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in track_intel_runtime_pm_wakeref()  argument
90 	if (!rpm->available)  in track_intel_runtime_pm_wakeref()
97 	spin_lock_irqsave(&rpm->debug.lock, flags);  in track_intel_runtime_pm_wakeref()
99 	if (!rpm->debug.count)  in track_intel_runtime_pm_wakeref()
100 		rpm->debug.last_acquire = stack;  in track_intel_runtime_pm_wakeref()
102 	stacks = krealloc(rpm->debug.owners,  in track_intel_runtime_pm_wakeref()
103 			  (rpm->debug.count + 1) * sizeof(*stacks),  in track_intel_runtime_pm_wakeref()
106 		stacks[rpm->debug.count++] = stack;  in track_intel_runtime_pm_wakeref()
107 		rpm->debug.owners = stacks;  in track_intel_runtime_pm_wakeref()
112 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in track_intel_runtime_pm_wakeref()
117 static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in untrack_intel_runtime_pm_wakeref()  argument
126 	spin_lock_irqsave(&rpm->debug.lock, flags);  in untrack_intel_runtime_pm_wakeref()
127 	for (n = rpm->debug.count; n--; ) {  in untrack_intel_runtime_pm_wakeref()
128 		if (rpm->debug.owners[n] == stack) {  in untrack_intel_runtime_pm_wakeref()
129 			memmove(rpm->debug.owners + n,  in untrack_intel_runtime_pm_wakeref()
130 				rpm->debug.owners + n + 1,  in untrack_intel_runtime_pm_wakeref()
131 				(--rpm->debug.count - n) * sizeof(stack));  in untrack_intel_runtime_pm_wakeref()
136 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in untrack_intel_runtime_pm_wakeref()
140 		 rpm->debug.count, atomic_read(&rpm->wakeref_count))) {  in untrack_intel_runtime_pm_wakeref()
150 		stack = READ_ONCE(rpm->debug.last_release);  in untrack_intel_runtime_pm_wakeref()
235 __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)  in __intel_wakeref_dec_and_check_tracking()  argument
240 	if (!atomic_dec_and_lock_irqsave(&rpm->wakeref_count,  in __intel_wakeref_dec_and_check_tracking()
241 					 &rpm->debug.lock,  in __intel_wakeref_dec_and_check_tracking()
245 	__untrack_all_wakerefs(&rpm->debug, &dbg);  in __intel_wakeref_dec_and_check_tracking()
246 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in __intel_wakeref_dec_and_check_tracking()
252 untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)  in untrack_all_intel_runtime_pm_wakerefs()  argument
257 	spin_lock_irqsave(&rpm->debug.lock, flags);  in untrack_all_intel_runtime_pm_wakerefs()
258 	__untrack_all_wakerefs(&rpm->debug, &dbg);  in untrack_all_intel_runtime_pm_wakerefs()
259 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in untrack_all_intel_runtime_pm_wakerefs()
264 void print_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in print_intel_runtime_pm_wakeref()  argument
273 		spin_lock_irq(&rpm->debug.lock);  in print_intel_runtime_pm_wakeref()
274 		dbg.count = rpm->debug.count;  in print_intel_runtime_pm_wakeref()
277 			       rpm->debug.owners,  in print_intel_runtime_pm_wakeref()
280 		dbg.last_acquire = rpm->debug.last_acquire;  in print_intel_runtime_pm_wakeref()
281 		dbg.last_release = rpm->debug.last_release;  in print_intel_runtime_pm_wakeref()
282 		spin_unlock_irq(&rpm->debug.lock);  in print_intel_runtime_pm_wakeref()
303 static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in init_intel_runtime_pm_wakeref()  argument
308 track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in track_intel_runtime_pm_wakeref()  argument
313 static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in untrack_intel_runtime_pm_wakeref()  argument
319 __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)  in __intel_wakeref_dec_and_check_tracking()  argument
321 	atomic_dec(&rpm->wakeref_count);  in __intel_wakeref_dec_and_check_tracking()
325 untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)  in untrack_all_intel_runtime_pm_wakerefs()  argument
332 intel_runtime_pm_acquire(struct intel_runtime_pm *rpm, bool wakelock)  in intel_runtime_pm_acquire()  argument
335 		atomic_add(1 + INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count);  in intel_runtime_pm_acquire()
336 		assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_acquire()
338 		atomic_inc(&rpm->wakeref_count);  in intel_runtime_pm_acquire()
339 		assert_rpm_raw_wakeref_held(rpm);  in intel_runtime_pm_acquire()
344 intel_runtime_pm_release(struct intel_runtime_pm *rpm, int wakelock)  in intel_runtime_pm_release()  argument
347 		assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_release()
348 		atomic_sub(INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count);  in intel_runtime_pm_release()
350 		assert_rpm_raw_wakeref_held(rpm);  in intel_runtime_pm_release()
353 	__intel_wakeref_dec_and_check_tracking(rpm);  in intel_runtime_pm_release()
356 static intel_wakeref_t __intel_runtime_pm_get(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_get()  argument
361 	ret = pm_runtime_get_sync(rpm->kdev);  in __intel_runtime_pm_get()
364 	intel_runtime_pm_acquire(rpm, wakelock);  in __intel_runtime_pm_get()
366 	return track_intel_runtime_pm_wakeref(rpm);  in __intel_runtime_pm_get()
386 intel_wakeref_t intel_runtime_pm_get_raw(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_raw()  argument
388 	return __intel_runtime_pm_get(rpm, false);  in intel_runtime_pm_get_raw()
403 intel_wakeref_t intel_runtime_pm_get(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get()  argument
405 	return __intel_runtime_pm_get(rpm, true);  in intel_runtime_pm_get()
422 intel_wakeref_t intel_runtime_pm_get_if_in_use(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_if_in_use()  argument
431 		if (pm_runtime_get_if_in_use(rpm->kdev) <= 0)  in intel_runtime_pm_get_if_in_use()
435 	intel_runtime_pm_acquire(rpm, true);  in intel_runtime_pm_get_if_in_use()
437 	return track_intel_runtime_pm_wakeref(rpm);  in intel_runtime_pm_get_if_in_use()
459 intel_wakeref_t intel_runtime_pm_get_noresume(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_noresume()  argument
461 	assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_get_noresume()
462 	pm_runtime_get_noresume(rpm->kdev);  in intel_runtime_pm_get_noresume()
464 	intel_runtime_pm_acquire(rpm, true);  in intel_runtime_pm_get_noresume()
466 	return track_intel_runtime_pm_wakeref(rpm);  in intel_runtime_pm_get_noresume()
469 static void __intel_runtime_pm_put(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_put()  argument
473 	struct device *kdev = rpm->kdev;  in __intel_runtime_pm_put()
475 	untrack_intel_runtime_pm_wakeref(rpm, wref);  in __intel_runtime_pm_put()
477 	intel_runtime_pm_release(rpm, wakelock);  in __intel_runtime_pm_put()
493 intel_runtime_pm_put_raw(struct intel_runtime_pm *rpm, intel_wakeref_t wref)  in intel_runtime_pm_put_raw()  argument
495 	__intel_runtime_pm_put(rpm, wref, false);  in intel_runtime_pm_put_raw()
510 void intel_runtime_pm_put_unchecked(struct intel_runtime_pm *rpm)  in intel_runtime_pm_put_unchecked()  argument
512 	__intel_runtime_pm_put(rpm, -1, true);  in intel_runtime_pm_put_unchecked()
525 void intel_runtime_pm_put(struct intel_runtime_pm *rpm, intel_wakeref_t wref)  in intel_runtime_pm_put()  argument
527 	__intel_runtime_pm_put(rpm, wref, true);  in intel_runtime_pm_put()
541 void intel_runtime_pm_enable(struct intel_runtime_pm *rpm)  in intel_runtime_pm_enable()  argument
543 	struct device *kdev = rpm->kdev;  in intel_runtime_pm_enable()
564 	if (!rpm->available) {  in intel_runtime_pm_enable()
582 void intel_runtime_pm_disable(struct intel_runtime_pm *rpm)  in intel_runtime_pm_disable()  argument
584 	struct device *kdev = rpm->kdev;  in intel_runtime_pm_disable()
592 	if (!rpm->available)  in intel_runtime_pm_disable()
596 void intel_runtime_pm_driver_release(struct intel_runtime_pm *rpm)  in intel_runtime_pm_driver_release()  argument
598 	int count = atomic_read(&rpm->wakeref_count);  in intel_runtime_pm_driver_release()
605 	untrack_all_intel_runtime_pm_wakerefs(rpm);  in intel_runtime_pm_driver_release()
608 void intel_runtime_pm_init_early(struct intel_runtime_pm *rpm)  in intel_runtime_pm_init_early()  argument
611 			container_of(rpm, struct drm_i915_private, runtime_pm);  in intel_runtime_pm_init_early()
615 	rpm->kdev = kdev;  in intel_runtime_pm_init_early()
616 	rpm->available = HAS_RUNTIME_PM(i915);  in intel_runtime_pm_init_early()
618 	init_intel_runtime_pm_wakeref(rpm);  in intel_runtime_pm_init_early()