1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2019 Intel Corporation
5 */
6
7 #include <linux/wait_bit.h>
8
9 #include "intel_runtime_pm.h"
10 #include "intel_wakeref.h"
11
rpm_get(struct intel_wakeref * wf)12 static void rpm_get(struct intel_wakeref *wf)
13 {
14 wf->wakeref = intel_runtime_pm_get(wf->rpm);
15 }
16
rpm_put(struct intel_wakeref * wf)17 static void rpm_put(struct intel_wakeref *wf)
18 {
19 intel_wakeref_t wakeref = fetch_and_zero(&wf->wakeref);
20
21 intel_runtime_pm_put(wf->rpm, wakeref);
22 INTEL_WAKEREF_BUG_ON(!wakeref);
23 }
24
__intel_wakeref_get_first(struct intel_wakeref * wf)25 int __intel_wakeref_get_first(struct intel_wakeref *wf)
26 {
27 /*
28 * Treat get/put as different subclasses, as we may need to run
29 * the put callback from under the shrinker and do not want to
30 * cross-contanimate that callback with any extra work performed
31 * upon acquiring the wakeref.
32 */
33 mutex_lock_nested(&wf->mutex, SINGLE_DEPTH_NESTING);
34 if (!atomic_read(&wf->count)) {
35 int err;
36
37 rpm_get(wf);
38
39 err = wf->ops->get(wf);
40 if (unlikely(err)) {
41 rpm_put(wf);
42 mutex_unlock(&wf->mutex);
43 return err;
44 }
45
46 smp_mb__before_atomic(); /* release wf->count */
47 }
48 atomic_inc(&wf->count);
49 mutex_unlock(&wf->mutex);
50
51 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
52 return 0;
53 }
54
____intel_wakeref_put_last(struct intel_wakeref * wf)55 static void ____intel_wakeref_put_last(struct intel_wakeref *wf)
56 {
57 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
58 if (unlikely(!atomic_dec_and_test(&wf->count)))
59 goto unlock;
60
61 /* ops->put() must reschedule its own release on error/deferral */
62 if (likely(!wf->ops->put(wf))) {
63 rpm_put(wf);
64 wake_up_var(&wf->wakeref);
65 }
66
67 unlock:
68 mutex_unlock(&wf->mutex);
69 }
70
__intel_wakeref_put_last(struct intel_wakeref * wf,unsigned long flags)71 void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags)
72 {
73 INTEL_WAKEREF_BUG_ON(delayed_work_pending(&wf->work));
74
75 /* Assume we are not in process context and so cannot sleep. */
76 if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) {
77 mod_delayed_work(system_wq, &wf->work,
78 FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags));
79 return;
80 }
81
82 ____intel_wakeref_put_last(wf);
83 }
84
__intel_wakeref_put_work(struct work_struct * wrk)85 static void __intel_wakeref_put_work(struct work_struct *wrk)
86 {
87 struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work.work);
88
89 if (atomic_add_unless(&wf->count, -1, 1))
90 return;
91
92 mutex_lock(&wf->mutex);
93 ____intel_wakeref_put_last(wf);
94 }
95
__intel_wakeref_init(struct intel_wakeref * wf,struct intel_runtime_pm * rpm,const struct intel_wakeref_ops * ops,struct intel_wakeref_lockclass * key)96 void __intel_wakeref_init(struct intel_wakeref *wf,
97 struct intel_runtime_pm *rpm,
98 const struct intel_wakeref_ops *ops,
99 struct intel_wakeref_lockclass *key)
100 {
101 wf->rpm = rpm;
102 wf->ops = ops;
103
104 __mutex_init(&wf->mutex, "wakeref.mutex", &key->mutex);
105 atomic_set(&wf->count, 0);
106 wf->wakeref = 0;
107
108 INIT_DELAYED_WORK(&wf->work, __intel_wakeref_put_work);
109 lockdep_init_map(&wf->work.work.lockdep_map,
110 "wakeref.work", &key->work, 0);
111 }
112
intel_wakeref_wait_for_idle(struct intel_wakeref * wf)113 int intel_wakeref_wait_for_idle(struct intel_wakeref *wf)
114 {
115 int err;
116
117 might_sleep();
118
119 err = wait_var_event_killable(&wf->wakeref,
120 !intel_wakeref_is_active(wf));
121 if (err)
122 return err;
123
124 intel_wakeref_unlock_wait(wf);
125 return 0;
126 }
127
wakeref_auto_timeout(struct timer_list * t)128 static void wakeref_auto_timeout(struct timer_list *t)
129 {
130 struct intel_wakeref_auto *wf = from_timer(wf, t, timer);
131 intel_wakeref_t wakeref;
132 unsigned long flags;
133
134 if (!refcount_dec_and_lock_irqsave(&wf->count, &wf->lock, &flags))
135 return;
136
137 wakeref = fetch_and_zero(&wf->wakeref);
138 spin_unlock_irqrestore(&wf->lock, flags);
139
140 intel_runtime_pm_put(wf->rpm, wakeref);
141 }
142
intel_wakeref_auto_init(struct intel_wakeref_auto * wf,struct intel_runtime_pm * rpm)143 void intel_wakeref_auto_init(struct intel_wakeref_auto *wf,
144 struct intel_runtime_pm *rpm)
145 {
146 spin_lock_init(&wf->lock);
147 timer_setup(&wf->timer, wakeref_auto_timeout, 0);
148 refcount_set(&wf->count, 0);
149 wf->wakeref = 0;
150 wf->rpm = rpm;
151 }
152
intel_wakeref_auto(struct intel_wakeref_auto * wf,unsigned long timeout)153 void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout)
154 {
155 unsigned long flags;
156
157 if (!timeout) {
158 if (del_timer_sync(&wf->timer))
159 wakeref_auto_timeout(&wf->timer);
160 return;
161 }
162
163 /* Our mission is that we only extend an already active wakeref */
164 assert_rpm_wakelock_held(wf->rpm);
165
166 if (!refcount_inc_not_zero(&wf->count)) {
167 spin_lock_irqsave(&wf->lock, flags);
168 if (!refcount_inc_not_zero(&wf->count)) {
169 INTEL_WAKEREF_BUG_ON(wf->wakeref);
170 wf->wakeref = intel_runtime_pm_get_if_in_use(wf->rpm);
171 refcount_set(&wf->count, 1);
172 }
173 spin_unlock_irqrestore(&wf->lock, flags);
174 }
175
176 /*
177 * If we extend a pending timer, we will only get a single timer
178 * callback and so need to cancel the local inc by running the
179 * elided callback to keep the wf->count balanced.
180 */
181 if (mod_timer(&wf->timer, jiffies + timeout))
182 wakeref_auto_timeout(&wf->timer);
183 }
184
intel_wakeref_auto_fini(struct intel_wakeref_auto * wf)185 void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf)
186 {
187 intel_wakeref_auto(wf, 0);
188 INTEL_WAKEREF_BUG_ON(wf->wakeref);
189 }
190