1 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
2 /*
3 *
4 * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved.
5 *
6 * This program is free software and is provided to you under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation, and any use by you of this program is subject to the terms
9 * of such GNU license.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
19 *
20 */
21
22 /*
23 * Implementation of the GPU clock rate trace manager.
24 */
25
26 #include <mali_kbase.h>
27 #include <mali_kbase_config_defaults.h>
28 #include <linux/clk.h>
29 #include <linux/pm_opp.h>
30 #include <asm/div64.h>
31 #include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h"
32
33 #ifdef CONFIG_TRACE_POWER_GPU_FREQUENCY
34 #include <trace/events/power_gpu_frequency.h>
35 #else
36 #include "mali_power_gpu_frequency_trace.h"
37 #endif
38
39 #ifndef CLK_RATE_TRACE_OPS
40 #define CLK_RATE_TRACE_OPS (NULL)
41 #endif
42
43 /**
44 * get_clk_rate_trace_callbacks() - Returns pointer to clk trace ops.
45 * @kbdev: Pointer to kbase device, used to check if arbitration is enabled
46 * when compiled with arbiter support.
47 * Return: Pointer to clk trace ops if supported or NULL.
48 */
49 static struct kbase_clk_rate_trace_op_conf *
get_clk_rate_trace_callbacks(__maybe_unused struct kbase_device * kbdev)50 get_clk_rate_trace_callbacks(__maybe_unused struct kbase_device *kbdev)
51 {
52 /* base case */
53 struct kbase_clk_rate_trace_op_conf *callbacks =
54 (struct kbase_clk_rate_trace_op_conf *)CLK_RATE_TRACE_OPS;
55 #if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF)
56 const void *arbiter_if_node;
57
58 if (WARN_ON(!kbdev) || WARN_ON(!kbdev->dev))
59 return callbacks;
60
61 arbiter_if_node =
62 of_get_property(kbdev->dev->of_node, "arbiter_if", NULL);
63 /* Arbitration enabled, override the callback pointer.*/
64 if (arbiter_if_node)
65 callbacks = &arb_clk_rate_trace_ops;
66 else
67 dev_dbg(kbdev->dev,
68 "Arbitration supported but disabled by platform. Leaving clk rate callbacks as default.\n");
69
70 #endif
71
72 return callbacks;
73 }
74
kbase_lowest_gpu_freq_init(struct kbase_device * kbdev)75 int kbase_lowest_gpu_freq_init(struct kbase_device *kbdev)
76 {
77 /* Uses default reference frequency defined in below macro */
78 u64 lowest_freq_khz = DEFAULT_REF_TIMEOUT_FREQ_KHZ;
79
80 /* Only check lowest frequency in cases when OPPs are used and
81 * present in the device tree.
82 */
83 #ifdef CONFIG_PM_OPP
84 struct dev_pm_opp *opp_ptr;
85 unsigned long found_freq = 0;
86
87 /* find lowest frequency OPP */
88 opp_ptr = dev_pm_opp_find_freq_ceil(kbdev->dev, &found_freq);
89 if (IS_ERR(opp_ptr)) {
90 dev_err(kbdev->dev,
91 "No OPPs found in device tree! Scaling timeouts using %llu kHz",
92 (unsigned long long)lowest_freq_khz);
93 } else {
94 #if KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE
95 dev_pm_opp_put(opp_ptr); /* decrease OPP refcount */
96 #endif
97 /* convert found frequency to KHz */
98 found_freq /= 1000;
99
100 /* If lowest frequency in OPP table is still higher
101 * than the reference, then keep the reference frequency
102 * as the one to use for scaling .
103 */
104 if (found_freq < lowest_freq_khz)
105 lowest_freq_khz = found_freq;
106 }
107 #else
108 dev_err(kbdev->dev,
109 "No operating-points-v2 node or operating-points property in DT");
110 #endif
111
112 kbdev->lowest_gpu_freq_khz = lowest_freq_khz;
113 dev_dbg(kbdev->dev, "Lowest frequency identified is %llu kHz",
114 kbdev->lowest_gpu_freq_khz);
115 return 0;
116 }
117
gpu_clk_rate_change_notifier(struct notifier_block * nb,unsigned long event,void * data)118 static int gpu_clk_rate_change_notifier(struct notifier_block *nb,
119 unsigned long event, void *data)
120 {
121 struct kbase_gpu_clk_notifier_data *ndata = data;
122 struct kbase_clk_data *clk_data =
123 container_of(nb, struct kbase_clk_data, clk_rate_change_nb);
124 struct kbase_clk_rate_trace_manager *clk_rtm = clk_data->clk_rtm;
125 unsigned long flags;
126
127 if (WARN_ON_ONCE(clk_data->gpu_clk_handle != ndata->gpu_clk_handle))
128 return NOTIFY_BAD;
129
130 spin_lock_irqsave(&clk_rtm->lock, flags);
131 if (event == POST_RATE_CHANGE) {
132 if (!clk_rtm->gpu_idle &&
133 (clk_data->clock_val != ndata->new_rate)) {
134 kbase_clk_rate_trace_manager_notify_all(
135 clk_rtm, clk_data->index, ndata->new_rate);
136 }
137
138 clk_data->clock_val = ndata->new_rate;
139 }
140 spin_unlock_irqrestore(&clk_rtm->lock, flags);
141
142 return NOTIFY_DONE;
143 }
144
gpu_clk_data_init(struct kbase_device * kbdev,void * gpu_clk_handle,unsigned int index)145 static int gpu_clk_data_init(struct kbase_device *kbdev,
146 void *gpu_clk_handle, unsigned int index)
147 {
148 struct kbase_clk_rate_trace_op_conf *callbacks;
149 struct kbase_clk_data *clk_data;
150 struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
151 int ret = 0;
152
153 callbacks = get_clk_rate_trace_callbacks(kbdev);
154
155 if (WARN_ON(!callbacks) ||
156 WARN_ON(!gpu_clk_handle) ||
157 WARN_ON(index >= BASE_MAX_NR_CLOCKS_REGULATORS))
158 return -EINVAL;
159
160 clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL);
161 if (!clk_data) {
162 dev_err(kbdev->dev, "Failed to allocate data for clock enumerated at index %u", index);
163 return -ENOMEM;
164 }
165
166 clk_data->index = (u8)index;
167 clk_data->gpu_clk_handle = gpu_clk_handle;
168 /* Store the initial value of clock */
169 clk_data->clock_val =
170 callbacks->get_gpu_clk_rate(kbdev, gpu_clk_handle);
171
172 {
173 /* At the initialization time, GPU is powered off. */
174 unsigned long flags;
175
176 spin_lock_irqsave(&clk_rtm->lock, flags);
177 kbase_clk_rate_trace_manager_notify_all(
178 clk_rtm, clk_data->index, 0);
179 spin_unlock_irqrestore(&clk_rtm->lock, flags);
180 }
181
182 clk_data->clk_rtm = clk_rtm;
183 clk_rtm->clks[index] = clk_data;
184
185 clk_data->clk_rate_change_nb.notifier_call =
186 gpu_clk_rate_change_notifier;
187
188 if (callbacks->gpu_clk_notifier_register)
189 ret = callbacks->gpu_clk_notifier_register(kbdev,
190 gpu_clk_handle, &clk_data->clk_rate_change_nb);
191 if (ret) {
192 dev_err(kbdev->dev, "Failed to register notifier for clock enumerated at index %u", index);
193 kfree(clk_data);
194 }
195
196 return ret;
197 }
198
kbase_clk_rate_trace_manager_init(struct kbase_device * kbdev)199 int kbase_clk_rate_trace_manager_init(struct kbase_device *kbdev)
200 {
201 struct kbase_clk_rate_trace_op_conf *callbacks;
202 struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
203 unsigned int i;
204 int ret = 0;
205
206 callbacks = get_clk_rate_trace_callbacks(kbdev);
207
208 spin_lock_init(&clk_rtm->lock);
209 INIT_LIST_HEAD(&clk_rtm->listeners);
210
211 /* Return early if no callbacks provided for clock rate tracing */
212 if (!callbacks) {
213 WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
214 return 0;
215 }
216
217 clk_rtm->gpu_idle = true;
218
219 for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
220 void *gpu_clk_handle =
221 callbacks->enumerate_gpu_clk(kbdev, i);
222
223 if (!gpu_clk_handle)
224 break;
225
226 ret = gpu_clk_data_init(kbdev, gpu_clk_handle, i);
227 if (ret)
228 goto error;
229 }
230
231 /* Activate clock rate trace manager if at least one GPU clock was
232 * enumerated.
233 */
234 if (i) {
235 WRITE_ONCE(clk_rtm->clk_rate_trace_ops, callbacks);
236 } else {
237 dev_info(kbdev->dev, "No clock(s) available for rate tracing");
238 WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
239 }
240
241 return 0;
242
243 error:
244 while (i--) {
245 clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister(
246 kbdev, clk_rtm->clks[i]->gpu_clk_handle,
247 &clk_rtm->clks[i]->clk_rate_change_nb);
248 kfree(clk_rtm->clks[i]);
249 }
250
251 return ret;
252 }
253
kbase_clk_rate_trace_manager_term(struct kbase_device * kbdev)254 void kbase_clk_rate_trace_manager_term(struct kbase_device *kbdev)
255 {
256 struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
257 unsigned int i;
258
259 WARN_ON(!list_empty(&clk_rtm->listeners));
260
261 if (!clk_rtm->clk_rate_trace_ops)
262 return;
263
264 for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
265 if (!clk_rtm->clks[i])
266 break;
267
268 if (clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister)
269 clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister
270 (kbdev, clk_rtm->clks[i]->gpu_clk_handle,
271 &clk_rtm->clks[i]->clk_rate_change_nb);
272 kfree(clk_rtm->clks[i]);
273 }
274
275 WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
276 }
277
kbase_clk_rate_trace_manager_gpu_active(struct kbase_device * kbdev)278 void kbase_clk_rate_trace_manager_gpu_active(struct kbase_device *kbdev)
279 {
280 struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
281 unsigned int i;
282 unsigned long flags;
283
284 if (!clk_rtm->clk_rate_trace_ops)
285 return;
286
287 spin_lock_irqsave(&clk_rtm->lock, flags);
288
289 for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
290 struct kbase_clk_data *clk_data = clk_rtm->clks[i];
291
292 if (!clk_data)
293 break;
294
295 if (unlikely(!clk_data->clock_val))
296 continue;
297
298 kbase_clk_rate_trace_manager_notify_all(
299 clk_rtm, clk_data->index, clk_data->clock_val);
300 }
301
302 clk_rtm->gpu_idle = false;
303 spin_unlock_irqrestore(&clk_rtm->lock, flags);
304 }
305
kbase_clk_rate_trace_manager_gpu_idle(struct kbase_device * kbdev)306 void kbase_clk_rate_trace_manager_gpu_idle(struct kbase_device *kbdev)
307 {
308 struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
309 unsigned int i;
310 unsigned long flags;
311
312 if (!clk_rtm->clk_rate_trace_ops)
313 return;
314
315 spin_lock_irqsave(&clk_rtm->lock, flags);
316
317 for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
318 struct kbase_clk_data *clk_data = clk_rtm->clks[i];
319
320 if (!clk_data)
321 break;
322
323 if (unlikely(!clk_data->clock_val))
324 continue;
325
326 kbase_clk_rate_trace_manager_notify_all(
327 clk_rtm, clk_data->index, 0);
328 }
329
330 clk_rtm->gpu_idle = true;
331 spin_unlock_irqrestore(&clk_rtm->lock, flags);
332 }
333
kbase_clk_rate_trace_manager_notify_all(struct kbase_clk_rate_trace_manager * clk_rtm,u32 clk_index,unsigned long new_rate)334 void kbase_clk_rate_trace_manager_notify_all(
335 struct kbase_clk_rate_trace_manager *clk_rtm,
336 u32 clk_index,
337 unsigned long new_rate)
338 {
339 struct kbase_clk_rate_listener *pos;
340 struct kbase_device *kbdev;
341
342 lockdep_assert_held(&clk_rtm->lock);
343
344 kbdev = container_of(clk_rtm, struct kbase_device, pm.clk_rtm);
345
346 dev_dbg(kbdev->dev, "%s - GPU clock %u rate changed to %lu, pid: %d",
347 __func__, clk_index, new_rate, current->pid);
348
349 /* Raise standard `power/gpu_frequency` ftrace event */
350 {
351 unsigned long new_rate_khz = new_rate;
352
353 #if BITS_PER_LONG == 64
354 do_div(new_rate_khz, 1000);
355 #elif BITS_PER_LONG == 32
356 new_rate_khz /= 1000;
357 #else
358 #error "unsigned long division is not supported for this architecture"
359 #endif
360
361 trace_gpu_frequency(new_rate_khz, clk_index);
362 }
363
364 /* Notify the listeners. */
365 list_for_each_entry(pos, &clk_rtm->listeners, node) {
366 pos->notify(pos, clk_index, new_rate);
367 }
368 }
369 KBASE_EXPORT_TEST_API(kbase_clk_rate_trace_manager_notify_all);
370