1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
3 */
4
5 #define pr_fmt(fmt) "[drm:%s:%d] " fmt, __func__, __LINE__
6
7 #include <linux/debugfs.h>
8 #include <linux/errno.h>
9 #include <linux/mutex.h>
10 #include <linux/pm_opp.h>
11 #include <linux/sort.h>
12 #include <linux/clk.h>
13 #include <linux/bitmap.h>
14
15 #include "dpu_kms.h"
16 #include "dpu_trace.h"
17 #include "dpu_crtc.h"
18 #include "dpu_core_perf.h"
19
20 /**
21 * enum dpu_perf_mode - performance tuning mode
22 * @DPU_PERF_MODE_NORMAL: performance controlled by user mode client
23 * @DPU_PERF_MODE_MINIMUM: performance bounded by minimum setting
24 * @DPU_PERF_MODE_FIXED: performance bounded by fixed setting
25 */
26 enum dpu_perf_mode {
27 DPU_PERF_MODE_NORMAL,
28 DPU_PERF_MODE_MINIMUM,
29 DPU_PERF_MODE_FIXED,
30 DPU_PERF_MODE_MAX
31 };
32
33 /**
34 * @_dpu_core_perf_calc_bw() - to calculate BW per crtc
35 * @kms - pointer to the dpu_kms
36 * @crtc - pointer to a crtc
37 * Return: returns aggregated BW for all planes in crtc.
38 */
_dpu_core_perf_calc_bw(struct dpu_kms * kms,struct drm_crtc * crtc)39 static u64 _dpu_core_perf_calc_bw(struct dpu_kms *kms,
40 struct drm_crtc *crtc)
41 {
42 struct drm_plane *plane;
43 struct dpu_plane_state *pstate;
44 u64 crtc_plane_bw = 0;
45 u32 bw_factor;
46
47 drm_atomic_crtc_for_each_plane(plane, crtc) {
48 pstate = to_dpu_plane_state(plane->state);
49 if (!pstate)
50 continue;
51
52 crtc_plane_bw += pstate->plane_fetch_bw;
53 }
54
55 bw_factor = kms->catalog->perf.bw_inefficiency_factor;
56 if (bw_factor) {
57 crtc_plane_bw *= bw_factor;
58 do_div(crtc_plane_bw, 100);
59 }
60
61 return crtc_plane_bw;
62 }
63
64 /**
65 * _dpu_core_perf_calc_clk() - to calculate clock per crtc
66 * @kms - pointer to the dpu_kms
67 * @crtc - pointer to a crtc
68 * @state - pointer to a crtc state
69 * Return: returns max clk for all planes in crtc.
70 */
_dpu_core_perf_calc_clk(struct dpu_kms * kms,struct drm_crtc * crtc,struct drm_crtc_state * state)71 static u64 _dpu_core_perf_calc_clk(struct dpu_kms *kms,
72 struct drm_crtc *crtc, struct drm_crtc_state *state)
73 {
74 struct drm_plane *plane;
75 struct dpu_plane_state *pstate;
76 struct drm_display_mode *mode;
77 u64 crtc_clk;
78 u32 clk_factor;
79
80 mode = &state->adjusted_mode;
81
82 crtc_clk = mode->vtotal * mode->hdisplay * drm_mode_vrefresh(mode);
83
84 drm_atomic_crtc_for_each_plane(plane, crtc) {
85 pstate = to_dpu_plane_state(plane->state);
86 if (!pstate)
87 continue;
88
89 crtc_clk = max(pstate->plane_clk, crtc_clk);
90 }
91
92 clk_factor = kms->catalog->perf.clk_inefficiency_factor;
93 if (clk_factor) {
94 crtc_clk *= clk_factor;
95 do_div(crtc_clk, 100);
96 }
97
98 return crtc_clk;
99 }
100
_dpu_crtc_get_kms(struct drm_crtc * crtc)101 static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
102 {
103 struct msm_drm_private *priv;
104 priv = crtc->dev->dev_private;
105 return to_dpu_kms(priv->kms);
106 }
107
_dpu_core_perf_calc_crtc(struct dpu_kms * kms,struct drm_crtc * crtc,struct drm_crtc_state * state,struct dpu_core_perf_params * perf)108 static void _dpu_core_perf_calc_crtc(struct dpu_kms *kms,
109 struct drm_crtc *crtc,
110 struct drm_crtc_state *state,
111 struct dpu_core_perf_params *perf)
112 {
113 struct dpu_crtc_state *dpu_cstate;
114
115 if (!kms || !kms->catalog || !crtc || !state || !perf) {
116 DPU_ERROR("invalid parameters\n");
117 return;
118 }
119
120 dpu_cstate = to_dpu_crtc_state(state);
121 memset(perf, 0, sizeof(struct dpu_core_perf_params));
122
123 if (kms->perf.perf_tune.mode == DPU_PERF_MODE_MINIMUM) {
124 perf->bw_ctl = 0;
125 perf->max_per_pipe_ib = 0;
126 perf->core_clk_rate = 0;
127 } else if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED) {
128 perf->bw_ctl = kms->perf.fix_core_ab_vote;
129 perf->max_per_pipe_ib = kms->perf.fix_core_ib_vote;
130 perf->core_clk_rate = kms->perf.fix_core_clk_rate;
131 } else {
132 perf->bw_ctl = _dpu_core_perf_calc_bw(kms, crtc);
133 perf->max_per_pipe_ib = kms->catalog->perf.min_dram_ib;
134 perf->core_clk_rate = _dpu_core_perf_calc_clk(kms, crtc, state);
135 }
136
137 DPU_DEBUG(
138 "crtc=%d clk_rate=%llu core_ib=%llu core_ab=%llu\n",
139 crtc->base.id, perf->core_clk_rate,
140 perf->max_per_pipe_ib, perf->bw_ctl);
141 }
142
dpu_core_perf_crtc_check(struct drm_crtc * crtc,struct drm_crtc_state * state)143 int dpu_core_perf_crtc_check(struct drm_crtc *crtc,
144 struct drm_crtc_state *state)
145 {
146 u32 bw, threshold;
147 u64 bw_sum_of_intfs = 0;
148 enum dpu_crtc_client_type curr_client_type;
149 struct dpu_crtc_state *dpu_cstate;
150 struct drm_crtc *tmp_crtc;
151 struct dpu_kms *kms;
152
153 if (!crtc || !state) {
154 DPU_ERROR("invalid crtc\n");
155 return -EINVAL;
156 }
157
158 kms = _dpu_crtc_get_kms(crtc);
159 if (!kms->catalog) {
160 DPU_ERROR("invalid parameters\n");
161 return 0;
162 }
163
164 /* we only need bandwidth check on real-time clients (interfaces) */
165 if (dpu_crtc_get_client_type(crtc) == NRT_CLIENT)
166 return 0;
167
168 dpu_cstate = to_dpu_crtc_state(state);
169
170 /* obtain new values */
171 _dpu_core_perf_calc_crtc(kms, crtc, state, &dpu_cstate->new_perf);
172
173 bw_sum_of_intfs = dpu_cstate->new_perf.bw_ctl;
174 curr_client_type = dpu_crtc_get_client_type(crtc);
175
176 drm_for_each_crtc(tmp_crtc, crtc->dev) {
177 if (tmp_crtc->enabled &&
178 (dpu_crtc_get_client_type(tmp_crtc) ==
179 curr_client_type) && (tmp_crtc != crtc)) {
180 struct dpu_crtc_state *tmp_cstate =
181 to_dpu_crtc_state(tmp_crtc->state);
182
183 DPU_DEBUG("crtc:%d bw:%llu ctrl:%d\n",
184 tmp_crtc->base.id, tmp_cstate->new_perf.bw_ctl,
185 tmp_cstate->bw_control);
186
187 bw_sum_of_intfs += tmp_cstate->new_perf.bw_ctl;
188 }
189
190 /* convert bandwidth to kb */
191 bw = DIV_ROUND_UP_ULL(bw_sum_of_intfs, 1000);
192 DPU_DEBUG("calculated bandwidth=%uk\n", bw);
193
194 threshold = kms->catalog->perf.max_bw_high;
195
196 DPU_DEBUG("final threshold bw limit = %d\n", threshold);
197
198 if (!threshold) {
199 DPU_ERROR("no bandwidth limits specified\n");
200 return -E2BIG;
201 } else if (bw > threshold) {
202 DPU_ERROR("exceeds bandwidth: %ukb > %ukb\n", bw,
203 threshold);
204 return -E2BIG;
205 }
206 }
207
208 return 0;
209 }
210
_dpu_core_perf_crtc_update_bus(struct dpu_kms * kms,struct drm_crtc * crtc)211 static int _dpu_core_perf_crtc_update_bus(struct dpu_kms *kms,
212 struct drm_crtc *crtc)
213 {
214 struct dpu_core_perf_params perf = { 0 };
215 enum dpu_crtc_client_type curr_client_type
216 = dpu_crtc_get_client_type(crtc);
217 struct drm_crtc *tmp_crtc;
218 struct dpu_crtc_state *dpu_cstate;
219 int i, ret = 0;
220 u64 avg_bw;
221
222 drm_for_each_crtc(tmp_crtc, crtc->dev) {
223 if (tmp_crtc->enabled &&
224 curr_client_type ==
225 dpu_crtc_get_client_type(tmp_crtc)) {
226 dpu_cstate = to_dpu_crtc_state(tmp_crtc->state);
227
228 perf.max_per_pipe_ib = max(perf.max_per_pipe_ib,
229 dpu_cstate->new_perf.max_per_pipe_ib);
230
231 perf.bw_ctl += dpu_cstate->new_perf.bw_ctl;
232
233 DPU_DEBUG("crtc=%d bw=%llu paths:%d\n",
234 tmp_crtc->base.id,
235 dpu_cstate->new_perf.bw_ctl, kms->num_paths);
236 }
237 }
238
239 if (!kms->num_paths)
240 return 0;
241
242 avg_bw = perf.bw_ctl;
243 do_div(avg_bw, (kms->num_paths * 1000)); /*Bps_to_icc*/
244
245 for (i = 0; i < kms->num_paths; i++)
246 icc_set_bw(kms->path[i], avg_bw, perf.max_per_pipe_ib);
247
248 return ret;
249 }
250
251 /**
252 * @dpu_core_perf_crtc_release_bw() - request zero bandwidth
253 * @crtc - pointer to a crtc
254 *
255 * Function checks a state variable for the crtc, if all pending commit
256 * requests are done, meaning no more bandwidth is needed, release
257 * bandwidth request.
258 */
dpu_core_perf_crtc_release_bw(struct drm_crtc * crtc)259 void dpu_core_perf_crtc_release_bw(struct drm_crtc *crtc)
260 {
261 struct dpu_crtc *dpu_crtc;
262 struct dpu_kms *kms;
263
264 if (!crtc) {
265 DPU_ERROR("invalid crtc\n");
266 return;
267 }
268
269 kms = _dpu_crtc_get_kms(crtc);
270 if (!kms->catalog) {
271 DPU_ERROR("invalid kms\n");
272 return;
273 }
274
275 dpu_crtc = to_dpu_crtc(crtc);
276
277 if (atomic_dec_return(&kms->bandwidth_ref) > 0)
278 return;
279
280 /* Release the bandwidth */
281 if (kms->perf.enable_bw_release) {
282 trace_dpu_cmd_release_bw(crtc->base.id);
283 DPU_DEBUG("Release BW crtc=%d\n", crtc->base.id);
284 dpu_crtc->cur_perf.bw_ctl = 0;
285 _dpu_core_perf_crtc_update_bus(kms, crtc);
286 }
287 }
288
_dpu_core_perf_set_core_clk_rate(struct dpu_kms * kms,u64 rate)289 static int _dpu_core_perf_set_core_clk_rate(struct dpu_kms *kms, u64 rate)
290 {
291 struct dss_clk *core_clk = kms->perf.core_clk;
292
293 if (core_clk->max_rate && (rate > core_clk->max_rate))
294 rate = core_clk->max_rate;
295
296 core_clk->rate = rate;
297 return dev_pm_opp_set_rate(&kms->pdev->dev, core_clk->rate);
298 }
299
_dpu_core_perf_get_core_clk_rate(struct dpu_kms * kms)300 static u64 _dpu_core_perf_get_core_clk_rate(struct dpu_kms *kms)
301 {
302 u64 clk_rate = kms->perf.perf_tune.min_core_clk;
303 struct drm_crtc *crtc;
304 struct dpu_crtc_state *dpu_cstate;
305
306 drm_for_each_crtc(crtc, kms->dev) {
307 if (crtc->enabled) {
308 dpu_cstate = to_dpu_crtc_state(crtc->state);
309 clk_rate = max(dpu_cstate->new_perf.core_clk_rate,
310 clk_rate);
311 clk_rate = clk_round_rate(kms->perf.core_clk->clk,
312 clk_rate);
313 }
314 }
315
316 if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED)
317 clk_rate = kms->perf.fix_core_clk_rate;
318
319 DPU_DEBUG("clk:%llu\n", clk_rate);
320
321 return clk_rate;
322 }
323
dpu_core_perf_crtc_update(struct drm_crtc * crtc,int params_changed,bool stop_req)324 int dpu_core_perf_crtc_update(struct drm_crtc *crtc,
325 int params_changed, bool stop_req)
326 {
327 struct dpu_core_perf_params *new, *old;
328 bool update_bus = false, update_clk = false;
329 u64 clk_rate = 0;
330 struct dpu_crtc *dpu_crtc;
331 struct dpu_crtc_state *dpu_cstate;
332 struct dpu_kms *kms;
333 int ret;
334
335 if (!crtc) {
336 DPU_ERROR("invalid crtc\n");
337 return -EINVAL;
338 }
339
340 kms = _dpu_crtc_get_kms(crtc);
341 if (!kms->catalog) {
342 DPU_ERROR("invalid kms\n");
343 return -EINVAL;
344 }
345
346 dpu_crtc = to_dpu_crtc(crtc);
347 dpu_cstate = to_dpu_crtc_state(crtc->state);
348
349 DPU_DEBUG("crtc:%d stop_req:%d core_clk:%llu\n",
350 crtc->base.id, stop_req, kms->perf.core_clk_rate);
351
352 old = &dpu_crtc->cur_perf;
353 new = &dpu_cstate->new_perf;
354
355 if (crtc->enabled && !stop_req) {
356 /*
357 * cases for bus bandwidth update.
358 * 1. new bandwidth vote - "ab or ib vote" is higher
359 * than current vote for update request.
360 * 2. new bandwidth vote - "ab or ib vote" is lower
361 * than current vote at end of commit or stop.
362 */
363 if ((params_changed && ((new->bw_ctl > old->bw_ctl) ||
364 (new->max_per_pipe_ib > old->max_per_pipe_ib))) ||
365 (!params_changed && ((new->bw_ctl < old->bw_ctl) ||
366 (new->max_per_pipe_ib < old->max_per_pipe_ib)))) {
367 DPU_DEBUG("crtc=%d p=%d new_bw=%llu,old_bw=%llu\n",
368 crtc->base.id, params_changed,
369 new->bw_ctl, old->bw_ctl);
370 old->bw_ctl = new->bw_ctl;
371 old->max_per_pipe_ib = new->max_per_pipe_ib;
372 update_bus = true;
373 }
374
375 if ((params_changed &&
376 (new->core_clk_rate > old->core_clk_rate)) ||
377 (!params_changed &&
378 (new->core_clk_rate < old->core_clk_rate))) {
379 old->core_clk_rate = new->core_clk_rate;
380 update_clk = true;
381 }
382 } else {
383 DPU_DEBUG("crtc=%d disable\n", crtc->base.id);
384 memset(old, 0, sizeof(*old));
385 memset(new, 0, sizeof(*new));
386 update_bus = true;
387 update_clk = true;
388 }
389
390 trace_dpu_perf_crtc_update(crtc->base.id, new->bw_ctl,
391 new->core_clk_rate, stop_req, update_bus, update_clk);
392
393 if (update_bus) {
394 ret = _dpu_core_perf_crtc_update_bus(kms, crtc);
395 if (ret) {
396 DPU_ERROR("crtc-%d: failed to update bus bw vote\n",
397 crtc->base.id);
398 return ret;
399 }
400 }
401
402 /*
403 * Update the clock after bandwidth vote to ensure
404 * bandwidth is available before clock rate is increased.
405 */
406 if (update_clk) {
407 clk_rate = _dpu_core_perf_get_core_clk_rate(kms);
408
409 trace_dpu_core_perf_update_clk(kms->dev, stop_req, clk_rate);
410
411 ret = _dpu_core_perf_set_core_clk_rate(kms, clk_rate);
412 if (ret) {
413 DPU_ERROR("failed to set %s clock rate %llu\n",
414 kms->perf.core_clk->clk_name, clk_rate);
415 return ret;
416 }
417
418 kms->perf.core_clk_rate = clk_rate;
419 DPU_DEBUG("update clk rate = %lld HZ\n", clk_rate);
420 }
421 return 0;
422 }
423
424 #ifdef CONFIG_DEBUG_FS
425
_dpu_core_perf_mode_write(struct file * file,const char __user * user_buf,size_t count,loff_t * ppos)426 static ssize_t _dpu_core_perf_mode_write(struct file *file,
427 const char __user *user_buf, size_t count, loff_t *ppos)
428 {
429 struct dpu_core_perf *perf = file->private_data;
430 struct dpu_perf_cfg *cfg = &perf->catalog->perf;
431 u32 perf_mode = 0;
432 int ret;
433
434 ret = kstrtouint_from_user(user_buf, count, 0, &perf_mode);
435 if (ret)
436 return ret;
437
438 if (perf_mode >= DPU_PERF_MODE_MAX)
439 return -EINVAL;
440
441 if (perf_mode == DPU_PERF_MODE_FIXED) {
442 DRM_INFO("fix performance mode\n");
443 } else if (perf_mode == DPU_PERF_MODE_MINIMUM) {
444 /* run the driver with max clk and BW vote */
445 perf->perf_tune.min_core_clk = perf->max_core_clk_rate;
446 perf->perf_tune.min_bus_vote =
447 (u64) cfg->max_bw_high * 1000;
448 DRM_INFO("minimum performance mode\n");
449 } else if (perf_mode == DPU_PERF_MODE_NORMAL) {
450 /* reset the perf tune params to 0 */
451 perf->perf_tune.min_core_clk = 0;
452 perf->perf_tune.min_bus_vote = 0;
453 DRM_INFO("normal performance mode\n");
454 }
455 perf->perf_tune.mode = perf_mode;
456
457 return count;
458 }
459
_dpu_core_perf_mode_read(struct file * file,char __user * buff,size_t count,loff_t * ppos)460 static ssize_t _dpu_core_perf_mode_read(struct file *file,
461 char __user *buff, size_t count, loff_t *ppos)
462 {
463 struct dpu_core_perf *perf = file->private_data;
464 int len;
465 char buf[128];
466
467 len = scnprintf(buf, sizeof(buf),
468 "mode %d min_mdp_clk %llu min_bus_vote %llu\n",
469 perf->perf_tune.mode,
470 perf->perf_tune.min_core_clk,
471 perf->perf_tune.min_bus_vote);
472
473 return simple_read_from_buffer(buff, count, ppos, buf, len);
474 }
475
476 static const struct file_operations dpu_core_perf_mode_fops = {
477 .open = simple_open,
478 .read = _dpu_core_perf_mode_read,
479 .write = _dpu_core_perf_mode_write,
480 };
481
dpu_core_perf_debugfs_init(struct dpu_kms * dpu_kms,struct dentry * parent)482 int dpu_core_perf_debugfs_init(struct dpu_kms *dpu_kms, struct dentry *parent)
483 {
484 struct dpu_core_perf *perf = &dpu_kms->perf;
485 struct dpu_mdss_cfg *catalog = perf->catalog;
486 struct dentry *entry;
487
488 entry = debugfs_create_dir("core_perf", parent);
489
490 debugfs_create_u64("max_core_clk_rate", 0600, entry,
491 &perf->max_core_clk_rate);
492 debugfs_create_u64("core_clk_rate", 0600, entry,
493 &perf->core_clk_rate);
494 debugfs_create_u32("enable_bw_release", 0600, entry,
495 (u32 *)&perf->enable_bw_release);
496 debugfs_create_u32("threshold_low", 0600, entry,
497 (u32 *)&catalog->perf.max_bw_low);
498 debugfs_create_u32("threshold_high", 0600, entry,
499 (u32 *)&catalog->perf.max_bw_high);
500 debugfs_create_u32("min_core_ib", 0600, entry,
501 (u32 *)&catalog->perf.min_core_ib);
502 debugfs_create_u32("min_llcc_ib", 0600, entry,
503 (u32 *)&catalog->perf.min_llcc_ib);
504 debugfs_create_u32("min_dram_ib", 0600, entry,
505 (u32 *)&catalog->perf.min_dram_ib);
506 debugfs_create_file("perf_mode", 0600, entry,
507 (u32 *)perf, &dpu_core_perf_mode_fops);
508 debugfs_create_u64("fix_core_clk_rate", 0600, entry,
509 &perf->fix_core_clk_rate);
510 debugfs_create_u64("fix_core_ib_vote", 0600, entry,
511 &perf->fix_core_ib_vote);
512 debugfs_create_u64("fix_core_ab_vote", 0600, entry,
513 &perf->fix_core_ab_vote);
514
515 return 0;
516 }
517 #endif
518
dpu_core_perf_destroy(struct dpu_core_perf * perf)519 void dpu_core_perf_destroy(struct dpu_core_perf *perf)
520 {
521 if (!perf) {
522 DPU_ERROR("invalid parameters\n");
523 return;
524 }
525
526 perf->max_core_clk_rate = 0;
527 perf->core_clk = NULL;
528 perf->catalog = NULL;
529 perf->dev = NULL;
530 }
531
dpu_core_perf_init(struct dpu_core_perf * perf,struct drm_device * dev,struct dpu_mdss_cfg * catalog,struct dss_clk * core_clk)532 int dpu_core_perf_init(struct dpu_core_perf *perf,
533 struct drm_device *dev,
534 struct dpu_mdss_cfg *catalog,
535 struct dss_clk *core_clk)
536 {
537 perf->dev = dev;
538 perf->catalog = catalog;
539 perf->core_clk = core_clk;
540
541 perf->max_core_clk_rate = core_clk->max_rate;
542 if (!perf->max_core_clk_rate) {
543 DPU_DEBUG("optional max core clk rate, use default\n");
544 perf->max_core_clk_rate = DPU_PERF_DEFAULT_MAX_CORE_CLK_RATE;
545 }
546
547 return 0;
548 }
549