1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * drivers/auth_ctl/auth_ctrl.c
4 *
5 * Copyright (c) 2022 Huawei Device Co., Ltd.
6 *
7 */
8 #include <linux/sched.h>
9 #include <linux/list.h>
10 #include <linux/mutex.h>
11 #include <linux/stop_machine.h>
12 #include <linux/sched/auth_ctrl.h>
13 #include <linux/sched/rtg_auth.h>
14 #include <linux/sched/qos_ctrl.h>
15 #include <linux/sched/qos_auth.h>
16 #include <uapi/linux/sched/types.h>
17
18 #include "auth_ctrl.h"
19 #include "qos_ctrl.h"
20
21 typedef long (*qos_ctrl_func)(int abi, void __user *uarg);
22
23 static long ctrl_qos_operation(int abi, void __user *uarg);
24 static long ctrl_qos_policy(int abi, void __user *uarg);
25
26 #define QOS_LEVEL_SET_MAX 5
27
28 static qos_ctrl_func g_func_array[QOS_CTRL_MAX_NR] = {
29 NULL, /* reserved */
30 ctrl_qos_operation,
31 ctrl_qos_policy,
32 };
33
34 static struct qos_policy_map qos_policy_array[QOS_POLICY_MAX_NR];
35
remove_qos_tasks(struct auth_struct * auth)36 void remove_qos_tasks(struct auth_struct *auth)
37 {
38 int i;
39 struct qos_task_struct *tmp, *next;
40 struct task_struct *p;
41
42 mutex_lock(&auth->mutex);
43 for (i = QOS_POLICY_MIN_LEVEL; i < NR_QOS; ++i) {
44 list_for_each_entry_safe(tmp, next, &auth->tasks[i], qos_list) {
45 p = container_of(tmp, struct task_struct, qts);
46 if (!list_empty(&tmp->qos_list)) {
47 list_del_init(&tmp->qos_list);
48 tmp->in_qos = NO_QOS;
49 put_task_struct(p);
50 }
51 }
52 }
53 mutex_unlock(&auth->mutex);
54 }
55
init_sched_attr(struct sched_attr * attr)56 static void init_sched_attr(struct sched_attr *attr)
57 {
58 memset(attr, 0, sizeof(struct sched_attr));
59 }
60
is_system(unsigned int uid)61 static inline bool is_system(unsigned int uid)
62 {
63 return uid == SYSTEM_UID;
64 }
65
66 /* This function must be called when p is valid. That means the p's refcount must exist */
sched_set_task_qos_attr(struct task_struct * p,int level,int status)67 static int sched_set_task_qos_attr(struct task_struct *p, int level, int status)
68 {
69 struct qos_policy_item *item;
70 struct qos_policy_map *policy_map;
71 struct sched_attr attr;
72
73 read_lock(&qos_policy_array[status].lock);
74 if (!qos_policy_array[status].initialized) {
75 pr_err("[QOS_CTRL] dirty qos policy, pid=%d, uid=%d, status=%d\n",
76 p->pid, p->cred->uid.val, status);
77 read_unlock(&qos_policy_array[status].lock);
78 return -DIRTY_QOS_POLICY;
79 }
80
81 policy_map = &qos_policy_array[status];
82 item = &policy_map->levels[level];
83
84 init_sched_attr(&attr);
85 attr.size = sizeof(struct sched_attr);
86 attr.sched_policy = SCHED_NORMAL;
87
88 if (policy_map->policy_flag & QOS_FLAG_NICE)
89 attr.sched_nice = item->nice;
90
91 if (policy_map->policy_flag & QOS_FLAG_LATENCY_NICE) {
92 attr.sched_flags |= SCHED_FLAG_LATENCY_NICE;
93 attr.sched_latency_nice = item->latency_nice;
94 }
95
96 if ((policy_map->policy_flag & QOS_FLAG_RT) && item->rt_sched_priority) {
97 attr.sched_policy = SCHED_FIFO;
98 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
99 attr.sched_priority = item->rt_sched_priority;
100 }
101
102 read_unlock(&qos_policy_array[status].lock);
103
104 if (unlikely(p->flags & PF_EXITING)) {
105 pr_info("[QOS_CTRL] dying task, no need to set qos\n");
106 return -THREAD_EXITING;
107 }
108
109 return sched_setattr_nocheck(p, &attr);
110 }
111
112 /*
113 * Switch qos mode when status changed.
114 * Lock auth before calling this function
115 */
qos_switch(struct auth_struct * auth,int target_status)116 void qos_switch(struct auth_struct *auth, int target_status)
117 {
118 int i;
119 int ret;
120 struct task_struct *task;
121 struct qos_task_struct *qts;
122
123 if (!auth) {
124 pr_err("[QOS_CTRL] auth no exist, qos switch failed\n");
125 return;
126 }
127
128 lockdep_assert_held(&auth->mutex);
129
130 if (auth->status == target_status) {
131 pr_info("[QOS_CTRL] same status, no need to switch qos\n");
132 return;
133 }
134
135 for (i = QOS_POLICY_MIN_LEVEL; i < NR_QOS; ++i) {
136 list_for_each_entry(qts, &auth->tasks[i], qos_list) {
137 task = container_of(qts, struct task_struct, qts);
138 ret = sched_set_task_qos_attr(task, i, target_status);
139 if (ret)
140 pr_err("[QOS_CTRL] set qos attr failed, qos switch failed\n");
141 }
142 }
143 }
144
qos_insert_task(struct task_struct * p,struct list_head * head,unsigned int level)145 static int qos_insert_task(struct task_struct *p, struct list_head *head, unsigned int level)
146 {
147 struct qos_task_struct *qts = &p->qts;
148
149 if (qts->in_qos > NO_QOS) {
150 pr_err("[QOS_CTRL] qos apply still active, no duplicate add\n");
151 return -PID_DUPLICATE;
152 }
153
154 if (likely(list_empty(&qts->qos_list))) {
155 get_task_struct(p);
156 list_add(&qts->qos_list, head);
157 qts->in_qos = level;
158 }
159
160 return 0;
161 }
162
qos_remove_task(struct task_struct * p)163 static int qos_remove_task(struct task_struct *p)
164 {
165 struct qos_task_struct *qts = (struct qos_task_struct *) &p->qts;
166
167 if (qts->in_qos == NO_QOS) {
168 pr_err("[QOS_CTRL] task not in qos, no need to remove\n");
169 return -PID_NOT_EXIST;
170 }
171
172 if (likely(!list_empty(&qts->qos_list))) {
173 list_del_init(&qts->qos_list);
174 qts->in_qos = NO_QOS;
175 put_task_struct(p);
176 }
177
178 return 0;
179 }
180
super_user(struct task_struct * p)181 static inline bool super_user(struct task_struct *p)
182 {
183 return super_uid(task_uid(p).val);
184 }
185
186 /*
187 * judge permission for changing tasks' qos
188 */
can_change_qos(struct task_struct * p,unsigned int qos_level)189 static bool can_change_qos(struct task_struct *p, unsigned int qos_level)
190 {
191 struct auth_struct *auth;
192 auth = get_authority(p);
193 /* just system & root user can set(be setted) high qos level */
194 if (!auth || (auth && !super_user(p) && qos_level > QOS_LEVEL_SET_MAX)) {
195 pr_err("[QOS_CTRL] %d have no permission to change qos\n", p->pid);
196 return false;
197 }
198
199 return true;
200 }
201
qos_apply(struct qos_ctrl_data * data)202 int qos_apply(struct qos_ctrl_data *data)
203 {
204 unsigned int level = data->level;
205 struct auth_struct *auth;
206 struct task_struct *p;
207 struct qos_task_struct *qts;
208 int pid = data->pid;
209 int ret;
210
211 if (level >= NR_QOS || level == NO_QOS) {
212 pr_err("[QOS_CTRL] no this qos level, qos apply failed\n");
213 ret = -ARG_INVALID;
214 goto out;
215 }
216
217 p = find_get_task_by_vpid((pid_t)pid);
218 if (unlikely(!p)) {
219 pr_err("[QOS_CTRL] no matching task for this pid, qos apply failed\n");
220 ret = -ESRCH;
221 goto out;
222 }
223
224 if (unlikely(p->flags & PF_EXITING)) {
225 pr_info("[QOS_CTRL] dying task, no need to set qos\n");
226 ret = -THREAD_EXITING;
227 goto out_put_task;
228 }
229
230 if (!can_change_qos(current, level)) {
231 pr_err("[QOS_CTRL] QOS apply not permit\n");
232 ret = -ARG_INVALID;
233 goto out_put_task;
234 }
235
236 auth = get_authority(p);
237 if (!auth) {
238 pr_err("[QOS_CTRL] no auth data for pid=%d(%s), qos apply failed\n",
239 p->tgid, p->comm);
240 ret = -PID_NOT_FOUND;
241 goto out_put_task;
242 }
243
244 mutex_lock(&auth->mutex);
245 if (auth->status == AUTH_STATUS_DEAD) {
246 pr_err("[QOS_CTRL] this auth data has been deleted\n");
247 ret = -INVALID_AUTH;
248 goto out_unlock;
249 }
250
251 if (auth->num[level] >= QOS_NUM_MAX) {
252 pr_err("[QOS_CTRL] qos num exceeds limit, cached only\n");
253 ret = -QOS_THREAD_NUM_EXCEED_LIMIT;
254 goto out_unlock;
255 }
256
257 qts = (struct qos_task_struct *) &p->qts;
258
259 /* effective qos must in range [NO_QOS, NR_QOS) */
260 if (qts->in_qos != NO_QOS) {
261 if (qts->in_qos == level) {
262 ret = 0;
263 goto out_unlock;
264 }
265
266 --auth->num[qts->in_qos];
267 qos_remove_task(p);
268 }
269
270 ret = qos_insert_task(p, &auth->tasks[level], level);
271 if (ret < 0) {
272 pr_err("[QOS_CTRL] insert task to qos list %d failed\n", level);
273 goto out_unlock;
274 }
275
276 ++auth->num[level];
277
278 ret = sched_set_task_qos_attr(p, level, auth->status);
279 if (ret) {
280 pr_err("[QOS_CTRL] set qos_level %d for thread %d on status %d failed\n",
281 level, p->pid, auth->status);
282 --auth->num[level];
283 qos_remove_task(p);
284 }
285
286 out_unlock:
287 mutex_unlock(&auth->mutex);
288 put_auth_struct(auth);
289 out_put_task:
290 put_task_struct(p);
291 out:
292 return ret;
293 }
294
qos_leave(struct qos_ctrl_data * data)295 int qos_leave(struct qos_ctrl_data *data)
296 {
297 unsigned int level;
298 struct auth_struct *auth;
299 struct task_struct *p;
300 struct qos_task_struct *qts;
301 int pid = data->pid;
302 int ret;
303
304 p = find_get_task_by_vpid((pid_t)pid);
305 if (!p) {
306 pr_err("[QOS_CTRL] no matching task for this pid, qos apply failed\n");
307 ret = -ESRCH;
308 goto out;
309 }
310
311 if (unlikely(p->flags & PF_EXITING)) {
312 pr_info("[QOS_CTRL] dying task, no need to set qos\n");
313 ret = -THREAD_EXITING;
314 goto out_put_task;
315 }
316
317 auth = get_authority(p);
318 if (!auth) {
319 pr_err("[QOS_CTRL] no auth data for pid=%d(%s), qos stop failed\n",
320 p->tgid, p->comm);
321 ret = -PID_NOT_FOUND;
322 goto out_put_task;
323 }
324
325 mutex_lock(&auth->mutex);
326
327 qts = (struct qos_task_struct *) &p->qts;
328
329 level = qts->in_qos;
330 if (level == NO_QOS) {
331 pr_err("[QOS_CTRL] task not in qos list, qos stop failed\n");
332 ret = -ARG_INVALID;
333 goto out_unlock;
334 }
335
336 if (!can_change_qos(current, 0)) {
337 pr_err("[QOS_CTRL] apply for others not permit\n");
338 ret = -ARG_INVALID;
339 goto out_unlock;
340 }
341
342 if (auth->status == AUTH_STATUS_DEAD) {
343 pr_err("[QOS_CTRL] this auth data has been deleted\n");
344 ret = -INVALID_AUTH;
345 goto out_unlock;
346 }
347
348 ret = qos_remove_task(p);
349 if (ret < 0) {
350 pr_err("[QOS_CTRL] remove task from qos list %d failed\n", level);
351 goto out_unlock;
352 }
353
354 --auth->num[level];
355
356 /*
357 * NO NEED to judge whether current status is AUTH_STATUS_DISABLE.
358 * In the auth destoring context, the removing of thread's sched attr was protected by
359 * auth->mutex, AUTH_STATUS_DISABLED will never appear here.
360 *
361 * The second param 3 means nothing, actually you can use any valid level here, cause the
362 * policy matching AUTH_STATUS_DISABLED has default parameters for all qos level, which can
363 * keep a powerful thread to behave like a ordinary thread.
364 */
365 ret = sched_set_task_qos_attr(p, 3, AUTH_STATUS_DISABLED);
366 if (ret)
367 pr_err("[QOS_CTRL] set qos_level %d for thread %d on status %d to default failed\n",
368 level, p->pid, auth->status);
369
370 out_unlock:
371 mutex_unlock(&auth->mutex);
372 put_auth_struct(auth);
373 out_put_task:
374 put_task_struct(p);
375 out:
376 return ret;
377 }
378
init_task_qos(struct task_struct * p)379 void init_task_qos(struct task_struct *p)
380 {
381 struct qos_task_struct *qts = (struct qos_task_struct *) &p->qts;
382
383 INIT_LIST_HEAD(&qts->qos_list);
384 qts->in_qos = NO_QOS;
385 }
386
387 /*
388 * Remove statistic info in auth when task exit
389 */
sched_exit_qos_list(struct task_struct * p)390 void sched_exit_qos_list(struct task_struct *p)
391 {
392 struct auth_struct *auth;
393 struct qos_task_struct *qts = (struct qos_task_struct *) &p->qts;
394
395 /*
396 * For common tasks(the vast majority):
397 * skip get authority, fast return here.
398 *
399 * For qos tasks:
400 * If contend with auth_delete() happens,
401 * 1. function return here, auth_delete() will do the clean up
402 * 2. function go on, either no auth return, either do clean up here
403 * Both cases guarantee data synchronization
404 */
405 if (likely(qts->in_qos == NO_QOS))
406 return;
407
408 auth = get_authority(p);
409 if (!auth)
410 goto out;
411
412 mutex_lock(&auth->mutex);
413 if (qts->in_qos == NO_QOS) {
414 mutex_unlock(&auth->mutex);
415 goto out_put_auth;
416 }
417 --auth->num[qts->in_qos];
418 list_del_init(&qts->qos_list);
419 qts->in_qos = NO_QOS;
420 put_task_struct(p);
421 mutex_unlock(&auth->mutex);
422
423 out_put_auth:
424 put_auth_struct(auth);
425 out:
426 return;
427 }
428
429 typedef int (*qos_manipulate_func)(struct qos_ctrl_data *data);
430
431 static qos_manipulate_func qos_func_array[QOS_OPERATION_CMD_MAX_NR] = {
432 NULL,
433 qos_apply, //1
434 qos_leave,
435 };
436
do_qos_manipulate(struct qos_ctrl_data * data)437 static long do_qos_manipulate(struct qos_ctrl_data *data)
438 {
439 long ret = 0;
440 unsigned int type = data->type;
441
442 if (type <= 0 || type >= QOS_OPERATION_CMD_MAX_NR) {
443 pr_err("[QOS_CTRL] CMD_ID_QOS_MANIPULATE type not valid\n");
444 return -ARG_INVALID;
445 }
446
447 if (qos_func_array[type])
448 ret = (long)(*qos_func_array[type])(data);
449
450 return ret;
451 }
452
ctrl_qos_operation(int abi,void __user * uarg)453 static long ctrl_qos_operation(int abi, void __user *uarg)
454 {
455 struct qos_ctrl_data qos_data;
456 int ret = -1;
457
458 #pragma GCC diagnostic push
459 #pragma GCC diagonstic ignored "-Wpointer-to-int-cast"
460
461 switch (abi) {
462 case QOS_IOCTL_ABI_ARM32:
463 ret = copy_from_user(&qos_data,
464 (void __user *)compat_ptr((compat_uptr_t)uarg),
465 sizeof(struct qos_ctrl_data));
466 break;
467 case QOS_IOCTL_ABI_AARCH64:
468 ret = copy_from_user(&qos_data, uarg, sizeof(struct qos_ctrl_data));
469 break;
470 default:
471 pr_err("[QOS_CTRL] abi format error\n");
472 break;
473 }
474
475 #pragma GCC diagnostic pop
476
477 if (ret) {
478 pr_err("[QOS_CTRL] %s copy user data failed\n", __func__);
479 return ret;
480 }
481
482 return do_qos_manipulate(&qos_data);
483 }
484
485 #define MAX_LATENCY_NICE 19
486 #define MIN_LATENCY_NICE -20
487
valid_nice(int nice)488 static inline bool valid_nice(int nice)
489 {
490 return nice >= MIN_NICE && nice <= MAX_NICE;
491 }
492
valid_latency_nice(int latency_nice)493 static inline bool valid_latency_nice(int latency_nice)
494 {
495 return latency_nice >= MIN_LATENCY_NICE && latency_nice <= MAX_LATENCY_NICE;
496 }
497
valid_uclamp(int uclamp_min,int uclamp_max)498 static inline bool valid_uclamp(int uclamp_min, int uclamp_max)
499 {
500 if (uclamp_min > uclamp_max)
501 return false;
502 if (uclamp_max > SCHED_CAPACITY_SCALE)
503 return false;
504
505 return true;
506 }
507
valid_rt(int sched_priority)508 static inline bool valid_rt(int sched_priority)
509 {
510 if (sched_priority > MAX_USER_RT_PRIO - 1 || sched_priority < 0)
511 return false;
512
513 return true;
514 }
515
valid_qos_flag(unsigned int qos_flag)516 static bool valid_qos_flag(unsigned int qos_flag)
517 {
518 if (qos_flag & ~QOS_FLAG_ALL)
519 return false;
520
521 return true;
522 }
523
valid_qos_item(struct qos_policy_datas * datas)524 static inline bool valid_qos_item(struct qos_policy_datas *datas)
525 {
526 int i;
527 int type = datas->policy_type;
528 struct qos_policy_data *data;
529
530 if (type <= 0 || type >= QOS_POLICY_MAX_NR) {
531 pr_err("[QOS_CTRL] not valid qos policy type, policy change failed\n");
532 goto out_failed;
533 }
534
535 if (!valid_qos_flag(datas->policy_flag)) {
536 pr_err("[QOS_CTRL] not valid qos flag, policy change failed\n");
537 goto out_failed;
538 }
539
540 /* check user space qos polcicy data, level 0 reserved */
541 for (i = 0; i < NR_QOS; ++i) {
542 data = &datas->policys[i];
543
544 if (!valid_nice(data->nice)) {
545 pr_err("[QOS_CTRL] invalid nice, policy change failed\n");
546 goto out_failed;
547 }
548
549 if (!valid_latency_nice(data->latency_nice)) {
550 pr_err("[QOS_CTRL] invalid latency_nice, policy change failed\n");
551 goto out_failed;
552 }
553
554 if (!valid_uclamp(data->uclamp_min, data->uclamp_max)) {
555 pr_err("[QOS_CTRL] invalid uclamp, policy change failed\n");
556 goto out_failed;
557 }
558
559 if (!valid_rt(data->rt_sched_priority)) {
560 pr_err("[QOS_CTRL] invalid rt, policy change failed\n");
561 goto out_failed;
562 }
563 }
564
565 return true;
566
567 out_failed:
568 pr_err("[QOS_CTRL] not valid qos policy params\n");
569 return false;
570 }
571
do_qos_policy_change(struct qos_policy_datas * datas)572 static long do_qos_policy_change(struct qos_policy_datas *datas)
573 {
574 long ret = 0;
575 int i;
576 struct qos_policy_item *item;
577 struct qos_policy_data *data;
578 int type = datas->policy_type;
579
580 if (type >= QOS_POLICY_MAX_NR) {
581 pr_err("[QOS_CTRL] not valid policy type\n");
582 goto out_failed;
583 }
584
585 if (!valid_qos_item(datas))
586 goto out_failed;
587
588 write_lock(&qos_policy_array[type].lock);
589 for (i = QOS_POLICY_MIN_LEVEL; i < NR_QOS; ++i) {
590 item = &qos_policy_array[type].levels[i];
591
592 /* user space policy params */
593 data = &datas->policys[i];
594
595 item->nice = data->nice;
596 item->latency_nice = data->latency_nice;
597 item->uclamp_min = data->uclamp_min;
598 item->uclamp_max = data->uclamp_max;
599 /* only specific qos level could use SCHED_FIFO */
600 item->rt_sched_priority = (i < MIN_RT_QOS_LEVEL) ? 0 :
601 data->rt_sched_priority;
602 }
603 qos_policy_array[type].policy_flag = datas->policy_flag;
604 qos_policy_array[type].initialized = true;
605 write_unlock(&qos_policy_array[type].lock);
606
607 return ret;
608
609 out_failed:
610 return -ARG_INVALID;
611 }
612
ctrl_qos_policy(int abi,void __user * uarg)613 static long ctrl_qos_policy(int abi, void __user *uarg)
614 {
615 struct qos_policy_datas policy_datas;
616 long ret = -1;
617
618 #pragma GCC diagnostic push
619 #pragma GCC diagnostic ignored "-Wpointer-to-int-cast"
620
621 switch (abi) {
622 case QOS_IOCTL_ABI_ARM32:
623 ret = copy_from_user(&policy_datas,
624 (void __user *)compat_ptr((compat_uptr_t)uarg),
625 sizeof(struct qos_policy_datas));
626 break;
627 case QOS_IOCTL_ABI_AARCH64:
628 ret = copy_from_user(&policy_datas, uarg, sizeof(struct qos_policy_datas));
629 break;
630 default:
631 pr_err("[QOS_CTRL] abi format error\n");
632 break;
633 }
634
635 #pragma GCC diagnostic pop
636
637 if (ret) {
638 pr_err("[QOS_RTG] %s copy user data failed\n", __func__);
639 return ret;
640 }
641
642 return do_qos_policy_change(&policy_datas);
643 }
644
do_qos_ctrl_ioctl(int abi,struct file * file,unsigned int cmd,unsigned long arg)645 long do_qos_ctrl_ioctl(int abi, struct file *file, unsigned int cmd, unsigned long arg)
646 {
647 void __user *uarg = (void __user *)arg;
648 unsigned int func_cmd = _IOC_NR(cmd);
649
650 if (uarg == NULL) {
651 pr_err("%s: invalid user uarg\n", __func__);
652 return -EINVAL;
653 }
654
655 if (_IOC_TYPE(cmd) != QOS_CTRL_IPC_MAGIG) {
656 pr_err("%s: qos ctrl magic fail, TYPE=%d\n",
657 __func__, _IOC_TYPE(cmd));
658 return -EINVAL;
659 }
660
661 if (func_cmd >= QOS_CTRL_MAX_NR) {
662 pr_err("%s: qos ctrl cmd error, cmd:%d\n",
663 __func__, _IOC_TYPE(cmd));
664 return -EINVAL;
665 }
666
667 #ifdef CONFIG_QOS_AUTHORITY
668 if (!check_authorized(func_cmd, QOS_AUTH_FLAG)) {
669 pr_err("[QOS_CTRL] %s: pid not authorized\n", __func__);
670 return -PID_NOT_AUTHORIZED;
671 }
672 #endif
673
674 if (g_func_array[func_cmd])
675 return (*g_func_array[func_cmd])(abi, uarg);
676
677 return -EINVAL;
678 }
679
init_qos_policy_array(void)680 static void init_qos_policy_array(void)
681 {
682 int i;
683
684 /* index 0 reserved */
685 for (i = 1; i < QOS_POLICY_MAX_NR; ++i)
686 rwlock_init(&qos_policy_array[i].lock);
687
688 pr_info("[QOS_CTRL] lock in qos policy initialized\n");
689 }
690
init_qos_ctrl(void)691 int __init init_qos_ctrl(void)
692 {
693 init_qos_policy_array();
694
695 return 0;
696 }
697
698