1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * drivers/auth_ctl/auth_ctrl.c
4 *
5 * Copyright (c) 2022 Huawei Device Co., Ltd.
6 *
7 */
8 #include <linux/sched.h>
9 #include <linux/list.h>
10 #include <linux/mutex.h>
11 #include <linux/stop_machine.h>
12 #include <linux/sched/auth_ctrl.h>
13 #include <linux/sched/rtg_auth.h>
14 #include <linux/sched/qos_ctrl.h>
15 #include <linux/sched/qos_auth.h>
16 #include <uapi/linux/sched/types.h>
17
18 #include "auth_ctrl.h"
19 #include "qos_ctrl.h"
20
21 typedef long (*qos_ctrl_func)(int abi, void __user *uarg);
22
23 static long ctrl_qos_operation(int abi, void __user *uarg);
24 static long ctrl_qos_policy(int abi, void __user *uarg);
25
26 static qos_ctrl_func g_func_array[QOS_CTRL_MAX_NR] = {
27 NULL, /* reserved */
28 ctrl_qos_operation,
29 ctrl_qos_policy,
30 };
31
32 static struct qos_policy_map qos_policy_array[QOS_POLICY_MAX_NR];
33
remove_qos_tasks(struct auth_struct * auth)34 void remove_qos_tasks(struct auth_struct *auth)
35 {
36 int i;
37 struct qos_task_struct *tmp, *next;
38 struct task_struct *p;
39
40 mutex_lock(&auth->mutex);
41 for (i = QOS_POLICY_MIN_LEVEL; i < NR_QOS; ++i) {
42 list_for_each_entry_safe(tmp, next, &auth->tasks[i], qos_list) {
43 p = container_of(tmp, struct task_struct, qts);
44 if (!list_empty(&tmp->qos_list)) {
45 list_del_init(&tmp->qos_list);
46 tmp->in_qos = NO_QOS;
47 put_task_struct(p);
48 }
49 }
50 }
51 mutex_unlock(&auth->mutex);
52 }
53
init_sched_attr(struct sched_attr * attr)54 static void init_sched_attr(struct sched_attr *attr)
55 {
56 memset(attr, 0, sizeof(struct sched_attr));
57 }
58
is_system(unsigned int uid)59 static inline bool is_system(unsigned int uid)
60 {
61 return uid == SYSTEM_UID;
62 }
63
64 /* This function must be called when p is valid. That means the p's refcount must exist */
sched_set_task_qos_attr(struct task_struct * p,int level,int status)65 static int sched_set_task_qos_attr(struct task_struct *p, int level, int status)
66 {
67 struct qos_policy_item *item;
68 struct qos_policy_map *policy_map;
69 struct sched_attr attr;
70
71 read_lock(&qos_policy_array[status].lock);
72 if (!qos_policy_array[status].initialized) {
73 pr_err("[QOS_CTRL] dirty qos policy, pid=%d, uid=%d, status=%d\n",
74 p->pid, p->cred->uid.val, status);
75 read_unlock(&qos_policy_array[status].lock);
76 return -DIRTY_QOS_POLICY;
77 }
78
79 policy_map = &qos_policy_array[status];
80 item = &policy_map->levels[level];
81
82 init_sched_attr(&attr);
83 attr.size = sizeof(struct sched_attr);
84 attr.sched_policy = SCHED_NORMAL;
85
86 if (policy_map->policy_flag & QOS_FLAG_NICE)
87 attr.sched_nice = item->nice;
88
89 if (policy_map->policy_flag & QOS_FLAG_LATENCY_NICE) {
90 attr.sched_flags |= SCHED_FLAG_LATENCY_NICE;
91 attr.sched_latency_nice = item->latency_nice;
92 }
93
94 if ((policy_map->policy_flag & QOS_FLAG_RT) && item->rt_sched_priority) {
95 attr.sched_policy = SCHED_FIFO;
96 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
97 attr.sched_priority = item->rt_sched_priority;
98 }
99
100 read_unlock(&qos_policy_array[status].lock);
101
102 if (unlikely(p->flags & PF_EXITING)) {
103 pr_info("[QOS_CTRL] dying task, no need to set qos\n");
104 return -THREAD_EXITING;
105 }
106
107 return sched_setattr_nocheck(p, &attr);
108 }
109
110 /*
111 * Switch qos mode when status changed.
112 * Lock auth before calling this function
113 */
qos_switch(struct auth_struct * auth,int target_status)114 void qos_switch(struct auth_struct *auth, int target_status)
115 {
116 int i;
117 int ret;
118 struct task_struct *task;
119 struct qos_task_struct *qts;
120
121 if (!auth) {
122 pr_err("[QOS_CTRL] auth no exist, qos switch failed\n");
123 return;
124 }
125
126 lockdep_assert_held(&auth->mutex);
127
128 if (auth->status == target_status) {
129 pr_info("[QOS_CTRL] same status, no need to switch qos\n");
130 return;
131 }
132
133 for (i = QOS_POLICY_MIN_LEVEL; i < NR_QOS; ++i) {
134 list_for_each_entry(qts, &auth->tasks[i], qos_list) {
135 task = container_of(qts, struct task_struct, qts);
136 ret = sched_set_task_qos_attr(task, i, target_status);
137 if (ret)
138 pr_err("[QOS_CTRL] set qos attr failed, qos switch failed\n");
139 }
140 }
141 }
142
qos_insert_task(struct task_struct * p,struct list_head * head,unsigned int level)143 static int qos_insert_task(struct task_struct *p, struct list_head *head, unsigned int level)
144 {
145 struct qos_task_struct *qts = &p->qts;
146
147 if (qts->in_qos > NO_QOS) {
148 pr_err("[QOS_CTRL] qos apply still active, no duplicate add\n");
149 return -PID_DUPLICATE;
150 }
151
152 if (likely(list_empty(&qts->qos_list))) {
153 get_task_struct(p);
154 list_add(&qts->qos_list, head);
155 qts->in_qos = level;
156 }
157
158 return 0;
159 }
160
qos_remove_task(struct task_struct * p)161 static int qos_remove_task(struct task_struct *p)
162 {
163 struct qos_task_struct *qts = (struct qos_task_struct *) &p->qts;
164
165 if (qts->in_qos == NO_QOS) {
166 pr_err("[QOS_CTRL] task not in qos, no need to remove\n");
167 return -PID_NOT_EXIST;
168 }
169
170 if (likely(!list_empty(&qts->qos_list))) {
171 list_del_init(&qts->qos_list);
172 qts->in_qos = NO_QOS;
173 put_task_struct(p);
174 }
175
176 return 0;
177 }
178
same_uid(struct task_struct * dude,struct task_struct * bro)179 static inline bool same_uid(struct task_struct *dude, struct task_struct *bro)
180 {
181 return uid_eq(task_uid(dude), task_uid(bro));
182 }
183
super_user(struct task_struct * p)184 static inline bool super_user(struct task_struct *p)
185 {
186 return super_uid(task_uid(p).val);
187 }
188
189 /*
190 * judge permission for changing other tasks' qos
191 */
can_change_qos(struct task_struct * p)192 static bool can_change_qos(struct task_struct *p)
193 {
194 if (p != current && !same_uid(current, p) && !super_user(current)) {
195 pr_err("[QOS_CTRL] %d apply for others not permit\n", p->pid);
196 return false;
197 }
198
199 return true;
200 }
201
qos_apply(struct qos_ctrl_data * data)202 int qos_apply(struct qos_ctrl_data *data)
203 {
204 unsigned int level = data->level;
205 struct auth_struct *auth;
206 struct task_struct *p;
207 struct qos_task_struct *qts;
208 int pid = data->pid;
209 int ret;
210
211 if (level >= NR_QOS || level == NO_QOS) {
212 pr_err("[QOS_CTRL] no this qos level, qos apply failed\n");
213 ret = -ARG_INVALID;
214 goto out;
215 }
216
217 p = find_get_task_by_vpid((pid_t)pid);
218 if (unlikely(!p)) {
219 pr_err("[QOS_CTRL] no matching task for this pid, qos apply failed\n");
220 ret = -ESRCH;
221 goto out;
222 }
223
224 if (unlikely(p->flags & PF_EXITING)) {
225 pr_info("[QOS_CTRL] dying task, no need to set qos\n");
226 ret = -THREAD_EXITING;
227 goto out_put_task;
228 }
229
230 if (!can_change_qos(p)) {
231 pr_err("[QOS_CTRL] apply for others not permit\n");
232 ret = -ARG_INVALID;
233 goto out_put_task;
234 }
235
236 auth = get_authority(p);
237 if (!auth) {
238 pr_err("[QOS_CTRL] no auth data for pid=%d(%s) this uid=%d, qos apply failed\n",
239 p->pid, p->comm, p->cred->uid.val);
240 put_task_struct(p);
241 ret = -UID_NOT_FOUND;
242 goto out;
243 }
244
245 mutex_lock(&auth->mutex);
246 if (auth->status == AUTH_STATUS_DEAD) {
247 pr_err("[QOS_CTRL] this auth data has been deleted\n");
248 ret = -INVALID_AUTH;
249 goto out_unlock;
250 }
251
252 if (auth->num[level] >= QOS_NUM_MAX) {
253 pr_err("[QOS_CTRL] qos num exceeds limit, cached only\n");
254 ret = -QOS_THREAD_NUM_EXCEED_LIMIT;
255 goto out_unlock;
256 }
257
258 qts = (struct qos_task_struct *) &p->qts;
259
260 /* effective qos must in range [NO_QOS, NR_QOS) */
261 if (qts->in_qos != NO_QOS) {
262 if (qts->in_qos == level) {
263 ret = 0;
264 goto out_unlock;
265 }
266
267 --auth->num[qts->in_qos];
268 qos_remove_task(p);
269 }
270
271 ret = qos_insert_task(p, &auth->tasks[level], level);
272 if (ret < 0) {
273 pr_err("[QOS_CTRL] insert task to qos list %d failed\n", level);
274 goto out_unlock;
275 }
276
277 ++auth->num[level];
278
279 ret = sched_set_task_qos_attr(p, level, auth->status);
280 if (ret) {
281 pr_err("[QOS_CTRL] set qos_level %d for thread %d on status %d failed\n",
282 level, p->pid, auth->status);
283 --auth->num[level];
284 qos_remove_task(p);
285 }
286
287 out_unlock:
288 mutex_unlock(&auth->mutex);
289 put_auth_struct(auth);
290 out_put_task:
291 put_task_struct(p);
292 out:
293 return ret;
294 }
295
qos_leave(struct qos_ctrl_data * data)296 int qos_leave(struct qos_ctrl_data *data)
297 {
298 unsigned int level;
299 struct auth_struct *auth;
300 struct task_struct *p;
301 struct qos_task_struct *qts;
302 int pid = data->pid;
303 int ret;
304
305 p = find_get_task_by_vpid((pid_t)pid);
306 if (!p) {
307 pr_err("[QOS_CTRL] no matching task for this pid, qos apply failed\n");
308 ret = -ESRCH;
309 goto out;
310 }
311
312 if (unlikely(p->flags & PF_EXITING)) {
313 pr_info("[QOS_CTRL] dying task, no need to set qos\n");
314 ret = -THREAD_EXITING;
315 goto out_put_task;
316 }
317
318 if (!can_change_qos(p)) {
319 pr_err("[QOS_CTRL] apply for others not permit\n");
320 ret = -ARG_INVALID;
321 goto out_put_task;
322 }
323
324 auth = get_authority(p);
325 if (!auth) {
326 pr_err("[QOS_CTRL] no auth data for pid=%d(%s) this uid=%d, qos stop failed\n",
327 p->pid, p->comm, p->cred->uid.val);
328 put_task_struct(p);
329 ret = -UID_NOT_FOUND;
330 goto out;
331 }
332
333 mutex_lock(&auth->mutex);
334
335 qts = (struct qos_task_struct *) &p->qts;
336
337 level = qts->in_qos;
338
339 if (level == NO_QOS) {
340 pr_err("[QOS_CTRL] task not in qos list, qos stop failed\n");
341 ret = -ARG_INVALID;
342 goto out_unlock;
343 }
344
345 if (auth->status == AUTH_STATUS_DEAD) {
346 pr_err("[QOS_CTRL] this auth data has been deleted\n");
347 ret = -INVALID_AUTH;
348 goto out_unlock;
349 }
350
351 ret = qos_remove_task(p);
352 if (ret < 0) {
353 pr_err("[QOS_CTRL] remove task from qos list %d failed\n", level);
354 goto out_unlock;
355 }
356
357 --auth->num[level];
358
359 /*
360 * NO NEED to judge whether current status is AUTH_STATUS_DISABLE.
361 * In the auth destoring context, the removing of thread's sched attr was protected by
362 * auth->mutex, AUTH_STATUS_DISABLED will never appear here.
363 *
364 * The second param 3 means nothing, actually you can use any valid level here, cause the
365 * policy matching AUTH_STATUS_DISABLED has default parameters for all qos level, which can
366 * keep a powerful thread to behave like a ordinary thread.
367 */
368 ret = sched_set_task_qos_attr(p, 3, AUTH_STATUS_DISABLED);
369 if (ret)
370 pr_err("[QOS_CTRL] set qos_level %d for thread %d on status %d to default failed\n",
371 level, p->pid, auth->status);
372
373 out_unlock:
374 mutex_unlock(&auth->mutex);
375 put_auth_struct(auth);
376 out_put_task:
377 put_task_struct(p);
378 out:
379 return ret;
380 }
381
init_task_qos(struct task_struct * p)382 void init_task_qos(struct task_struct *p)
383 {
384 struct qos_task_struct *qts = (struct qos_task_struct *) &p->qts;
385
386 INIT_LIST_HEAD(&qts->qos_list);
387 qts->in_qos = NO_QOS;
388 }
389
390 /*
391 * Remove statistic info in auth when task exit
392 */
sched_exit_qos_list(struct task_struct * p)393 void sched_exit_qos_list(struct task_struct *p)
394 {
395 struct auth_struct *auth;
396 struct qos_task_struct *qts = (struct qos_task_struct *) &p->qts;
397
398 /*
399 * For common tasks(the vast majority):
400 * skip get authority, fast return here.
401 *
402 * For qos tasks:
403 * If contend with auth_delete() happens,
404 * 1. function return here, auth_delete() will do the clean up
405 * 2. function go on, either no auth return, either do clean up here
406 * Both cases guarantee data synchronization
407 */
408 if (likely(qts->in_qos == NO_QOS))
409 return;
410
411 auth = get_authority(p);
412 if (!auth)
413 goto out;
414
415 mutex_lock(&auth->mutex);
416 if (qts->in_qos == NO_QOS) {
417 mutex_unlock(&auth->mutex);
418 goto out_put_auth;
419 }
420 --auth->num[qts->in_qos];
421 list_del_init(&qts->qos_list);
422 qts->in_qos = NO_QOS;
423 put_task_struct(p);
424 mutex_unlock(&auth->mutex);
425
426 out_put_auth:
427 put_auth_struct(auth);
428 out:
429 return;
430 }
431
432 typedef int (*qos_manipulate_func)(struct qos_ctrl_data *data);
433
434 static qos_manipulate_func qos_func_array[QOS_OPERATION_CMD_MAX_NR] = {
435 NULL,
436 qos_apply, //1
437 qos_leave,
438 };
439
do_qos_manipulate(struct qos_ctrl_data * data)440 static long do_qos_manipulate(struct qos_ctrl_data *data)
441 {
442 long ret = 0;
443 unsigned int type = data->type;
444
445 if (type <= 0 || type >= QOS_OPERATION_CMD_MAX_NR) {
446 pr_err("[QOS_CTRL] CMD_ID_QOS_MANIPULATE type not valid\n");
447 return -ARG_INVALID;
448 }
449
450 if (qos_func_array[type])
451 ret = (long)(*qos_func_array[type])(data);
452
453 return ret;
454 }
455
ctrl_qos_operation(int abi,void __user * uarg)456 static long ctrl_qos_operation(int abi, void __user *uarg)
457 {
458 struct qos_ctrl_data qos_data;
459 int ret = -1;
460
461 #pragma GCC diagnostic push
462 #pragma GCC diagonstic ignored "-Wpointer-to-int-cast"
463
464 switch (abi) {
465 case QOS_IOCTL_ABI_ARM32:
466 ret = copy_from_user(&qos_data,
467 (void __user *)compat_ptr((compat_uptr_t)uarg),
468 sizeof(struct qos_ctrl_data));
469 break;
470 case QOS_IOCTL_ABI_AARCH64:
471 ret = copy_from_user(&qos_data, uarg, sizeof(struct qos_ctrl_data));
472 break;
473 default:
474 pr_err("[QOS_CTRL] abi format error\n");
475 break;
476 }
477
478 #pragma GCC diagnostic pop
479
480 if (ret) {
481 pr_err("[QOS_CTRL] %s copy user data failed\n", __func__);
482 return ret;
483 }
484
485 /* transfer user space qos level to kernel space qos level */
486 qos_data.level -= QOS_LEVEL_INTERVAL;
487
488 return do_qos_manipulate(&qos_data);
489 }
490
491 #define MAX_LATENCY_NICE 19
492 #define MIN_LATENCY_NICE -20
493
valid_nice(int nice)494 static inline bool valid_nice(int nice)
495 {
496 return nice >= MIN_NICE && nice <= MAX_NICE;
497 }
498
valid_latency_nice(int latency_nice)499 static inline bool valid_latency_nice(int latency_nice)
500 {
501 return latency_nice >= MIN_LATENCY_NICE && latency_nice <= MAX_LATENCY_NICE;
502 }
503
valid_uclamp(int uclamp_min,int uclamp_max)504 static inline bool valid_uclamp(int uclamp_min, int uclamp_max)
505 {
506 if (uclamp_min > uclamp_max)
507 return false;
508 if (uclamp_max > SCHED_CAPACITY_SCALE)
509 return false;
510
511 return true;
512 }
513
valid_rt(int sched_priority)514 static inline bool valid_rt(int sched_priority)
515 {
516 if (sched_priority > MAX_USER_RT_PRIO - 1 || sched_priority < 0)
517 return false;
518
519 return true;
520 }
521
valid_qos_flag(unsigned int qos_flag)522 static bool valid_qos_flag(unsigned int qos_flag)
523 {
524 if (qos_flag & ~QOS_FLAG_ALL)
525 return false;
526
527 return true;
528 }
529
valid_qos_item(struct qos_policy_datas * datas)530 static inline bool valid_qos_item(struct qos_policy_datas *datas)
531 {
532 int i;
533 int type = datas->policy_type;
534 struct qos_policy_data *data;
535
536 if (type <= 0 || type >= QOS_POLICY_MAX_NR) {
537 pr_err("[QOS_CTRL] not valid qos policy type, policy change failed\n");
538 goto out_failed;
539 }
540
541 if (!valid_qos_flag(datas->policy_flag)) {
542 pr_err("[QOS_CTRL] not valid qos flag, policy change failed\n");
543 goto out_failed;
544 }
545
546 /* check user space qos polcicy data, level 0 reserved */
547 for (i = 0; i <= NR_QOS; ++i) {
548 data = &datas->policys[i];
549
550 if (!valid_nice(data->nice)) {
551 pr_err("[QOS_CTRL] invalid nice, policy change failed\n");
552 goto out_failed;
553 }
554
555 if (!valid_latency_nice(data->latency_nice)) {
556 pr_err("[QOS_CTRL] invalid latency_nice, policy change failed\n");
557 goto out_failed;
558 }
559
560 if (!valid_uclamp(data->uclamp_min, data->uclamp_max)) {
561 pr_err("[QOS_CTRL] invalid uclamp, policy change failed\n");
562 goto out_failed;
563 }
564
565 if (!valid_rt(data->rt_sched_priority)) {
566 pr_err("[QOS_CTRL] invalid rt, policy change failed\n");
567 goto out_failed;
568 }
569 }
570
571 return true;
572
573 out_failed:
574 pr_err("[QOS_CTRL] not valid qos policy params\n");
575 return false;
576 }
577
do_qos_policy_change(struct qos_policy_datas * datas)578 static long do_qos_policy_change(struct qos_policy_datas *datas)
579 {
580 long ret = 0;
581 int i;
582 struct qos_policy_item *item;
583 struct qos_policy_data *data;
584 int type = datas->policy_type;
585
586 if (type >= QOS_POLICY_MAX_NR) {
587 pr_err("[QOS_CTRL] not valid policy type\n");
588 goto out_failed;
589 }
590
591 if (!valid_qos_item(datas))
592 goto out_failed;
593
594 write_lock(&qos_policy_array[type].lock);
595 for (i = QOS_POLICY_MIN_LEVEL; i < NR_QOS; ++i) {
596 item = &qos_policy_array[type].levels[i];
597
598 /* user space policy params */
599 data = &datas->policys[i + QOS_LEVEL_INTERVAL];
600
601 item->nice = data->nice;
602 item->latency_nice = data->latency_nice;
603 item->uclamp_min = data->uclamp_min;
604 item->uclamp_max = data->uclamp_max;
605 /* only specific qos level could use SCHED_FIFO */
606 item->rt_sched_priority = (i < MIN_RT_QOS_LEVEL) ? 0 :
607 data->rt_sched_priority;
608 }
609 qos_policy_array[type].policy_flag = datas->policy_flag;
610 qos_policy_array[type].initialized = true;
611 write_unlock(&qos_policy_array[type].lock);
612
613 return ret;
614
615 out_failed:
616 return -ARG_INVALID;
617 }
618
ctrl_qos_policy(int abi,void __user * uarg)619 static long ctrl_qos_policy(int abi, void __user *uarg)
620 {
621 struct qos_policy_datas policy_datas;
622 long ret = -1;
623
624 #pragma GCC diagnostic push
625 #pragma GCC diagnostic ignored "-Wpointer-to-int-cast"
626
627 switch (abi) {
628 case QOS_IOCTL_ABI_ARM32:
629 ret = copy_from_user(&policy_datas,
630 (void __user *)compat_ptr((compat_uptr_t)uarg),
631 sizeof(struct qos_policy_datas));
632 break;
633 case QOS_IOCTL_ABI_AARCH64:
634 ret = copy_from_user(&policy_datas, uarg, sizeof(struct qos_policy_datas));
635 break;
636 default:
637 pr_err("[QOS_CTRL] abi format error\n");
638 break;
639 }
640
641 #pragma GCC diagnostic pop
642
643 if (ret) {
644 pr_err("[QOS_RTG] %s copy user data failed\n", __func__);
645 return ret;
646 }
647
648 return do_qos_policy_change(&policy_datas);
649 }
650
do_qos_ctrl_ioctl(int abi,struct file * file,unsigned int cmd,unsigned long arg)651 long do_qos_ctrl_ioctl(int abi, struct file *file, unsigned int cmd, unsigned long arg)
652 {
653 void __user *uarg = (void __user *)arg;
654 unsigned int func_cmd = _IOC_NR(cmd);
655
656 if (uarg == NULL) {
657 pr_err("%s: invalid user uarg\n", __func__);
658 return -EINVAL;
659 }
660
661 if (_IOC_TYPE(cmd) != QOS_CTRL_IPC_MAGIG) {
662 pr_err("%s: qos ctrl magic fail, TYPE=%d\n",
663 __func__, _IOC_TYPE(cmd));
664 return -EINVAL;
665 }
666
667 if (func_cmd >= QOS_CTRL_MAX_NR) {
668 pr_err("%s: qos ctrl cmd error, cmd:%d\n",
669 __func__, _IOC_TYPE(cmd));
670 return -EINVAL;
671 }
672
673 #ifdef CONFIG_QOS_AUTHORITY
674 if (!check_authorized(func_cmd, QOS_AUTH_FLAG)) {
675 pr_err("[QOS_CTRL] %s: uid not authorized\n", __func__);
676 return -UID_NOT_AUTHORIZED;
677 }
678 #endif
679
680 if (g_func_array[func_cmd])
681 return (*g_func_array[func_cmd])(abi, uarg);
682
683 return -EINVAL;
684 }
685
init_qos_policy_array(void)686 static void init_qos_policy_array(void)
687 {
688 int i;
689
690 /* index 0 reserved */
691 for (i = 1; i < QOS_POLICY_MAX_NR; ++i)
692 rwlock_init(&qos_policy_array[i].lock);
693
694 pr_info("[QOS_CTRL] lock in qos policy initialized\n");
695 }
696
init_qos_ctrl(void)697 int __init init_qos_ctrl(void)
698 {
699 init_qos_policy_array();
700
701 return 0;
702 }
703
704