1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * drivers/auth_ctl/auth_ctrl.c
4 *
5 * Copyright (c) 2022 Huawei Device Co., Ltd.
6 *
7 */
8 #include <linux/sched.h>
9 #include <linux/list.h>
10 #include <linux/mutex.h>
11 #include <linux/stop_machine.h>
12 #include <linux/sched/auth_ctrl.h>
13 #include <linux/sched/rtg_auth.h>
14 #include <linux/sched/qos_ctrl.h>
15 #include <linux/sched/qos_auth.h>
16 #include <uapi/linux/sched/types.h>
17
18 #include "auth_ctrl.h"
19 #include "qos_ctrl.h"
20
21 typedef long (*qos_ctrl_func)(int abi, void __user *uarg);
22
23 static long ctrl_qos_operation(int abi, void __user *uarg);
24 static long ctrl_qos_policy(int abi, void __user *uarg);
25
26 static qos_ctrl_func g_func_array[QOS_CTRL_MAX_NR] = {
27 NULL, /* reserved */
28 ctrl_qos_operation,
29 ctrl_qos_policy,
30 };
31
32 static struct qos_policy_map qos_policy_array[QOS_POLICY_MAX_NR];
33
remove_qos_tasks(struct auth_struct * auth)34 void remove_qos_tasks(struct auth_struct *auth)
35 {
36 int i;
37 struct qos_task_struct *tmp, *next;
38 struct task_struct *p;
39
40 mutex_lock(&auth->mutex);
41 for (i = QOS_POLICY_MIN_LEVEL; i < NR_QOS; ++i) {
42 list_for_each_entry_safe(tmp, next, &auth->tasks[i], qos_list) {
43 p = container_of(tmp, struct task_struct, qts);
44 if (!list_empty(&tmp->qos_list)) {
45 list_del_init(&tmp->qos_list);
46 tmp->in_qos = NO_QOS;
47 put_task_struct(p);
48 }
49 }
50 }
51 mutex_unlock(&auth->mutex);
52 }
53
init_sched_attr(struct sched_attr * attr)54 static void init_sched_attr(struct sched_attr *attr)
55 {
56 memset(attr, 0, sizeof(struct sched_attr));
57 }
58
is_system(unsigned int uid)59 static inline bool is_system(unsigned int uid)
60 {
61 return uid == SYSTEM_UID;
62 }
63
64 /* This function must be called when p is valid. That means the p's refcount must exist */
sched_set_task_qos_attr(struct task_struct * p,int level,int status)65 static int sched_set_task_qos_attr(struct task_struct *p, int level, int status)
66 {
67 struct qos_policy_item *item;
68 struct qos_policy_map *policy_map;
69 struct sched_attr attr;
70
71 read_lock(&qos_policy_array[status].lock);
72 if (!qos_policy_array[status].initialized) {
73 pr_err("[QOS_CTRL] dirty qos policy, pid=%d, uid=%d, status=%d\n",
74 p->pid, p->cred->uid.val, status);
75 read_unlock(&qos_policy_array[status].lock);
76 return -DIRTY_QOS_POLICY;
77 }
78
79 policy_map = &qos_policy_array[status];
80 item = &policy_map->levels[level];
81
82 init_sched_attr(&attr);
83 attr.size = sizeof(struct sched_attr);
84 attr.sched_policy = SCHED_NORMAL;
85
86 if (policy_map->policy_flag & QOS_FLAG_NICE)
87 attr.sched_nice = item->nice;
88
89 if (policy_map->policy_flag & QOS_FLAG_LATENCY_NICE) {
90 attr.sched_flags |= SCHED_FLAG_LATENCY_NICE;
91 attr.sched_latency_nice = item->latency_nice;
92 }
93
94 if ((policy_map->policy_flag & QOS_FLAG_RT) && item->rt_sched_priority) {
95 attr.sched_policy = SCHED_FIFO;
96 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
97 attr.sched_priority = item->rt_sched_priority;
98 }
99
100 read_unlock(&qos_policy_array[status].lock);
101
102 if (unlikely(p->flags & PF_EXITING)) {
103 pr_info("[QOS_CTRL] dying task, no need to set qos\n");
104 return -THREAD_EXITING;
105 }
106
107 return sched_setattr_nocheck(p, &attr);
108 }
109
110 /*
111 * Switch qos mode when status changed.
112 * Lock auth before calling this function
113 */
qos_switch(struct auth_struct * auth,int target_status)114 void qos_switch(struct auth_struct *auth, int target_status)
115 {
116 int i;
117 int ret;
118 struct task_struct *task;
119 struct qos_task_struct *qts;
120
121 if (!auth) {
122 pr_err("[QOS_CTRL] auth no exist, qos switch failed\n");
123 return;
124 }
125
126 lockdep_assert_held(&auth->mutex);
127
128 if (auth->status == target_status) {
129 pr_info("[QOS_CTRL] same status, no need to switch qos\n");
130 return;
131 }
132
133 for (i = QOS_POLICY_MIN_LEVEL; i < NR_QOS; ++i) {
134 list_for_each_entry(qts, &auth->tasks[i], qos_list) {
135 task = container_of(qts, struct task_struct, qts);
136 ret = sched_set_task_qos_attr(task, i, target_status);
137 if (ret)
138 pr_err("[QOS_CTRL] set qos attr failed, qos switch failed\n");
139 }
140 }
141 }
142
qos_insert_task(struct task_struct * p,struct list_head * head,unsigned int level)143 static int qos_insert_task(struct task_struct *p, struct list_head *head, unsigned int level)
144 {
145 struct qos_task_struct *qts = &p->qts;
146
147 if (qts->in_qos > NO_QOS) {
148 pr_err("[QOS_CTRL] qos apply still active, no duplicate add\n");
149 return -PID_DUPLICATE;
150 }
151
152 if (likely(list_empty(&qts->qos_list))) {
153 get_task_struct(p);
154 list_add(&qts->qos_list, head);
155 qts->in_qos = level;
156 }
157
158 return 0;
159 }
160
qos_remove_task(struct task_struct * p)161 static int qos_remove_task(struct task_struct *p)
162 {
163 struct qos_task_struct *qts = (struct qos_task_struct *) &p->qts;
164
165 if (qts->in_qos == NO_QOS) {
166 pr_err("[QOS_CTRL] task not in qos, no need to remove\n");
167 return -PID_NOT_EXIST;
168 }
169
170 if (likely(!list_empty(&qts->qos_list))) {
171 list_del_init(&qts->qos_list);
172 qts->in_qos = NO_QOS;
173 put_task_struct(p);
174 }
175
176 return 0;
177 }
178
same_uid(struct task_struct * dude,struct task_struct * bro)179 static inline bool same_uid(struct task_struct *dude, struct task_struct *bro)
180 {
181 return uid_eq(task_uid(dude), task_uid(bro));
182 }
183
super_user(struct task_struct * p)184 static inline bool super_user(struct task_struct *p)
185 {
186 return super_uid(task_uid(p).val);
187 }
188
189 /*
190 * judge permission for changing other tasks' qos
191 */
can_change_qos(struct task_struct * p)192 static bool can_change_qos(struct task_struct *p)
193 {
194 if (p != current && !same_uid(current, p) && !super_user(current)) {
195 pr_err("[QOS_CTRL] %d apply for others not permit\n", p->pid);
196 return false;
197 }
198
199 return true;
200 }
201
qos_apply(struct qos_ctrl_data * data)202 int qos_apply(struct qos_ctrl_data *data)
203 {
204 unsigned int level = data->level;
205 struct auth_struct *auth;
206 struct task_struct *p;
207 struct qos_task_struct *qts;
208 int pid = data->pid;
209 int ret;
210
211 if (level >= NR_QOS || level == NO_QOS) {
212 pr_err("[QOS_CTRL] no this qos level, qos apply failed\n");
213 ret = -ARG_INVALID;
214 goto out;
215 }
216
217 p = find_get_task_by_vpid((pid_t)pid);
218 if (unlikely(!p)) {
219 pr_err("[QOS_CTRL] no matching task for this pid, qos apply failed\n");
220 ret = -ESRCH;
221 goto out;
222 }
223
224 if (unlikely(p->flags & PF_EXITING)) {
225 pr_info("[QOS_CTRL] dying task, no need to set qos\n");
226 ret = -THREAD_EXITING;
227 goto out_put_task;
228 }
229
230 if (!can_change_qos(p)) {
231 pr_err("[QOS_CTRL] apply for others not permit\n");
232 ret = -ARG_INVALID;
233 goto out_put_task;
234 }
235
236 auth = get_authority(p);
237 if (!auth) {
238 pr_err("[QOS_CTRL] no auth data for pid=%d(%s) this uid=%d, qos apply failed\n",
239 p->pid, p->comm, p->cred->uid.val);
240 put_task_struct(p);
241 ret = -UID_NOT_FOUND;
242 goto out;
243 }
244
245 mutex_lock(&auth->mutex);
246 if (auth->status == AUTH_STATUS_DEAD) {
247 pr_err("[QOS_CTRL] this auth data has been deleted\n");
248 ret = -INVALID_AUTH;
249 goto out_unlock;
250 }
251
252 if (auth->num[level] >= QOS_NUM_MAX) {
253 pr_err("[QOS_CTRL] qos num exceeds limit, cached only\n");
254 ret = -QOS_THREAD_NUM_EXCEED_LIMIT;
255 goto out_unlock;
256 }
257
258 qts = (struct qos_task_struct *) &p->qts;
259
260 /* effective qos must in range [NO_QOS, NR_QOS) */
261 if (qts->in_qos != NO_QOS) {
262 if (qts->in_qos == level) {
263 ret = 0;
264 goto out_unlock;
265 }
266
267 --auth->num[qts->in_qos];
268 qos_remove_task(p);
269 }
270
271 ret = qos_insert_task(p, &auth->tasks[level], level);
272 if (ret < 0) {
273 pr_err("[QOS_CTRL] insert task to qos list %d failed\n", level);
274 goto out_unlock;
275 }
276
277 ++auth->num[level];
278
279 ret = sched_set_task_qos_attr(p, level, auth->status);
280 if (ret) {
281 pr_err("[QOS_CTRL] set qos_level %d for thread %d on status %d failed\n",
282 level, p->pid, auth->status);
283 --auth->num[level];
284 qos_remove_task(p);
285 }
286
287 out_unlock:
288 mutex_unlock(&auth->mutex);
289 put_auth_struct(auth);
290 out_put_task:
291 put_task_struct(p);
292 out:
293 return ret;
294 }
295
qos_leave(struct qos_ctrl_data * data)296 int qos_leave(struct qos_ctrl_data *data)
297 {
298 unsigned int level;
299 struct auth_struct *auth;
300 struct task_struct *p;
301 struct qos_task_struct *qts;
302 int pid = data->pid;
303 int ret;
304
305 p = find_get_task_by_vpid((pid_t)pid);
306 if (!p) {
307 pr_err("[QOS_CTRL] no matching task for this pid, qos apply failed\n");
308 ret = -ESRCH;
309 goto out;
310 }
311
312 if (unlikely(p->flags & PF_EXITING)) {
313 pr_info("[QOS_CTRL] dying task, no need to set qos\n");
314 ret = -THREAD_EXITING;
315 goto out_put_task;
316 }
317
318 if (!can_change_qos(p)) {
319 pr_err("[QOS_CTRL] apply for others not permit\n");
320 ret = -ARG_INVALID;
321 goto out_put_task;
322 }
323
324 auth = get_authority(p);
325 if (!auth) {
326 pr_err("[QOS_CTRL] no auth data for pid=%d(%s) this uid=%d, qos stop failed\n",
327 p->pid, p->comm, p->cred->uid.val);
328 put_task_struct(p);
329 ret = -UID_NOT_FOUND;
330 goto out;
331 }
332
333 mutex_lock(&auth->mutex);
334
335 qts = (struct qos_task_struct *) &p->qts;
336
337 level = qts->in_qos;
338
339 if (level == NO_QOS) {
340 pr_err("[QOS_CTRL] task not in qos list, qos stop failed\n");
341 ret = -ARG_INVALID;
342 goto out_unlock;
343 }
344
345 if (auth->status == AUTH_STATUS_DEAD) {
346 pr_err("[QOS_CTRL] this auth data has been deleted\n");
347 ret = -INVALID_AUTH;
348 goto out_unlock;
349 }
350
351 ret = qos_remove_task(p);
352 if (ret < 0) {
353 pr_err("[QOS_CTRL] remove task from qos list %d failed\n", level);
354 goto out_unlock;
355 }
356
357 --auth->num[level];
358
359 /*
360 * NO NEED to judge whether current status is AUTH_STATUS_DISABLE.
361 * In the auth destoring context, the removing of thread's sched attr was protected by
362 * auth->mutex, AUTH_STATUS_DISABLED will never appear here.
363 *
364 * The second param 3 means nothing, actually you can use any valid level here, cause the
365 * policy matching AUTH_STATUS_DISABLED has default parameters for all qos level, which can
366 * keep a powerful thread to behave like a ordinary thread.
367 */
368 ret = sched_set_task_qos_attr(p, 3, AUTH_STATUS_DISABLED);
369 if (ret)
370 pr_err("[QOS_CTRL] set qos_level %d for thread %d on status %d to default failed\n",
371 level, p->pid, auth->status);
372
373 out_unlock:
374 mutex_unlock(&auth->mutex);
375 put_auth_struct(auth);
376 out_put_task:
377 put_task_struct(p);
378 out:
379 return ret;
380 }
381
init_task_qos(struct task_struct * p)382 void init_task_qos(struct task_struct *p)
383 {
384 struct qos_task_struct *qts = (struct qos_task_struct *) &p->qts;
385
386 INIT_LIST_HEAD(&qts->qos_list);
387 qts->in_qos = NO_QOS;
388 }
389
390 /*
391 * Remove statistic info in auth when task exit
392 */
sched_exit_qos_list(struct task_struct * p)393 void sched_exit_qos_list(struct task_struct *p)
394 {
395 struct auth_struct *auth;
396 struct qos_task_struct *qts = (struct qos_task_struct *) &p->qts;
397
398 /*
399 * For common tasks(the vast majority):
400 * skip get authority, fast return here.
401 *
402 * For qos tasks:
403 * If contend with auth_delete() happens,
404 * 1. function return here, auth_delete() will do the clean up
405 * 2. function go on, either no auth return, either do clean up here
406 * Both cases guarantee data synchronization
407 */
408 if (likely(qts->in_qos == NO_QOS))
409 return;
410
411 auth = get_authority(p);
412 if (!auth)
413 goto out;
414
415 mutex_lock(&auth->mutex);
416 if (qts->in_qos == NO_QOS) {
417 mutex_unlock(&auth->mutex);
418 goto out_put_auth;
419 }
420 --auth->num[qts->in_qos];
421 list_del_init(&qts->qos_list);
422 qts->in_qos = NO_QOS;
423 mutex_unlock(&auth->mutex);
424
425 out_put_auth:
426 put_auth_struct(auth);
427 out:
428 return;
429 }
430
431 typedef int (*qos_manipulate_func)(struct qos_ctrl_data *data);
432
433 static qos_manipulate_func qos_func_array[QOS_OPERATION_CMD_MAX_NR] = {
434 NULL,
435 qos_apply, //1
436 qos_leave,
437 };
438
do_qos_manipulate(struct qos_ctrl_data * data)439 static long do_qos_manipulate(struct qos_ctrl_data *data)
440 {
441 long ret = 0;
442 unsigned int type = data->type;
443
444 if (type <= 0 || type >= QOS_OPERATION_CMD_MAX_NR) {
445 pr_err("[QOS_CTRL] CMD_ID_QOS_MANIPULATE type not valid\n");
446 return -ARG_INVALID;
447 }
448
449 if (qos_func_array[type])
450 ret = (long)(*qos_func_array[type])(data);
451
452 return ret;
453 }
454
ctrl_qos_operation(int abi,void __user * uarg)455 static long ctrl_qos_operation(int abi, void __user *uarg)
456 {
457 struct qos_ctrl_data qos_data;
458 int ret = -1;
459
460 #pragma GCC diagnostic push
461 #pragma GCC diagonstic ignored "-Wpointer-to-int-cast"
462
463 switch (abi) {
464 case QOS_IOCTL_ABI_ARM32:
465 ret = copy_from_user(&qos_data,
466 (void __user *)compat_ptr((compat_uptr_t)uarg),
467 sizeof(struct qos_ctrl_data));
468 break;
469 case QOS_IOCTL_ABI_AARCH64:
470 ret = copy_from_user(&qos_data, uarg, sizeof(struct qos_ctrl_data));
471 break;
472 default:
473 pr_err("[QOS_CTRL] abi format error\n");
474 break;
475 }
476
477 #pragma GCC diagnostic pop
478
479 if (ret) {
480 pr_err("[QOS_CTRL] %s copy user data failed\n", __func__);
481 return ret;
482 }
483
484 /* transfer user space qos level to kernel space qos level */
485 qos_data.level -= QOS_LEVEL_INTERVAL;
486
487 return do_qos_manipulate(&qos_data);
488 }
489
490 #define MAX_LATENCY_NICE 19
491 #define MIN_LATENCY_NICE -20
492
valid_nice(int nice)493 static inline bool valid_nice(int nice)
494 {
495 return nice >= MIN_NICE && nice <= MAX_NICE;
496 }
497
valid_latency_nice(int latency_nice)498 static inline bool valid_latency_nice(int latency_nice)
499 {
500 return latency_nice >= MIN_LATENCY_NICE && latency_nice <= MAX_LATENCY_NICE;
501 }
502
valid_uclamp(int uclamp_min,int uclamp_max)503 static inline bool valid_uclamp(int uclamp_min, int uclamp_max)
504 {
505 if (uclamp_min > uclamp_max)
506 return false;
507 if (uclamp_max > SCHED_CAPACITY_SCALE)
508 return false;
509
510 return true;
511 }
512
valid_rt(int sched_priority)513 static inline bool valid_rt(int sched_priority)
514 {
515 if (sched_priority > MAX_USER_RT_PRIO - 1 || sched_priority < 0)
516 return false;
517
518 return true;
519 }
520
valid_qos_flag(unsigned int qos_flag)521 static bool valid_qos_flag(unsigned int qos_flag)
522 {
523 if (qos_flag & ~QOS_FLAG_ALL)
524 return false;
525
526 return true;
527 }
528
valid_qos_item(struct qos_policy_datas * datas)529 static inline bool valid_qos_item(struct qos_policy_datas *datas)
530 {
531 int i;
532 int type = datas->policy_type;
533 struct qos_policy_data *data;
534
535 if (type <= 0 || type >= QOS_POLICY_MAX_NR) {
536 pr_err("[QOS_CTRL] not valid qos policy type, policy change failed\n");
537 goto out_failed;
538 }
539
540 if (!valid_qos_flag(datas->policy_flag)) {
541 pr_err("[QOS_CTRL] not valid qos flag, policy change failed\n");
542 goto out_failed;
543 }
544
545 /* check user space qos polcicy data, level 0 reserved */
546 for (i = 0; i <= NR_QOS; ++i) {
547 data = &datas->policys[i];
548
549 if (!valid_nice(data->nice)) {
550 pr_err("[QOS_CTRL] invalid nice, policy change failed\n");
551 goto out_failed;
552 }
553
554 if (!valid_latency_nice(data->latency_nice)) {
555 pr_err("[QOS_CTRL] invalid latency_nice, policy change failed\n");
556 goto out_failed;
557 }
558
559 if (!valid_uclamp(data->uclamp_min, data->uclamp_max)) {
560 pr_err("[QOS_CTRL] invalid uclamp, policy change failed\n");
561 goto out_failed;
562 }
563
564 if (!valid_rt(data->rt_sched_priority)) {
565 pr_err("[QOS_CTRL] invalid rt, policy change failed\n");
566 goto out_failed;
567 }
568 }
569
570 return true;
571
572 out_failed:
573 pr_err("[QOS_CTRL] not valid qos policy params\n");
574 return false;
575 }
576
do_qos_policy_change(struct qos_policy_datas * datas)577 static long do_qos_policy_change(struct qos_policy_datas *datas)
578 {
579 long ret = 0;
580 int i;
581 struct qos_policy_item *item;
582 struct qos_policy_data *data;
583 int type = datas->policy_type;
584
585 if (type >= QOS_POLICY_MAX_NR) {
586 pr_err("[QOS_CTRL] not valid policy type\n");
587 goto out_failed;
588 }
589
590 if (!valid_qos_item(datas))
591 goto out_failed;
592
593 write_lock(&qos_policy_array[type].lock);
594 for (i = QOS_POLICY_MIN_LEVEL; i < NR_QOS; ++i) {
595 item = &qos_policy_array[type].levels[i];
596
597 /* user space policy params */
598 data = &datas->policys[i + QOS_LEVEL_INTERVAL];
599
600 item->nice = data->nice;
601 item->latency_nice = data->latency_nice;
602 item->uclamp_min = data->uclamp_min;
603 item->uclamp_max = data->uclamp_max;
604 /* only specific qos level could use SCHED_FIFO */
605 item->rt_sched_priority = (i < MIN_RT_QOS_LEVEL) ? 0 :
606 data->rt_sched_priority;
607 }
608 qos_policy_array[type].policy_flag = datas->policy_flag;
609 qos_policy_array[type].initialized = true;
610 write_unlock(&qos_policy_array[type].lock);
611
612 return ret;
613
614 out_failed:
615 return -ARG_INVALID;
616 }
617
ctrl_qos_policy(int abi,void __user * uarg)618 static long ctrl_qos_policy(int abi, void __user *uarg)
619 {
620 struct qos_policy_datas policy_datas;
621 long ret = -1;
622
623 #pragma GCC diagnostic push
624 #pragma GCC diagnostic ignored "-Wpointer-to-int-cast"
625
626 switch (abi) {
627 case QOS_IOCTL_ABI_ARM32:
628 ret = copy_from_user(&policy_datas,
629 (void __user *)compat_ptr((compat_uptr_t)uarg),
630 sizeof(struct qos_policy_datas));
631 break;
632 case QOS_IOCTL_ABI_AARCH64:
633 ret = copy_from_user(&policy_datas, uarg, sizeof(struct qos_policy_datas));
634 break;
635 default:
636 pr_err("[QOS_CTRL] abi format error\n");
637 break;
638 }
639
640 #pragma GCC diagnostic pop
641
642 if (ret) {
643 pr_err("[QOS_RTG] %s copy user data failed\n", __func__);
644 return ret;
645 }
646
647 return do_qos_policy_change(&policy_datas);
648 }
649
do_qos_ctrl_ioctl(int abi,struct file * file,unsigned int cmd,unsigned long arg)650 long do_qos_ctrl_ioctl(int abi, struct file *file, unsigned int cmd, unsigned long arg)
651 {
652 void __user *uarg = (void __user *)arg;
653 unsigned int func_cmd = _IOC_NR(cmd);
654
655 if (uarg == NULL) {
656 pr_err("%s: invalid user uarg\n", __func__);
657 return -EINVAL;
658 }
659
660 if (_IOC_TYPE(cmd) != QOS_CTRL_IPC_MAGIG) {
661 pr_err("%s: qos ctrl magic fail, TYPE=%d\n",
662 __func__, _IOC_TYPE(cmd));
663 return -EINVAL;
664 }
665
666 if (func_cmd >= QOS_CTRL_MAX_NR) {
667 pr_err("%s: qos ctrl cmd error, cmd:%d\n",
668 __func__, _IOC_TYPE(cmd));
669 return -EINVAL;
670 }
671
672 #ifdef CONFIG_QOS_AUTHORITY
673 if (!check_authorized(func_cmd, QOS_AUTH_FLAG)) {
674 pr_err("[QOS_CTRL] %s: uid not authorized\n", __func__);
675 return -UID_NOT_AUTHORIZED;
676 }
677 #endif
678
679 if (g_func_array[func_cmd])
680 return (*g_func_array[func_cmd])(abi, uarg);
681
682 return -EINVAL;
683 }
684
init_qos_policy_array(void)685 static void init_qos_policy_array(void)
686 {
687 int i;
688
689 /* index 0 reserved */
690 for (i = 1; i < QOS_POLICY_MAX_NR; ++i)
691 rwlock_init(&qos_policy_array[i].lock);
692
693 pr_info("[QOS_CTRL] lock in qos policy initialized\n");
694 }
695
init_qos_ctrl(void)696 int __init init_qos_ctrl(void)
697 {
698 init_qos_policy_array();
699
700 return 0;
701 }
702
703