• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_SCHED_H
3 #define _LINUX_SCHED_H
4 
5 /*
6  * Define 'struct task_struct' and provide the main scheduler
7  * APIs (schedule(), wakeup variants, etc.)
8  */
9 
10 #include <uapi/linux/sched.h>
11 
12 #include <asm/current.h>
13 
14 #include <linux/pid.h>
15 #include <linux/sem.h>
16 #include <linux/shm.h>
17 #include <linux/kcov.h>
18 #include <linux/mutex.h>
19 #include <linux/plist.h>
20 #include <linux/hrtimer.h>
21 #include <linux/irqflags.h>
22 #include <linux/seccomp.h>
23 #include <linux/nodemask.h>
24 #include <linux/rcupdate.h>
25 #include <linux/refcount.h>
26 #include <linux/resource.h>
27 #include <linux/latencytop.h>
28 #include <linux/sched/prio.h>
29 #include <linux/sched/types.h>
30 #include <linux/signal_types.h>
31 #include <linux/mm_types_task.h>
32 #include <linux/task_io_accounting.h>
33 #include <linux/posix-timers.h>
34 #include <linux/rseq.h>
35 #include <linux/seqlock.h>
36 #include <linux/kcsan.h>
37 #include <linux/android_vendor.h>
38 #include <linux/android_kabi.h>
39 
40 /* task_struct member predeclarations (sorted alphabetically): */
41 struct audit_context;
42 struct backing_dev_info;
43 struct bio_list;
44 struct blk_plug;
45 struct capture_control;
46 struct cfs_rq;
47 struct fs_struct;
48 struct futex_pi_state;
49 struct io_context;
50 struct mempolicy;
51 struct nameidata;
52 struct nsproxy;
53 struct perf_event_context;
54 struct pid_namespace;
55 struct pipe_inode_info;
56 struct rcu_node;
57 struct reclaim_state;
58 struct robust_list_head;
59 struct root_domain;
60 struct rq;
61 struct sched_attr;
62 struct sched_param;
63 struct seq_file;
64 struct sighand_struct;
65 struct signal_struct;
66 struct task_delay_info;
67 struct task_group;
68 struct io_uring_task;
69 
70 /*
71  * Task state bitmask. NOTE! These bits are also
72  * encoded in fs/proc/array.c: get_task_state().
73  *
74  * We have two separate sets of flags: task->state
75  * is about runnability, while task->exit_state are
76  * about the task exiting. Confusing, but this way
77  * modifying one set can't modify the other one by
78  * mistake.
79  */
80 
81 /* Used in tsk->state: */
82 #define TASK_RUNNING			0x0000
83 #define TASK_INTERRUPTIBLE		0x0001
84 #define TASK_UNINTERRUPTIBLE		0x0002
85 #define __TASK_STOPPED			0x0004
86 #define __TASK_TRACED			0x0008
87 /* Used in tsk->exit_state: */
88 #define EXIT_DEAD			0x0010
89 #define EXIT_ZOMBIE			0x0020
90 #define EXIT_TRACE			(EXIT_ZOMBIE | EXIT_DEAD)
91 /* Used in tsk->state again: */
92 #define TASK_PARKED			0x0040
93 #define TASK_DEAD			0x0080
94 #define TASK_WAKEKILL			0x0100
95 #define TASK_WAKING			0x0200
96 #define TASK_NOLOAD			0x0400
97 #define TASK_NEW			0x0800
98 #define TASK_STATE_MAX			0x1000
99 
100 /* Convenience macros for the sake of set_current_state: */
101 #define TASK_KILLABLE			(TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
102 #define TASK_STOPPED			(TASK_WAKEKILL | __TASK_STOPPED)
103 #define TASK_TRACED			(TASK_WAKEKILL | __TASK_TRACED)
104 
105 #define TASK_IDLE			(TASK_UNINTERRUPTIBLE | TASK_NOLOAD)
106 
107 /* Convenience macros for the sake of wake_up(): */
108 #define TASK_NORMAL			(TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
109 
110 /* get_task_state(): */
111 #define TASK_REPORT			(TASK_RUNNING | TASK_INTERRUPTIBLE | \
112 					 TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
113 					 __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \
114 					 TASK_PARKED)
115 
116 #define task_is_traced(task)		((task->state & __TASK_TRACED) != 0)
117 
118 #define task_is_stopped(task)		((task->state & __TASK_STOPPED) != 0)
119 
120 #define task_is_stopped_or_traced(task)	((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
121 
122 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
123 
124 /*
125  * Special states are those that do not use the normal wait-loop pattern. See
126  * the comment with set_special_state().
127  */
128 #define is_special_task_state(state)				\
129 	((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | TASK_DEAD))
130 
131 #define __set_current_state(state_value)			\
132 	do {							\
133 		WARN_ON_ONCE(is_special_task_state(state_value));\
134 		current->task_state_change = _THIS_IP_;		\
135 		current->state = (state_value);			\
136 	} while (0)
137 
138 #define set_current_state(state_value)				\
139 	do {							\
140 		WARN_ON_ONCE(is_special_task_state(state_value));\
141 		current->task_state_change = _THIS_IP_;		\
142 		smp_store_mb(current->state, (state_value));	\
143 	} while (0)
144 
145 #define set_special_state(state_value)					\
146 	do {								\
147 		unsigned long flags; /* may shadow */			\
148 		WARN_ON_ONCE(!is_special_task_state(state_value));	\
149 		raw_spin_lock_irqsave(&current->pi_lock, flags);	\
150 		current->task_state_change = _THIS_IP_;			\
151 		current->state = (state_value);				\
152 		raw_spin_unlock_irqrestore(&current->pi_lock, flags);	\
153 	} while (0)
154 #else
155 /*
156  * set_current_state() includes a barrier so that the write of current->state
157  * is correctly serialised wrt the caller's subsequent test of whether to
158  * actually sleep:
159  *
160  *   for (;;) {
161  *	set_current_state(TASK_UNINTERRUPTIBLE);
162  *	if (CONDITION)
163  *	   break;
164  *
165  *	schedule();
166  *   }
167  *   __set_current_state(TASK_RUNNING);
168  *
169  * If the caller does not need such serialisation (because, for instance, the
170  * CONDITION test and condition change and wakeup are under the same lock) then
171  * use __set_current_state().
172  *
173  * The above is typically ordered against the wakeup, which does:
174  *
175  *   CONDITION = 1;
176  *   wake_up_state(p, TASK_UNINTERRUPTIBLE);
177  *
178  * where wake_up_state()/try_to_wake_up() executes a full memory barrier before
179  * accessing p->state.
180  *
181  * Wakeup will do: if (@state & p->state) p->state = TASK_RUNNING, that is,
182  * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a
183  * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING).
184  *
185  * However, with slightly different timing the wakeup TASK_RUNNING store can
186  * also collide with the TASK_UNINTERRUPTIBLE store. Losing that store is not
187  * a problem either because that will result in one extra go around the loop
188  * and our @cond test will save the day.
189  *
190  * Also see the comments of try_to_wake_up().
191  */
192 #define __set_current_state(state_value)				\
193 	current->state = (state_value)
194 
195 #define set_current_state(state_value)					\
196 	smp_store_mb(current->state, (state_value))
197 
198 /*
199  * set_special_state() should be used for those states when the blocking task
200  * can not use the regular condition based wait-loop. In that case we must
201  * serialize against wakeups such that any possible in-flight TASK_RUNNING stores
202  * will not collide with our state change.
203  */
204 #define set_special_state(state_value)					\
205 	do {								\
206 		unsigned long flags; /* may shadow */			\
207 		raw_spin_lock_irqsave(&current->pi_lock, flags);	\
208 		current->state = (state_value);				\
209 		raw_spin_unlock_irqrestore(&current->pi_lock, flags);	\
210 	} while (0)
211 
212 #endif
213 
214 /* Task command name length: */
215 #define TASK_COMM_LEN			16
216 
217 extern void scheduler_tick(void);
218 
219 #define	MAX_SCHEDULE_TIMEOUT		LONG_MAX
220 
221 extern long schedule_timeout(long timeout);
222 extern long schedule_timeout_interruptible(long timeout);
223 extern long schedule_timeout_killable(long timeout);
224 extern long schedule_timeout_uninterruptible(long timeout);
225 extern long schedule_timeout_idle(long timeout);
226 asmlinkage void schedule(void);
227 extern void schedule_preempt_disabled(void);
228 asmlinkage void preempt_schedule_irq(void);
229 
230 extern int __must_check io_schedule_prepare(void);
231 extern void io_schedule_finish(int token);
232 extern long io_schedule_timeout(long timeout);
233 extern void io_schedule(void);
234 
235 /**
236  * struct prev_cputime - snapshot of system and user cputime
237  * @utime: time spent in user mode
238  * @stime: time spent in system mode
239  * @lock: protects the above two fields
240  *
241  * Stores previous user/system time values such that we can guarantee
242  * monotonicity.
243  */
244 struct prev_cputime {
245 #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
246 	u64				utime;
247 	u64				stime;
248 	raw_spinlock_t			lock;
249 #endif
250 };
251 
252 enum vtime_state {
253 	/* Task is sleeping or running in a CPU with VTIME inactive: */
254 	VTIME_INACTIVE = 0,
255 	/* Task is idle */
256 	VTIME_IDLE,
257 	/* Task runs in kernelspace in a CPU with VTIME active: */
258 	VTIME_SYS,
259 	/* Task runs in userspace in a CPU with VTIME active: */
260 	VTIME_USER,
261 	/* Task runs as guests in a CPU with VTIME active: */
262 	VTIME_GUEST,
263 };
264 
265 struct vtime {
266 	seqcount_t		seqcount;
267 	unsigned long long	starttime;
268 	enum vtime_state	state;
269 	unsigned int		cpu;
270 	u64			utime;
271 	u64			stime;
272 	u64			gtime;
273 };
274 
275 /*
276  * Utilization clamp constraints.
277  * @UCLAMP_MIN:	Minimum utilization
278  * @UCLAMP_MAX:	Maximum utilization
279  * @UCLAMP_CNT:	Utilization clamp constraints count
280  */
281 enum uclamp_id {
282 	UCLAMP_MIN = 0,
283 	UCLAMP_MAX,
284 	UCLAMP_CNT
285 };
286 
287 #ifdef CONFIG_SMP
288 extern struct root_domain def_root_domain;
289 extern struct mutex sched_domains_mutex;
290 #endif
291 
292 struct sched_info {
293 #ifdef CONFIG_SCHED_INFO
294 	/* Cumulative counters: */
295 
296 	/* # of times we have run on this CPU: */
297 	unsigned long			pcount;
298 
299 	/* Time spent waiting on a runqueue: */
300 	unsigned long long		run_delay;
301 
302 	/* Timestamps: */
303 
304 	/* When did we last run on a CPU? */
305 	unsigned long long		last_arrival;
306 
307 	/* When were we last queued to run? */
308 	unsigned long long		last_queued;
309 
310 #endif /* CONFIG_SCHED_INFO */
311 };
312 
313 /*
314  * Integer metrics need fixed point arithmetic, e.g., sched/fair
315  * has a few: load, load_avg, util_avg, freq, and capacity.
316  *
317  * We define a basic fixed point arithmetic range, and then formalize
318  * all these metrics based on that basic range.
319  */
320 # define SCHED_FIXEDPOINT_SHIFT		10
321 # define SCHED_FIXEDPOINT_SCALE		(1L << SCHED_FIXEDPOINT_SHIFT)
322 
323 /* Increase resolution of cpu_capacity calculations */
324 # define SCHED_CAPACITY_SHIFT		SCHED_FIXEDPOINT_SHIFT
325 # define SCHED_CAPACITY_SCALE		(1L << SCHED_CAPACITY_SHIFT)
326 
327 struct load_weight {
328 	unsigned long			weight;
329 	u32				inv_weight;
330 };
331 
332 /**
333  * struct util_est - Estimation utilization of FAIR tasks
334  * @enqueued: instantaneous estimated utilization of a task/cpu
335  * @ewma:     the Exponential Weighted Moving Average (EWMA)
336  *            utilization of a task
337  *
338  * Support data structure to track an Exponential Weighted Moving Average
339  * (EWMA) of a FAIR task's utilization. New samples are added to the moving
340  * average each time a task completes an activation. Sample's weight is chosen
341  * so that the EWMA will be relatively insensitive to transient changes to the
342  * task's workload.
343  *
344  * The enqueued attribute has a slightly different meaning for tasks and cpus:
345  * - task:   the task's util_avg at last task dequeue time
346  * - cfs_rq: the sum of util_est.enqueued for each RUNNABLE task on that CPU
347  * Thus, the util_est.enqueued of a task represents the contribution on the
348  * estimated utilization of the CPU where that task is currently enqueued.
349  *
350  * Only for tasks we track a moving average of the past instantaneous
351  * estimated utilization. This allows to absorb sporadic drops in utilization
352  * of an otherwise almost periodic task.
353  *
354  * The UTIL_AVG_UNCHANGED flag is used to synchronize util_est with util_avg
355  * updates. When a task is dequeued, its util_est should not be updated if its
356  * util_avg has not been updated in the meantime.
357  * This information is mapped into the MSB bit of util_est.enqueued at dequeue
358  * time. Since max value of util_est.enqueued for a task is 1024 (PELT util_avg
359  * for a task) it is safe to use MSB.
360  */
361 struct util_est {
362 	unsigned int			enqueued;
363 	unsigned int			ewma;
364 #define UTIL_EST_WEIGHT_SHIFT		2
365 #define UTIL_AVG_UNCHANGED		0x80000000
366 } __attribute__((__aligned__(sizeof(u64))));
367 
368 /*
369  * The load/runnable/util_avg accumulates an infinite geometric series
370  * (see __update_load_avg_cfs_rq() in kernel/sched/pelt.c).
371  *
372  * [load_avg definition]
373  *
374  *   load_avg = runnable% * scale_load_down(load)
375  *
376  * [runnable_avg definition]
377  *
378  *   runnable_avg = runnable% * SCHED_CAPACITY_SCALE
379  *
380  * [util_avg definition]
381  *
382  *   util_avg = running% * SCHED_CAPACITY_SCALE
383  *
384  * where runnable% is the time ratio that a sched_entity is runnable and
385  * running% the time ratio that a sched_entity is running.
386  *
387  * For cfs_rq, they are the aggregated values of all runnable and blocked
388  * sched_entities.
389  *
390  * The load/runnable/util_avg doesn't directly factor frequency scaling and CPU
391  * capacity scaling. The scaling is done through the rq_clock_pelt that is used
392  * for computing those signals (see update_rq_clock_pelt())
393  *
394  * N.B., the above ratios (runnable% and running%) themselves are in the
395  * range of [0, 1]. To do fixed point arithmetics, we therefore scale them
396  * to as large a range as necessary. This is for example reflected by
397  * util_avg's SCHED_CAPACITY_SCALE.
398  *
399  * [Overflow issue]
400  *
401  * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities
402  * with the highest load (=88761), always runnable on a single cfs_rq,
403  * and should not overflow as the number already hits PID_MAX_LIMIT.
404  *
405  * For all other cases (including 32-bit kernels), struct load_weight's
406  * weight will overflow first before we do, because:
407  *
408  *    Max(load_avg) <= Max(load.weight)
409  *
410  * Then it is the load_weight's responsibility to consider overflow
411  * issues.
412  */
413 struct sched_avg {
414 	u64				last_update_time;
415 	u64				load_sum;
416 	u64				runnable_sum;
417 	u32				util_sum;
418 	u32				period_contrib;
419 	unsigned long			load_avg;
420 	unsigned long			runnable_avg;
421 	unsigned long			util_avg;
422 	struct util_est			util_est;
423 } ____cacheline_aligned;
424 
425 struct sched_statistics {
426 #ifdef CONFIG_SCHEDSTATS
427 	u64				wait_start;
428 	u64				wait_max;
429 	u64				wait_count;
430 	u64				wait_sum;
431 	u64				iowait_count;
432 	u64				iowait_sum;
433 
434 	u64				sleep_start;
435 	u64				sleep_max;
436 	s64				sum_sleep_runtime;
437 
438 	u64				block_start;
439 	u64				block_max;
440 	u64				exec_max;
441 	u64				slice_max;
442 
443 	u64				nr_migrations_cold;
444 	u64				nr_failed_migrations_affine;
445 	u64				nr_failed_migrations_running;
446 	u64				nr_failed_migrations_hot;
447 	u64				nr_forced_migrations;
448 
449 	u64				nr_wakeups;
450 	u64				nr_wakeups_sync;
451 	u64				nr_wakeups_migrate;
452 	u64				nr_wakeups_local;
453 	u64				nr_wakeups_remote;
454 	u64				nr_wakeups_affine;
455 	u64				nr_wakeups_affine_attempts;
456 	u64				nr_wakeups_passive;
457 	u64				nr_wakeups_idle;
458 #endif
459 };
460 
461 struct sched_entity {
462 	/* For load-balancing: */
463 	struct load_weight		load;
464 	struct rb_node			run_node;
465 	struct list_head		group_node;
466 	unsigned int			on_rq;
467 
468 	u64				exec_start;
469 	u64				sum_exec_runtime;
470 	u64				vruntime;
471 	u64				prev_sum_exec_runtime;
472 
473 	u64				nr_migrations;
474 
475 	struct sched_statistics		statistics;
476 
477 #ifdef CONFIG_FAIR_GROUP_SCHED
478 	int				depth;
479 	struct sched_entity		*parent;
480 	/* rq on which this entity is (to be) queued: */
481 	struct cfs_rq			*cfs_rq;
482 	/* rq "owned" by this entity/group: */
483 	struct cfs_rq			*my_q;
484 	/* cached value of my_q->h_nr_running */
485 	unsigned long			runnable_weight;
486 #endif
487 
488 #ifdef CONFIG_SMP
489 	/*
490 	 * Per entity load average tracking.
491 	 *
492 	 * Put into separate cache line so it does not
493 	 * collide with read-mostly values above.
494 	 */
495 	struct sched_avg		avg;
496 #endif
497 
498 	ANDROID_KABI_RESERVE(1);
499 	ANDROID_KABI_RESERVE(2);
500 	ANDROID_KABI_RESERVE(3);
501 	ANDROID_KABI_RESERVE(4);
502 };
503 
504 struct sched_rt_entity {
505 	struct list_head		run_list;
506 	unsigned long			timeout;
507 	unsigned long			watchdog_stamp;
508 	unsigned int			time_slice;
509 	unsigned short			on_rq;
510 	unsigned short			on_list;
511 
512 	struct sched_rt_entity		*back;
513 #ifdef CONFIG_RT_GROUP_SCHED
514 	struct sched_rt_entity		*parent;
515 	/* rq on which this entity is (to be) queued: */
516 	struct rt_rq			*rt_rq;
517 	/* rq "owned" by this entity/group: */
518 	struct rt_rq			*my_q;
519 #endif
520 
521 	ANDROID_KABI_RESERVE(1);
522 	ANDROID_KABI_RESERVE(2);
523 	ANDROID_KABI_RESERVE(3);
524 	ANDROID_KABI_RESERVE(4);
525 } __randomize_layout;
526 
527 struct sched_dl_entity {
528 	struct rb_node			rb_node;
529 
530 	/*
531 	 * Original scheduling parameters. Copied here from sched_attr
532 	 * during sched_setattr(), they will remain the same until
533 	 * the next sched_setattr().
534 	 */
535 	u64				dl_runtime;	/* Maximum runtime for each instance	*/
536 	u64				dl_deadline;	/* Relative deadline of each instance	*/
537 	u64				dl_period;	/* Separation of two instances (period) */
538 	u64				dl_bw;		/* dl_runtime / dl_period		*/
539 	u64				dl_density;	/* dl_runtime / dl_deadline		*/
540 
541 	/*
542 	 * Actual scheduling parameters. Initialized with the values above,
543 	 * they are continuously updated during task execution. Note that
544 	 * the remaining runtime could be < 0 in case we are in overrun.
545 	 */
546 	s64				runtime;	/* Remaining runtime for this instance	*/
547 	u64				deadline;	/* Absolute deadline for this instance	*/
548 	unsigned int			flags;		/* Specifying the scheduler behaviour	*/
549 
550 	/*
551 	 * Some bool flags:
552 	 *
553 	 * @dl_throttled tells if we exhausted the runtime. If so, the
554 	 * task has to wait for a replenishment to be performed at the
555 	 * next firing of dl_timer.
556 	 *
557 	 * @dl_yielded tells if task gave up the CPU before consuming
558 	 * all its available runtime during the last job.
559 	 *
560 	 * @dl_non_contending tells if the task is inactive while still
561 	 * contributing to the active utilization. In other words, it
562 	 * indicates if the inactive timer has been armed and its handler
563 	 * has not been executed yet. This flag is useful to avoid race
564 	 * conditions between the inactive timer handler and the wakeup
565 	 * code.
566 	 *
567 	 * @dl_overrun tells if the task asked to be informed about runtime
568 	 * overruns.
569 	 */
570 	unsigned int			dl_throttled      : 1;
571 	unsigned int			dl_yielded        : 1;
572 	unsigned int			dl_non_contending : 1;
573 	unsigned int			dl_overrun	  : 1;
574 
575 	/*
576 	 * Bandwidth enforcement timer. Each -deadline task has its
577 	 * own bandwidth to be enforced, thus we need one timer per task.
578 	 */
579 	struct hrtimer			dl_timer;
580 
581 	/*
582 	 * Inactive timer, responsible for decreasing the active utilization
583 	 * at the "0-lag time". When a -deadline task blocks, it contributes
584 	 * to GRUB's active utilization until the "0-lag time", hence a
585 	 * timer is needed to decrease the active utilization at the correct
586 	 * time.
587 	 */
588 	struct hrtimer inactive_timer;
589 
590 #ifdef CONFIG_RT_MUTEXES
591 	/*
592 	 * Priority Inheritance. When a DEADLINE scheduling entity is boosted
593 	 * pi_se points to the donor, otherwise points to the dl_se it belongs
594 	 * to (the original one/itself).
595 	 */
596 	struct sched_dl_entity *pi_se;
597 #endif
598 };
599 
600 #ifdef CONFIG_UCLAMP_TASK
601 /* Number of utilization clamp buckets (shorter alias) */
602 #define UCLAMP_BUCKETS CONFIG_UCLAMP_BUCKETS_COUNT
603 
604 /*
605  * Utilization clamp for a scheduling entity
606  * @value:		clamp value "assigned" to a se
607  * @bucket_id:		bucket index corresponding to the "assigned" value
608  * @active:		the se is currently refcounted in a rq's bucket
609  * @user_defined:	the requested clamp value comes from user-space
610  *
611  * The bucket_id is the index of the clamp bucket matching the clamp value
612  * which is pre-computed and stored to avoid expensive integer divisions from
613  * the fast path.
614  *
615  * The active bit is set whenever a task has got an "effective" value assigned,
616  * which can be different from the clamp value "requested" from user-space.
617  * This allows to know a task is refcounted in the rq's bucket corresponding
618  * to the "effective" bucket_id.
619  *
620  * The user_defined bit is set whenever a task has got a task-specific clamp
621  * value requested from userspace, i.e. the system defaults apply to this task
622  * just as a restriction. This allows to relax default clamps when a less
623  * restrictive task-specific value has been requested, thus allowing to
624  * implement a "nice" semantic. For example, a task running with a 20%
625  * default boost can still drop its own boosting to 0%.
626  */
627 struct uclamp_se {
628 	unsigned int value		: bits_per(SCHED_CAPACITY_SCALE);
629 	unsigned int bucket_id		: bits_per(UCLAMP_BUCKETS);
630 	unsigned int active		: 1;
631 	unsigned int user_defined	: 1;
632 };
633 #endif /* CONFIG_UCLAMP_TASK */
634 
635 union rcu_special {
636 	struct {
637 		u8			blocked;
638 		u8			need_qs;
639 		u8			exp_hint; /* Hint for performance. */
640 		u8			need_mb; /* Readers need smp_mb(). */
641 	} b; /* Bits. */
642 	u32 s; /* Set of bits. */
643 };
644 
645 enum perf_event_task_context {
646 	perf_invalid_context = -1,
647 	perf_hw_context = 0,
648 	perf_sw_context,
649 	perf_nr_task_contexts,
650 };
651 
652 struct wake_q_node {
653 	struct wake_q_node *next;
654 };
655 
656 struct task_struct {
657 #ifdef CONFIG_THREAD_INFO_IN_TASK
658 	/*
659 	 * For reasons of header soup (see current_thread_info()), this
660 	 * must be the first element of task_struct.
661 	 */
662 	struct thread_info		thread_info;
663 #endif
664 	/* -1 unrunnable, 0 runnable, >0 stopped: */
665 	volatile long			state;
666 
667 	/*
668 	 * This begins the randomizable portion of task_struct. Only
669 	 * scheduling-critical items should be added above here.
670 	 */
671 	randomized_struct_fields_start
672 
673 	void				*stack;
674 	refcount_t			usage;
675 	/* Per task flags (PF_*), defined further below: */
676 	unsigned int			flags;
677 	unsigned int			ptrace;
678 
679 #ifdef CONFIG_SMP
680 	int				on_cpu;
681 	struct __call_single_node	wake_entry;
682 #ifdef CONFIG_THREAD_INFO_IN_TASK
683 	/* Current CPU: */
684 	unsigned int			cpu;
685 #endif
686 	unsigned int			wakee_flips;
687 	unsigned long			wakee_flip_decay_ts;
688 	struct task_struct		*last_wakee;
689 
690 	/*
691 	 * recent_used_cpu is initially set as the last CPU used by a task
692 	 * that wakes affine another task. Waker/wakee relationships can
693 	 * push tasks around a CPU where each wakeup moves to the next one.
694 	 * Tracking a recently used CPU allows a quick search for a recently
695 	 * used CPU that may be idle.
696 	 */
697 	int				recent_used_cpu;
698 	int				wake_cpu;
699 #endif
700 	int				on_rq;
701 
702 	int				prio;
703 	int				static_prio;
704 	int				normal_prio;
705 	unsigned int			rt_priority;
706 
707 	const struct sched_class	*sched_class;
708 	struct sched_entity		se;
709 	struct sched_rt_entity		rt;
710 #ifdef CONFIG_CGROUP_SCHED
711 	struct task_group		*sched_task_group;
712 #endif
713 	struct sched_dl_entity		dl;
714 
715 #ifdef CONFIG_UCLAMP_TASK
716 	/*
717 	 * Clamp values requested for a scheduling entity.
718 	 * Must be updated with task_rq_lock() held.
719 	 */
720 	struct uclamp_se		uclamp_req[UCLAMP_CNT];
721 	/*
722 	 * Effective clamp values used for a scheduling entity.
723 	 * Must be updated with task_rq_lock() held.
724 	 */
725 	struct uclamp_se		uclamp[UCLAMP_CNT];
726 #endif
727 
728 #ifdef CONFIG_HOTPLUG_CPU
729 	struct list_head		percpu_kthread_node;
730 #endif
731 
732 #ifdef CONFIG_PREEMPT_NOTIFIERS
733 	/* List of struct preempt_notifier: */
734 	struct hlist_head		preempt_notifiers;
735 #endif
736 
737 #ifdef CONFIG_BLK_DEV_IO_TRACE
738 	unsigned int			btrace_seq;
739 #endif
740 
741 	unsigned int			policy;
742 	int				nr_cpus_allowed;
743 	const cpumask_t			*cpus_ptr;
744 	cpumask_t			cpus_mask;
745 
746 #ifdef CONFIG_PREEMPT_RCU
747 	int				rcu_read_lock_nesting;
748 	union rcu_special		rcu_read_unlock_special;
749 	struct list_head		rcu_node_entry;
750 	struct rcu_node			*rcu_blocked_node;
751 #endif /* #ifdef CONFIG_PREEMPT_RCU */
752 
753 #ifdef CONFIG_TASKS_RCU
754 	unsigned long			rcu_tasks_nvcsw;
755 	u8				rcu_tasks_holdout;
756 	u8				rcu_tasks_idx;
757 	int				rcu_tasks_idle_cpu;
758 	struct list_head		rcu_tasks_holdout_list;
759 #endif /* #ifdef CONFIG_TASKS_RCU */
760 
761 #ifdef CONFIG_TASKS_TRACE_RCU
762 	int				trc_reader_nesting;
763 	int				trc_ipi_to_cpu;
764 	union rcu_special		trc_reader_special;
765 	bool				trc_reader_checked;
766 	struct list_head		trc_holdout_list;
767 #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
768 
769 	struct sched_info		sched_info;
770 
771 	struct list_head		tasks;
772 #ifdef CONFIG_SMP
773 	struct plist_node		pushable_tasks;
774 	struct rb_node			pushable_dl_tasks;
775 #endif
776 
777 	struct mm_struct		*mm;
778 	struct mm_struct		*active_mm;
779 
780 	/* Per-thread vma caching: */
781 	struct vmacache			vmacache;
782 
783 #ifdef SPLIT_RSS_COUNTING
784 	struct task_rss_stat		rss_stat;
785 #endif
786 	int				exit_state;
787 	int				exit_code;
788 	int				exit_signal;
789 	/* The signal sent when the parent dies: */
790 	int				pdeath_signal;
791 	/* JOBCTL_*, siglock protected: */
792 	unsigned long			jobctl;
793 
794 	/* Used for emulating ABI behavior of previous Linux versions: */
795 	unsigned int			personality;
796 
797 	/* Scheduler bits, serialized by scheduler locks: */
798 	unsigned			sched_reset_on_fork:1;
799 	unsigned			sched_contributes_to_load:1;
800 	unsigned			sched_migrated:1;
801 #ifdef CONFIG_PSI
802 	unsigned			sched_psi_wake_requeue:1;
803 #endif
804 
805 	/* Force alignment to the next boundary: */
806 	unsigned			:0;
807 
808 	/* Unserialized, strictly 'current' */
809 
810 	/*
811 	 * This field must not be in the scheduler word above due to wakelist
812 	 * queueing no longer being serialized by p->on_cpu. However:
813 	 *
814 	 * p->XXX = X;			ttwu()
815 	 * schedule()			  if (p->on_rq && ..) // false
816 	 *   smp_mb__after_spinlock();	  if (smp_load_acquire(&p->on_cpu) && //true
817 	 *   deactivate_task()		      ttwu_queue_wakelist())
818 	 *     p->on_rq = 0;			p->sched_remote_wakeup = Y;
819 	 *
820 	 * guarantees all stores of 'current' are visible before
821 	 * ->sched_remote_wakeup gets used, so it can be in this word.
822 	 */
823 	unsigned			sched_remote_wakeup:1;
824 
825 	/* Bit to tell LSMs we're in execve(): */
826 	unsigned			in_execve:1;
827 	unsigned			in_iowait:1;
828 #ifndef TIF_RESTORE_SIGMASK
829 	unsigned			restore_sigmask:1;
830 #endif
831 #ifdef CONFIG_MEMCG
832 	unsigned			in_user_fault:1;
833 #endif
834 #ifdef CONFIG_COMPAT_BRK
835 	unsigned			brk_randomized:1;
836 #endif
837 #ifdef CONFIG_CGROUPS
838 	/* disallow userland-initiated cgroup migration */
839 	unsigned			no_cgroup_migration:1;
840 	/* task is frozen/stopped (used by the cgroup freezer) */
841 	unsigned			frozen:1;
842 #endif
843 #ifdef CONFIG_BLK_CGROUP
844 	unsigned			use_memdelay:1;
845 #endif
846 #ifdef CONFIG_PSI
847 	/* Stalled due to lack of memory */
848 	unsigned			in_memstall:1;
849 #endif
850 
851 	unsigned long			atomic_flags; /* Flags requiring atomic access. */
852 
853 	struct restart_block		restart_block;
854 
855 	pid_t				pid;
856 	pid_t				tgid;
857 
858 #ifdef CONFIG_STACKPROTECTOR
859 	/* Canary value for the -fstack-protector GCC feature: */
860 	unsigned long			stack_canary;
861 #endif
862 	/*
863 	 * Pointers to the (original) parent process, youngest child, younger sibling,
864 	 * older sibling, respectively.  (p->father can be replaced with
865 	 * p->real_parent->pid)
866 	 */
867 
868 	/* Real parent process: */
869 	struct task_struct __rcu	*real_parent;
870 
871 	/* Recipient of SIGCHLD, wait4() reports: */
872 	struct task_struct __rcu	*parent;
873 
874 	/*
875 	 * Children/sibling form the list of natural children:
876 	 */
877 	struct list_head		children;
878 	struct list_head		sibling;
879 	struct task_struct		*group_leader;
880 
881 	/*
882 	 * 'ptraced' is the list of tasks this task is using ptrace() on.
883 	 *
884 	 * This includes both natural children and PTRACE_ATTACH targets.
885 	 * 'ptrace_entry' is this task's link on the p->parent->ptraced list.
886 	 */
887 	struct list_head		ptraced;
888 	struct list_head		ptrace_entry;
889 
890 	/* PID/PID hash table linkage. */
891 	struct pid			*thread_pid;
892 	struct hlist_node		pid_links[PIDTYPE_MAX];
893 	struct list_head		thread_group;
894 	struct list_head		thread_node;
895 
896 	struct completion		*vfork_done;
897 
898 	/* CLONE_CHILD_SETTID: */
899 	int __user			*set_child_tid;
900 
901 	/* CLONE_CHILD_CLEARTID: */
902 	int __user			*clear_child_tid;
903 
904 	u64				utime;
905 	u64				stime;
906 #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
907 	u64				utimescaled;
908 	u64				stimescaled;
909 #endif
910 	u64				gtime;
911 #ifdef CONFIG_CPU_FREQ_TIMES
912 	u64				*time_in_state;
913 	unsigned int			max_state;
914 #endif
915 	struct prev_cputime		prev_cputime;
916 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
917 	struct vtime			vtime;
918 #endif
919 
920 #ifdef CONFIG_NO_HZ_FULL
921 	atomic_t			tick_dep_mask;
922 #endif
923 	/* Context switch counts: */
924 	unsigned long			nvcsw;
925 	unsigned long			nivcsw;
926 
927 	/* Monotonic time in nsecs: */
928 	u64				start_time;
929 
930 	/* Boot based time in nsecs: */
931 	u64				start_boottime;
932 
933 	/* MM fault and swap info: this can arguably be seen as either mm-specific or thread-specific: */
934 	unsigned long			min_flt;
935 	unsigned long			maj_flt;
936 
937 	/* Empty if CONFIG_POSIX_CPUTIMERS=n */
938 	struct posix_cputimers		posix_cputimers;
939 
940 #ifdef CONFIG_POSIX_CPU_TIMERS_TASK_WORK
941 	struct posix_cputimers_work	posix_cputimers_work;
942 #endif
943 
944 	/* Process credentials: */
945 
946 	/* Tracer's credentials at attach: */
947 	const struct cred __rcu		*ptracer_cred;
948 
949 	/* Objective and real subjective task credentials (COW): */
950 	const struct cred __rcu		*real_cred;
951 
952 	/* Effective (overridable) subjective task credentials (COW): */
953 	const struct cred __rcu		*cred;
954 
955 #ifdef CONFIG_KEYS
956 	/* Cached requested key. */
957 	struct key			*cached_requested_key;
958 #endif
959 
960 	/*
961 	 * executable name, excluding path.
962 	 *
963 	 * - normally initialized setup_new_exec()
964 	 * - access it with [gs]et_task_comm()
965 	 * - lock it with task_lock()
966 	 */
967 	char				comm[TASK_COMM_LEN];
968 
969 	struct nameidata		*nameidata;
970 
971 #ifdef CONFIG_SYSVIPC
972 	struct sysv_sem			sysvsem;
973 	struct sysv_shm			sysvshm;
974 #endif
975 #ifdef CONFIG_DETECT_HUNG_TASK
976 	unsigned long			last_switch_count;
977 	unsigned long			last_switch_time;
978 #endif
979 	/* Filesystem information: */
980 	struct fs_struct		*fs;
981 
982 	/* Open file information: */
983 	struct files_struct		*files;
984 
985 #ifdef CONFIG_IO_URING
986 	struct io_uring_task		*io_uring;
987 #endif
988 
989 	/* Namespaces: */
990 	struct nsproxy			*nsproxy;
991 
992 	/* Signal handlers: */
993 	struct signal_struct		*signal;
994 	struct sighand_struct __rcu		*sighand;
995 	sigset_t			blocked;
996 	sigset_t			real_blocked;
997 	/* Restored if set_restore_sigmask() was used: */
998 	sigset_t			saved_sigmask;
999 	struct sigpending		pending;
1000 	unsigned long			sas_ss_sp;
1001 	size_t				sas_ss_size;
1002 	unsigned int			sas_ss_flags;
1003 
1004 	struct callback_head		*task_works;
1005 
1006 #ifdef CONFIG_AUDIT
1007 #ifdef CONFIG_AUDITSYSCALL
1008 	struct audit_context		*audit_context;
1009 #endif
1010 	kuid_t				loginuid;
1011 	unsigned int			sessionid;
1012 #endif
1013 	struct seccomp			seccomp;
1014 
1015 	/* Thread group tracking: */
1016 	u64				parent_exec_id;
1017 	u64				self_exec_id;
1018 
1019 	/* Protection against (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, mempolicy: */
1020 	spinlock_t			alloc_lock;
1021 
1022 	/* Protection of the PI data structures: */
1023 	raw_spinlock_t			pi_lock;
1024 
1025 	struct wake_q_node		wake_q;
1026 	int				wake_q_count;
1027 
1028 #ifdef CONFIG_RT_MUTEXES
1029 	/* PI waiters blocked on a rt_mutex held by this task: */
1030 	struct rb_root_cached		pi_waiters;
1031 	/* Updated under owner's pi_lock and rq lock */
1032 	struct task_struct		*pi_top_task;
1033 	/* Deadlock detection and priority inheritance handling: */
1034 	struct rt_mutex_waiter		*pi_blocked_on;
1035 #endif
1036 
1037 #ifdef CONFIG_DEBUG_MUTEXES
1038 	/* Mutex deadlock detection: */
1039 	struct mutex_waiter		*blocked_on;
1040 #endif
1041 
1042 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
1043 	int				non_block_count;
1044 #endif
1045 
1046 #ifdef CONFIG_TRACE_IRQFLAGS
1047 	struct irqtrace_events		irqtrace;
1048 	unsigned int			hardirq_threaded;
1049 	u64				hardirq_chain_key;
1050 	int				softirqs_enabled;
1051 	int				softirq_context;
1052 	int				irq_config;
1053 #endif
1054 
1055 #ifdef CONFIG_LOCKDEP
1056 # define MAX_LOCK_DEPTH			48UL
1057 	u64				curr_chain_key;
1058 	int				lockdep_depth;
1059 	unsigned int			lockdep_recursion;
1060 	struct held_lock		held_locks[MAX_LOCK_DEPTH];
1061 #endif
1062 
1063 #if defined(CONFIG_UBSAN) && !defined(CONFIG_UBSAN_TRAP)
1064 	unsigned int			in_ubsan;
1065 #endif
1066 
1067 	/* Journalling filesystem info: */
1068 	void				*journal_info;
1069 
1070 	/* Stacked block device info: */
1071 	struct bio_list			*bio_list;
1072 
1073 #ifdef CONFIG_BLOCK
1074 	/* Stack plugging: */
1075 	struct blk_plug			*plug;
1076 #endif
1077 
1078 	/* VM state: */
1079 	struct reclaim_state		*reclaim_state;
1080 
1081 	struct backing_dev_info		*backing_dev_info;
1082 
1083 	struct io_context		*io_context;
1084 
1085 #ifdef CONFIG_COMPACTION
1086 	struct capture_control		*capture_control;
1087 #endif
1088 	/* Ptrace state: */
1089 	unsigned long			ptrace_message;
1090 	kernel_siginfo_t		*last_siginfo;
1091 
1092 	struct task_io_accounting	ioac;
1093 #ifdef CONFIG_PSI
1094 	/* Pressure stall state */
1095 	unsigned int			psi_flags;
1096 #endif
1097 #ifdef CONFIG_TASK_XACCT
1098 	/* Accumulated RSS usage: */
1099 	u64				acct_rss_mem1;
1100 	/* Accumulated virtual memory usage: */
1101 	u64				acct_vm_mem1;
1102 	/* stime + utime since last update: */
1103 	u64				acct_timexpd;
1104 #endif
1105 #ifdef CONFIG_CPUSETS
1106 	/* Protected by ->alloc_lock: */
1107 	nodemask_t			mems_allowed;
1108 	/* Seqence number to catch updates: */
1109 	seqcount_spinlock_t		mems_allowed_seq;
1110 	int				cpuset_mem_spread_rotor;
1111 	int				cpuset_slab_spread_rotor;
1112 #endif
1113 #ifdef CONFIG_CGROUPS
1114 	/* Control Group info protected by css_set_lock: */
1115 	struct css_set __rcu		*cgroups;
1116 	/* cg_list protected by css_set_lock and tsk->alloc_lock: */
1117 	struct list_head		cg_list;
1118 #endif
1119 #ifdef CONFIG_X86_CPU_RESCTRL
1120 	u32				closid;
1121 	u32				rmid;
1122 #endif
1123 #ifdef CONFIG_FUTEX
1124 	struct robust_list_head __user	*robust_list;
1125 #ifdef CONFIG_COMPAT
1126 	struct compat_robust_list_head __user *compat_robust_list;
1127 #endif
1128 	struct list_head		pi_state_list;
1129 	struct futex_pi_state		*pi_state_cache;
1130 	struct mutex			futex_exit_mutex;
1131 	unsigned int			futex_state;
1132 #endif
1133 #ifdef CONFIG_PERF_EVENTS
1134 	struct perf_event_context	*perf_event_ctxp[perf_nr_task_contexts];
1135 	struct mutex			perf_event_mutex;
1136 	struct list_head		perf_event_list;
1137 #endif
1138 #ifdef CONFIG_DEBUG_PREEMPT
1139 	unsigned long			preempt_disable_ip;
1140 #endif
1141 #ifdef CONFIG_NUMA
1142 	/* Protected by alloc_lock: */
1143 	struct mempolicy		*mempolicy;
1144 	short				il_prev;
1145 	short				pref_node_fork;
1146 #endif
1147 #ifdef CONFIG_NUMA_BALANCING
1148 	int				numa_scan_seq;
1149 	unsigned int			numa_scan_period;
1150 	unsigned int			numa_scan_period_max;
1151 	int				numa_preferred_nid;
1152 	unsigned long			numa_migrate_retry;
1153 	/* Migration stamp: */
1154 	u64				node_stamp;
1155 	u64				last_task_numa_placement;
1156 	u64				last_sum_exec_runtime;
1157 	struct callback_head		numa_work;
1158 
1159 	/*
1160 	 * This pointer is only modified for current in syscall and
1161 	 * pagefault context (and for tasks being destroyed), so it can be read
1162 	 * from any of the following contexts:
1163 	 *  - RCU read-side critical section
1164 	 *  - current->numa_group from everywhere
1165 	 *  - task's runqueue locked, task not running
1166 	 */
1167 	struct numa_group __rcu		*numa_group;
1168 
1169 	/*
1170 	 * numa_faults is an array split into four regions:
1171 	 * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer
1172 	 * in this precise order.
1173 	 *
1174 	 * faults_memory: Exponential decaying average of faults on a per-node
1175 	 * basis. Scheduling placement decisions are made based on these
1176 	 * counts. The values remain static for the duration of a PTE scan.
1177 	 * faults_cpu: Track the nodes the process was running on when a NUMA
1178 	 * hinting fault was incurred.
1179 	 * faults_memory_buffer and faults_cpu_buffer: Record faults per node
1180 	 * during the current scan window. When the scan completes, the counts
1181 	 * in faults_memory and faults_cpu decay and these values are copied.
1182 	 */
1183 	unsigned long			*numa_faults;
1184 	unsigned long			total_numa_faults;
1185 
1186 	/*
1187 	 * numa_faults_locality tracks if faults recorded during the last
1188 	 * scan window were remote/local or failed to migrate. The task scan
1189 	 * period is adapted based on the locality of the faults with different
1190 	 * weights depending on whether they were shared or private faults
1191 	 */
1192 	unsigned long			numa_faults_locality[3];
1193 
1194 	unsigned long			numa_pages_migrated;
1195 #endif /* CONFIG_NUMA_BALANCING */
1196 
1197 #ifdef CONFIG_RSEQ
1198 	struct rseq __user *rseq;
1199 	u32 rseq_sig;
1200 	/*
1201 	 * RmW on rseq_event_mask must be performed atomically
1202 	 * with respect to preemption.
1203 	 */
1204 	unsigned long rseq_event_mask;
1205 #endif
1206 
1207 	struct tlbflush_unmap_batch	tlb_ubc;
1208 
1209 	union {
1210 		refcount_t		rcu_users;
1211 		struct rcu_head		rcu;
1212 	};
1213 
1214 	/* Cache last used pipe for splice(): */
1215 	struct pipe_inode_info		*splice_pipe;
1216 
1217 	struct page_frag		task_frag;
1218 
1219 #ifdef CONFIG_TASK_DELAY_ACCT
1220 	struct task_delay_info		*delays;
1221 #endif
1222 
1223 #ifdef CONFIG_FAULT_INJECTION
1224 	int				make_it_fail;
1225 	unsigned int			fail_nth;
1226 #endif
1227 	/*
1228 	 * When (nr_dirtied >= nr_dirtied_pause), it's time to call
1229 	 * balance_dirty_pages() for a dirty throttling pause:
1230 	 */
1231 	int				nr_dirtied;
1232 	int				nr_dirtied_pause;
1233 	/* Start of a write-and-pause period: */
1234 	unsigned long			dirty_paused_when;
1235 
1236 #ifdef CONFIG_LATENCYTOP
1237 	int				latency_record_count;
1238 	struct latency_record		latency_record[LT_SAVECOUNT];
1239 #endif
1240 	/*
1241 	 * Time slack values; these are used to round up poll() and
1242 	 * select() etc timeout values. These are in nanoseconds.
1243 	 */
1244 	u64				timer_slack_ns;
1245 	u64				default_timer_slack_ns;
1246 
1247 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
1248 	unsigned int			kasan_depth;
1249 #endif
1250 
1251 #ifdef CONFIG_KCSAN
1252 	struct kcsan_ctx		kcsan_ctx;
1253 #ifdef CONFIG_TRACE_IRQFLAGS
1254 	struct irqtrace_events		kcsan_save_irqtrace;
1255 #endif
1256 #endif
1257 
1258 #if IS_ENABLED(CONFIG_KUNIT)
1259 	struct kunit			*kunit_test;
1260 #endif
1261 
1262 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1263 	/* Index of current stored address in ret_stack: */
1264 	int				curr_ret_stack;
1265 	int				curr_ret_depth;
1266 
1267 	/* Stack of return addresses for return function tracing: */
1268 	struct ftrace_ret_stack		*ret_stack;
1269 
1270 	/* Timestamp for last schedule: */
1271 	unsigned long long		ftrace_timestamp;
1272 
1273 	/*
1274 	 * Number of functions that haven't been traced
1275 	 * because of depth overrun:
1276 	 */
1277 	atomic_t			trace_overrun;
1278 
1279 	/* Pause tracing: */
1280 	atomic_t			tracing_graph_pause;
1281 #endif
1282 
1283 #ifdef CONFIG_TRACING
1284 	/* State flags for use by tracers: */
1285 	unsigned long			trace;
1286 
1287 	/* Bitmask and counter of trace recursion: */
1288 	unsigned long			trace_recursion;
1289 #endif /* CONFIG_TRACING */
1290 
1291 #ifdef CONFIG_KCOV
1292 	/* See kernel/kcov.c for more details. */
1293 
1294 	/* Coverage collection mode enabled for this task (0 if disabled): */
1295 	unsigned int			kcov_mode;
1296 
1297 	/* Size of the kcov_area: */
1298 	unsigned int			kcov_size;
1299 
1300 	/* Buffer for coverage collection: */
1301 	void				*kcov_area;
1302 
1303 	/* KCOV descriptor wired with this task or NULL: */
1304 	struct kcov			*kcov;
1305 
1306 	/* KCOV common handle for remote coverage collection: */
1307 	u64				kcov_handle;
1308 
1309 	/* KCOV sequence number: */
1310 	int				kcov_sequence;
1311 
1312 	/* Collect coverage from softirq context: */
1313 	unsigned int			kcov_softirq;
1314 #endif
1315 
1316 #ifdef CONFIG_MEMCG
1317 	struct mem_cgroup		*memcg_in_oom;
1318 	gfp_t				memcg_oom_gfp_mask;
1319 	int				memcg_oom_order;
1320 
1321 	/* Number of pages to reclaim on returning to userland: */
1322 	unsigned int			memcg_nr_pages_over_high;
1323 
1324 	/* Used by memcontrol for targeted memcg charge: */
1325 	struct mem_cgroup		*active_memcg;
1326 #endif
1327 
1328 #ifdef CONFIG_BLK_CGROUP
1329 	struct request_queue		*throttle_queue;
1330 #endif
1331 
1332 #ifdef CONFIG_UPROBES
1333 	struct uprobe_task		*utask;
1334 #endif
1335 #if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE)
1336 	unsigned int			sequential_io;
1337 	unsigned int			sequential_io_avg;
1338 #endif
1339 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
1340 	unsigned long			task_state_change;
1341 #endif
1342 	int				pagefault_disabled;
1343 #ifdef CONFIG_MMU
1344 	struct task_struct		*oom_reaper_list;
1345 #endif
1346 #ifdef CONFIG_VMAP_STACK
1347 	struct vm_struct		*stack_vm_area;
1348 #endif
1349 #ifdef CONFIG_THREAD_INFO_IN_TASK
1350 	/* A live task holds one reference: */
1351 	refcount_t			stack_refcount;
1352 #endif
1353 #ifdef CONFIG_LIVEPATCH
1354 	int patch_state;
1355 #endif
1356 #ifdef CONFIG_SECURITY
1357 	/* Used by LSM modules for access restriction: */
1358 	void				*security;
1359 #endif
1360 
1361 #ifdef CONFIG_GCC_PLUGIN_STACKLEAK
1362 	unsigned long			lowest_stack;
1363 	unsigned long			prev_lowest_stack;
1364 #endif
1365 
1366 #ifdef CONFIG_X86_MCE
1367 	void __user			*mce_vaddr;
1368 	__u64				mce_kflags;
1369 	u64				mce_addr;
1370 	__u64				mce_ripv : 1,
1371 					mce_whole_page : 1,
1372 					__mce_reserved : 62;
1373 	struct callback_head		mce_kill_me;
1374 	int				mce_count;
1375 #endif
1376 	ANDROID_VENDOR_DATA_ARRAY(1, 64);
1377 	ANDROID_OEM_DATA_ARRAY(1, 32);
1378 
1379 	/* PF_IO_WORKER */
1380 	ANDROID_KABI_USE(1, void *pf_io_worker);
1381 
1382 	ANDROID_KABI_RESERVE(2);
1383 	ANDROID_KABI_RESERVE(3);
1384 	ANDROID_KABI_RESERVE(4);
1385 	ANDROID_KABI_RESERVE(5);
1386 	ANDROID_KABI_RESERVE(6);
1387 	ANDROID_KABI_RESERVE(7);
1388 	ANDROID_KABI_RESERVE(8);
1389 
1390 	/*
1391 	 * New fields for task_struct should be added above here, so that
1392 	 * they are included in the randomized portion of task_struct.
1393 	 */
1394 	randomized_struct_fields_end
1395 
1396 	/* CPU-specific state of this task: */
1397 	struct thread_struct		thread;
1398 
1399 	/*
1400 	 * WARNING: on x86, 'thread_struct' contains a variable-sized
1401 	 * structure.  It *MUST* be at the end of 'task_struct'.
1402 	 *
1403 	 * Do not put anything below here!
1404 	 */
1405 };
1406 
task_pid(struct task_struct * task)1407 static inline struct pid *task_pid(struct task_struct *task)
1408 {
1409 	return task->thread_pid;
1410 }
1411 
1412 /*
1413  * the helpers to get the task's different pids as they are seen
1414  * from various namespaces
1415  *
1416  * task_xid_nr()     : global id, i.e. the id seen from the init namespace;
1417  * task_xid_vnr()    : virtual id, i.e. the id seen from the pid namespace of
1418  *                     current.
1419  * task_xid_nr_ns()  : id seen from the ns specified;
1420  *
1421  * see also pid_nr() etc in include/linux/pid.h
1422  */
1423 pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, struct pid_namespace *ns);
1424 
task_pid_nr(struct task_struct * tsk)1425 static inline pid_t task_pid_nr(struct task_struct *tsk)
1426 {
1427 	return tsk->pid;
1428 }
1429 
task_pid_nr_ns(struct task_struct * tsk,struct pid_namespace * ns)1430 static inline pid_t task_pid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1431 {
1432 	return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns);
1433 }
1434 
task_pid_vnr(struct task_struct * tsk)1435 static inline pid_t task_pid_vnr(struct task_struct *tsk)
1436 {
1437 	return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL);
1438 }
1439 
1440 
task_tgid_nr(struct task_struct * tsk)1441 static inline pid_t task_tgid_nr(struct task_struct *tsk)
1442 {
1443 	return tsk->tgid;
1444 }
1445 
1446 /**
1447  * pid_alive - check that a task structure is not stale
1448  * @p: Task structure to be checked.
1449  *
1450  * Test if a process is not yet dead (at most zombie state)
1451  * If pid_alive fails, then pointers within the task structure
1452  * can be stale and must not be dereferenced.
1453  *
1454  * Return: 1 if the process is alive. 0 otherwise.
1455  */
pid_alive(const struct task_struct * p)1456 static inline int pid_alive(const struct task_struct *p)
1457 {
1458 	return p->thread_pid != NULL;
1459 }
1460 
task_pgrp_nr_ns(struct task_struct * tsk,struct pid_namespace * ns)1461 static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1462 {
1463 	return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns);
1464 }
1465 
task_pgrp_vnr(struct task_struct * tsk)1466 static inline pid_t task_pgrp_vnr(struct task_struct *tsk)
1467 {
1468 	return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL);
1469 }
1470 
1471 
task_session_nr_ns(struct task_struct * tsk,struct pid_namespace * ns)1472 static inline pid_t task_session_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1473 {
1474 	return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns);
1475 }
1476 
task_session_vnr(struct task_struct * tsk)1477 static inline pid_t task_session_vnr(struct task_struct *tsk)
1478 {
1479 	return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL);
1480 }
1481 
task_tgid_nr_ns(struct task_struct * tsk,struct pid_namespace * ns)1482 static inline pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1483 {
1484 	return __task_pid_nr_ns(tsk, PIDTYPE_TGID, ns);
1485 }
1486 
task_tgid_vnr(struct task_struct * tsk)1487 static inline pid_t task_tgid_vnr(struct task_struct *tsk)
1488 {
1489 	return __task_pid_nr_ns(tsk, PIDTYPE_TGID, NULL);
1490 }
1491 
task_ppid_nr_ns(const struct task_struct * tsk,struct pid_namespace * ns)1492 static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns)
1493 {
1494 	pid_t pid = 0;
1495 
1496 	rcu_read_lock();
1497 	if (pid_alive(tsk))
1498 		pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns);
1499 	rcu_read_unlock();
1500 
1501 	return pid;
1502 }
1503 
task_ppid_nr(const struct task_struct * tsk)1504 static inline pid_t task_ppid_nr(const struct task_struct *tsk)
1505 {
1506 	return task_ppid_nr_ns(tsk, &init_pid_ns);
1507 }
1508 
1509 /* Obsolete, do not use: */
task_pgrp_nr(struct task_struct * tsk)1510 static inline pid_t task_pgrp_nr(struct task_struct *tsk)
1511 {
1512 	return task_pgrp_nr_ns(tsk, &init_pid_ns);
1513 }
1514 
1515 #define TASK_REPORT_IDLE	(TASK_REPORT + 1)
1516 #define TASK_REPORT_MAX		(TASK_REPORT_IDLE << 1)
1517 
task_state_index(struct task_struct * tsk)1518 static inline unsigned int task_state_index(struct task_struct *tsk)
1519 {
1520 	unsigned int tsk_state = READ_ONCE(tsk->state);
1521 	unsigned int state = (tsk_state | tsk->exit_state) & TASK_REPORT;
1522 
1523 	BUILD_BUG_ON_NOT_POWER_OF_2(TASK_REPORT_MAX);
1524 
1525 	if (tsk_state == TASK_IDLE)
1526 		state = TASK_REPORT_IDLE;
1527 
1528 	return fls(state);
1529 }
1530 
task_index_to_char(unsigned int state)1531 static inline char task_index_to_char(unsigned int state)
1532 {
1533 	static const char state_char[] = "RSDTtXZPI";
1534 
1535 	BUILD_BUG_ON(1 + ilog2(TASK_REPORT_MAX) != sizeof(state_char) - 1);
1536 
1537 	return state_char[state];
1538 }
1539 
task_state_to_char(struct task_struct * tsk)1540 static inline char task_state_to_char(struct task_struct *tsk)
1541 {
1542 	return task_index_to_char(task_state_index(tsk));
1543 }
1544 
1545 /**
1546  * is_global_init - check if a task structure is init. Since init
1547  * is free to have sub-threads we need to check tgid.
1548  * @tsk: Task structure to be checked.
1549  *
1550  * Check if a task structure is the first user space task the kernel created.
1551  *
1552  * Return: 1 if the task structure is init. 0 otherwise.
1553  */
is_global_init(struct task_struct * tsk)1554 static inline int is_global_init(struct task_struct *tsk)
1555 {
1556 	return task_tgid_nr(tsk) == 1;
1557 }
1558 
1559 extern struct pid *cad_pid;
1560 
1561 /*
1562  * Per process flags
1563  */
1564 #define PF_VCPU			0x00000001	/* I'm a virtual CPU */
1565 #define PF_IDLE			0x00000002	/* I am an IDLE thread */
1566 #define PF_EXITING		0x00000004	/* Getting shut down */
1567 #define PF_IO_WORKER		0x00000010	/* Task is an IO worker */
1568 #define PF_WQ_WORKER		0x00000020	/* I'm a workqueue worker */
1569 #define PF_FORKNOEXEC		0x00000040	/* Forked but didn't exec */
1570 #define PF_MCE_PROCESS		0x00000080      /* Process policy on mce errors */
1571 #define PF_SUPERPRIV		0x00000100	/* Used super-user privileges */
1572 #define PF_DUMPCORE		0x00000200	/* Dumped core */
1573 #define PF_SIGNALED		0x00000400	/* Killed by a signal */
1574 #define PF_MEMALLOC		0x00000800	/* Allocating memory */
1575 #define PF_NPROC_EXCEEDED	0x00001000	/* set_user() noticed that RLIMIT_NPROC was exceeded */
1576 #define PF_USED_MATH		0x00002000	/* If unset the fpu must be initialized before use */
1577 #define PF_NOFREEZE		0x00008000	/* This thread should not be frozen */
1578 #define PF_FROZEN		0x00010000	/* Frozen for system suspend */
1579 #define PF_KSWAPD		0x00020000	/* I am kswapd */
1580 #define PF_MEMALLOC_NOFS	0x00040000	/* All allocation requests will inherit GFP_NOFS */
1581 #define PF_MEMALLOC_NOIO	0x00080000	/* All allocation requests will inherit GFP_NOIO */
1582 #define PF_LOCAL_THROTTLE	0x00100000	/* Throttle writes only against the bdi I write to,
1583 						 * I am cleaning dirty pages from some other bdi. */
1584 #define PF_KTHREAD		0x00200000	/* I am a kernel thread */
1585 #define PF_RANDOMIZE		0x00400000	/* Randomize virtual address space */
1586 #define PF_SWAPWRITE		0x00800000	/* Allowed to write to swap */
1587 #define PF_NO_SETAFFINITY	0x04000000	/* Userland is not allowed to meddle with cpus_mask */
1588 #define PF_MCE_EARLY		0x08000000      /* Early kill for mce process policy */
1589 #define PF_MEMALLOC_NOCMA	0x10000000	/* All allocation request will have _GFP_MOVABLE cleared */
1590 #define PF_FREEZER_SKIP		0x40000000	/* Freezer should not count it as freezable */
1591 #define PF_SUSPEND_TASK		0x80000000      /* This thread called freeze_processes() and should not be frozen */
1592 
1593 /*
1594  * Only the _current_ task can read/write to tsk->flags, but other
1595  * tasks can access tsk->flags in readonly mode for example
1596  * with tsk_used_math (like during threaded core dumping).
1597  * There is however an exception to this rule during ptrace
1598  * or during fork: the ptracer task is allowed to write to the
1599  * child->flags of its traced child (same goes for fork, the parent
1600  * can write to the child->flags), because we're guaranteed the
1601  * child is not running and in turn not changing child->flags
1602  * at the same time the parent does it.
1603  */
1604 #define clear_stopped_child_used_math(child)	do { (child)->flags &= ~PF_USED_MATH; } while (0)
1605 #define set_stopped_child_used_math(child)	do { (child)->flags |= PF_USED_MATH; } while (0)
1606 #define clear_used_math()			clear_stopped_child_used_math(current)
1607 #define set_used_math()				set_stopped_child_used_math(current)
1608 
1609 #define conditional_stopped_child_used_math(condition, child) \
1610 	do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0)
1611 
1612 #define conditional_used_math(condition)	conditional_stopped_child_used_math(condition, current)
1613 
1614 #define copy_to_stopped_child_used_math(child) \
1615 	do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0)
1616 
1617 /* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */
1618 #define tsk_used_math(p)			((p)->flags & PF_USED_MATH)
1619 #define used_math()				tsk_used_math(current)
1620 
is_percpu_thread(void)1621 static __always_inline bool is_percpu_thread(void)
1622 {
1623 #ifdef CONFIG_SMP
1624 	return (current->flags & PF_NO_SETAFFINITY) &&
1625 		(current->nr_cpus_allowed  == 1);
1626 #else
1627 	return true;
1628 #endif
1629 }
1630 
1631 /* Per-process atomic flags. */
1632 #define PFA_NO_NEW_PRIVS		0	/* May not gain new privileges. */
1633 #define PFA_SPREAD_PAGE			1	/* Spread page cache over cpuset */
1634 #define PFA_SPREAD_SLAB			2	/* Spread some slab caches over cpuset */
1635 #define PFA_SPEC_SSB_DISABLE		3	/* Speculative Store Bypass disabled */
1636 #define PFA_SPEC_SSB_FORCE_DISABLE	4	/* Speculative Store Bypass force disabled*/
1637 #define PFA_SPEC_IB_DISABLE		5	/* Indirect branch speculation restricted */
1638 #define PFA_SPEC_IB_FORCE_DISABLE	6	/* Indirect branch speculation permanently restricted */
1639 #define PFA_SPEC_SSB_NOEXEC		7	/* Speculative Store Bypass clear on execve() */
1640 
1641 #define TASK_PFA_TEST(name, func)					\
1642 	static inline bool task_##func(struct task_struct *p)		\
1643 	{ return test_bit(PFA_##name, &p->atomic_flags); }
1644 
1645 #define TASK_PFA_SET(name, func)					\
1646 	static inline void task_set_##func(struct task_struct *p)	\
1647 	{ set_bit(PFA_##name, &p->atomic_flags); }
1648 
1649 #define TASK_PFA_CLEAR(name, func)					\
1650 	static inline void task_clear_##func(struct task_struct *p)	\
1651 	{ clear_bit(PFA_##name, &p->atomic_flags); }
1652 
TASK_PFA_TEST(NO_NEW_PRIVS,no_new_privs)1653 TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs)
1654 TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs)
1655 
1656 TASK_PFA_TEST(SPREAD_PAGE, spread_page)
1657 TASK_PFA_SET(SPREAD_PAGE, spread_page)
1658 TASK_PFA_CLEAR(SPREAD_PAGE, spread_page)
1659 
1660 TASK_PFA_TEST(SPREAD_SLAB, spread_slab)
1661 TASK_PFA_SET(SPREAD_SLAB, spread_slab)
1662 TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)
1663 
1664 TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable)
1665 TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable)
1666 TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable)
1667 
1668 TASK_PFA_TEST(SPEC_SSB_NOEXEC, spec_ssb_noexec)
1669 TASK_PFA_SET(SPEC_SSB_NOEXEC, spec_ssb_noexec)
1670 TASK_PFA_CLEAR(SPEC_SSB_NOEXEC, spec_ssb_noexec)
1671 
1672 TASK_PFA_TEST(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
1673 TASK_PFA_SET(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
1674 
1675 TASK_PFA_TEST(SPEC_IB_DISABLE, spec_ib_disable)
1676 TASK_PFA_SET(SPEC_IB_DISABLE, spec_ib_disable)
1677 TASK_PFA_CLEAR(SPEC_IB_DISABLE, spec_ib_disable)
1678 
1679 TASK_PFA_TEST(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
1680 TASK_PFA_SET(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
1681 
1682 static inline void
1683 current_restore_flags(unsigned long orig_flags, unsigned long flags)
1684 {
1685 	current->flags &= ~flags;
1686 	current->flags |= orig_flags & flags;
1687 }
1688 
1689 extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
1690 
1691 #ifdef CONFIG_RT_SOFTINT_OPTIMIZATION
1692 extern bool cpupri_check_rt(void);
1693 #else
cpupri_check_rt(void)1694 static inline bool cpupri_check_rt(void)
1695 {
1696 	return false;
1697 }
1698 #endif
1699 
1700 extern int task_can_attach(struct task_struct *p);
1701 extern int dl_bw_alloc(int cpu, u64 dl_bw);
1702 extern void dl_bw_free(int cpu, u64 dl_bw);
1703 #ifdef CONFIG_SMP
1704 extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask);
1705 extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask);
1706 extern void force_compatible_cpus_allowed_ptr(struct task_struct *p);
1707 #else
do_set_cpus_allowed(struct task_struct * p,const struct cpumask * new_mask)1708 static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
1709 {
1710 }
set_cpus_allowed_ptr(struct task_struct * p,const struct cpumask * new_mask)1711 static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
1712 {
1713 	if (!cpumask_test_cpu(0, new_mask))
1714 		return -EINVAL;
1715 	return 0;
1716 }
1717 #endif
1718 
1719 extern int yield_to(struct task_struct *p, bool preempt);
1720 extern void set_user_nice(struct task_struct *p, long nice);
1721 extern int task_prio(const struct task_struct *p);
1722 
1723 /**
1724  * task_nice - return the nice value of a given task.
1725  * @p: the task in question.
1726  *
1727  * Return: The nice value [ -20 ... 0 ... 19 ].
1728  */
task_nice(const struct task_struct * p)1729 static inline int task_nice(const struct task_struct *p)
1730 {
1731 	return PRIO_TO_NICE((p)->static_prio);
1732 }
1733 
1734 extern int can_nice(const struct task_struct *p, const int nice);
1735 extern int task_curr(const struct task_struct *p);
1736 extern int idle_cpu(int cpu);
1737 extern int available_idle_cpu(int cpu);
1738 extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *);
1739 extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *);
1740 extern void sched_set_fifo(struct task_struct *p);
1741 extern void sched_set_fifo_low(struct task_struct *p);
1742 extern void sched_set_normal(struct task_struct *p, int nice);
1743 extern int sched_setattr(struct task_struct *, const struct sched_attr *);
1744 extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *);
1745 extern struct task_struct *idle_task(int cpu);
1746 
1747 /**
1748  * is_idle_task - is the specified task an idle task?
1749  * @p: the task in question.
1750  *
1751  * Return: 1 if @p is an idle task. 0 otherwise.
1752  */
is_idle_task(const struct task_struct * p)1753 static __always_inline bool is_idle_task(const struct task_struct *p)
1754 {
1755 	return !!(p->flags & PF_IDLE);
1756 }
1757 
1758 extern struct task_struct *curr_task(int cpu);
1759 extern void ia64_set_curr_task(int cpu, struct task_struct *p);
1760 
1761 void yield(void);
1762 
1763 union thread_union {
1764 #ifndef CONFIG_ARCH_TASK_STRUCT_ON_STACK
1765 	struct task_struct task;
1766 #endif
1767 #ifndef CONFIG_THREAD_INFO_IN_TASK
1768 	struct thread_info thread_info;
1769 #endif
1770 	unsigned long stack[THREAD_SIZE/sizeof(long)];
1771 };
1772 
1773 #ifndef CONFIG_THREAD_INFO_IN_TASK
1774 extern struct thread_info init_thread_info;
1775 #endif
1776 
1777 extern unsigned long init_stack[THREAD_SIZE / sizeof(unsigned long)];
1778 
1779 #ifdef CONFIG_THREAD_INFO_IN_TASK
task_thread_info(struct task_struct * task)1780 static inline struct thread_info *task_thread_info(struct task_struct *task)
1781 {
1782 	return &task->thread_info;
1783 }
1784 #elif !defined(__HAVE_THREAD_FUNCTIONS)
1785 # define task_thread_info(task)	((struct thread_info *)(task)->stack)
1786 #endif
1787 
1788 /*
1789  * find a task by one of its numerical ids
1790  *
1791  * find_task_by_pid_ns():
1792  *      finds a task by its pid in the specified namespace
1793  * find_task_by_vpid():
1794  *      finds a task by its virtual pid
1795  *
1796  * see also find_vpid() etc in include/linux/pid.h
1797  */
1798 
1799 extern struct task_struct *find_task_by_vpid(pid_t nr);
1800 extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns);
1801 
1802 /*
1803  * find a task by its virtual pid and get the task struct
1804  */
1805 extern struct task_struct *find_get_task_by_vpid(pid_t nr);
1806 
1807 extern int wake_up_state(struct task_struct *tsk, unsigned int state);
1808 extern int wake_up_process(struct task_struct *tsk);
1809 extern void wake_up_new_task(struct task_struct *tsk);
1810 
1811 #ifdef CONFIG_SMP
1812 extern void kick_process(struct task_struct *tsk);
1813 #else
kick_process(struct task_struct * tsk)1814 static inline void kick_process(struct task_struct *tsk) { }
1815 #endif
1816 
1817 extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec);
1818 
set_task_comm(struct task_struct * tsk,const char * from)1819 static inline void set_task_comm(struct task_struct *tsk, const char *from)
1820 {
1821 	__set_task_comm(tsk, from, false);
1822 }
1823 
1824 extern char *__get_task_comm(char *to, size_t len, struct task_struct *tsk);
1825 #define get_task_comm(buf, tsk) ({			\
1826 	BUILD_BUG_ON(sizeof(buf) != TASK_COMM_LEN);	\
1827 	__get_task_comm(buf, sizeof(buf), tsk);		\
1828 })
1829 
1830 #ifdef CONFIG_SMP
scheduler_ipi(void)1831 static __always_inline void scheduler_ipi(void)
1832 {
1833 	/*
1834 	 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
1835 	 * TIF_NEED_RESCHED remotely (for the first time) will also send
1836 	 * this IPI.
1837 	 */
1838 	preempt_fold_need_resched();
1839 }
1840 extern unsigned long wait_task_inactive(struct task_struct *, long match_state);
1841 #else
scheduler_ipi(void)1842 static inline void scheduler_ipi(void) { }
wait_task_inactive(struct task_struct * p,long match_state)1843 static inline unsigned long wait_task_inactive(struct task_struct *p, long match_state)
1844 {
1845 	return 1;
1846 }
1847 #endif
1848 
1849 /*
1850  * Set thread flags in other task's structures.
1851  * See asm/thread_info.h for TIF_xxxx flags available:
1852  */
set_tsk_thread_flag(struct task_struct * tsk,int flag)1853 static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag)
1854 {
1855 	set_ti_thread_flag(task_thread_info(tsk), flag);
1856 }
1857 
clear_tsk_thread_flag(struct task_struct * tsk,int flag)1858 static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag)
1859 {
1860 	clear_ti_thread_flag(task_thread_info(tsk), flag);
1861 }
1862 
update_tsk_thread_flag(struct task_struct * tsk,int flag,bool value)1863 static inline void update_tsk_thread_flag(struct task_struct *tsk, int flag,
1864 					  bool value)
1865 {
1866 	update_ti_thread_flag(task_thread_info(tsk), flag, value);
1867 }
1868 
test_and_set_tsk_thread_flag(struct task_struct * tsk,int flag)1869 static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
1870 {
1871 	return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
1872 }
1873 
test_and_clear_tsk_thread_flag(struct task_struct * tsk,int flag)1874 static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
1875 {
1876 	return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
1877 }
1878 
test_tsk_thread_flag(struct task_struct * tsk,int flag)1879 static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
1880 {
1881 	return test_ti_thread_flag(task_thread_info(tsk), flag);
1882 }
1883 
set_tsk_need_resched(struct task_struct * tsk)1884 static inline void set_tsk_need_resched(struct task_struct *tsk)
1885 {
1886 	set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
1887 }
1888 
clear_tsk_need_resched(struct task_struct * tsk)1889 static inline void clear_tsk_need_resched(struct task_struct *tsk)
1890 {
1891 	clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
1892 }
1893 
test_tsk_need_resched(struct task_struct * tsk)1894 static inline int test_tsk_need_resched(struct task_struct *tsk)
1895 {
1896 	return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
1897 }
1898 
1899 /*
1900  * cond_resched() and cond_resched_lock(): latency reduction via
1901  * explicit rescheduling in places that are safe. The return
1902  * value indicates whether a reschedule was done in fact.
1903  * cond_resched_lock() will drop the spinlock before scheduling,
1904  */
1905 #ifndef CONFIG_PREEMPTION
1906 extern int _cond_resched(void);
1907 #else
_cond_resched(void)1908 static inline int _cond_resched(void) { return 0; }
1909 #endif
1910 
1911 #define cond_resched() ({			\
1912 	___might_sleep(__FILE__, __LINE__, 0);	\
1913 	_cond_resched();			\
1914 })
1915 
1916 extern int __cond_resched_lock(spinlock_t *lock);
1917 
1918 #define cond_resched_lock(lock) ({				\
1919 	___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\
1920 	__cond_resched_lock(lock);				\
1921 })
1922 
cond_resched_rcu(void)1923 static inline void cond_resched_rcu(void)
1924 {
1925 #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU)
1926 	rcu_read_unlock();
1927 	cond_resched();
1928 	rcu_read_lock();
1929 #endif
1930 }
1931 
1932 /*
1933  * Does a critical section need to be broken due to another
1934  * task waiting?: (technically does not depend on CONFIG_PREEMPTION,
1935  * but a general need for low latency)
1936  */
spin_needbreak(spinlock_t * lock)1937 static inline int spin_needbreak(spinlock_t *lock)
1938 {
1939 #ifdef CONFIG_PREEMPTION
1940 	return spin_is_contended(lock);
1941 #else
1942 	return 0;
1943 #endif
1944 }
1945 
need_resched(void)1946 static __always_inline bool need_resched(void)
1947 {
1948 	return unlikely(tif_need_resched());
1949 }
1950 
1951 /*
1952  * Wrappers for p->thread_info->cpu access. No-op on UP.
1953  */
1954 #ifdef CONFIG_SMP
1955 
task_cpu(const struct task_struct * p)1956 static inline unsigned int task_cpu(const struct task_struct *p)
1957 {
1958 #ifdef CONFIG_THREAD_INFO_IN_TASK
1959 	return READ_ONCE(p->cpu);
1960 #else
1961 	return READ_ONCE(task_thread_info(p)->cpu);
1962 #endif
1963 }
1964 
1965 extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
1966 
1967 #else
1968 
task_cpu(const struct task_struct * p)1969 static inline unsigned int task_cpu(const struct task_struct *p)
1970 {
1971 	return 0;
1972 }
1973 
set_task_cpu(struct task_struct * p,unsigned int cpu)1974 static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
1975 {
1976 }
1977 
1978 #endif /* CONFIG_SMP */
1979 
1980 /*
1981  * In order to reduce various lock holder preemption latencies provide an
1982  * interface to see if a vCPU is currently running or not.
1983  *
1984  * This allows us to terminate optimistic spin loops and block, analogous to
1985  * the native optimistic spin heuristic of testing if the lock owner task is
1986  * running or not.
1987  */
1988 #ifndef vcpu_is_preempted
vcpu_is_preempted(int cpu)1989 static inline bool vcpu_is_preempted(int cpu)
1990 {
1991 	return false;
1992 }
1993 #endif
1994 
1995 extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
1996 extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
1997 
1998 #ifndef TASK_SIZE_OF
1999 #define TASK_SIZE_OF(tsk)	TASK_SIZE
2000 #endif
2001 
2002 #ifdef CONFIG_RSEQ
2003 
2004 /*
2005  * Map the event mask on the user-space ABI enum rseq_cs_flags
2006  * for direct mask checks.
2007  */
2008 enum rseq_event_mask_bits {
2009 	RSEQ_EVENT_PREEMPT_BIT	= RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT,
2010 	RSEQ_EVENT_SIGNAL_BIT	= RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT,
2011 	RSEQ_EVENT_MIGRATE_BIT	= RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT,
2012 };
2013 
2014 enum rseq_event_mask {
2015 	RSEQ_EVENT_PREEMPT	= (1U << RSEQ_EVENT_PREEMPT_BIT),
2016 	RSEQ_EVENT_SIGNAL	= (1U << RSEQ_EVENT_SIGNAL_BIT),
2017 	RSEQ_EVENT_MIGRATE	= (1U << RSEQ_EVENT_MIGRATE_BIT),
2018 };
2019 
rseq_set_notify_resume(struct task_struct * t)2020 static inline void rseq_set_notify_resume(struct task_struct *t)
2021 {
2022 	if (t->rseq)
2023 		set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);
2024 }
2025 
2026 void __rseq_handle_notify_resume(struct ksignal *sig, struct pt_regs *regs);
2027 
rseq_handle_notify_resume(struct ksignal * ksig,struct pt_regs * regs)2028 static inline void rseq_handle_notify_resume(struct ksignal *ksig,
2029 					     struct pt_regs *regs)
2030 {
2031 	if (current->rseq)
2032 		__rseq_handle_notify_resume(ksig, regs);
2033 }
2034 
rseq_signal_deliver(struct ksignal * ksig,struct pt_regs * regs)2035 static inline void rseq_signal_deliver(struct ksignal *ksig,
2036 				       struct pt_regs *regs)
2037 {
2038 	preempt_disable();
2039 	__set_bit(RSEQ_EVENT_SIGNAL_BIT, &current->rseq_event_mask);
2040 	preempt_enable();
2041 	rseq_handle_notify_resume(ksig, regs);
2042 }
2043 
2044 /* rseq_preempt() requires preemption to be disabled. */
rseq_preempt(struct task_struct * t)2045 static inline void rseq_preempt(struct task_struct *t)
2046 {
2047 	__set_bit(RSEQ_EVENT_PREEMPT_BIT, &t->rseq_event_mask);
2048 	rseq_set_notify_resume(t);
2049 }
2050 
2051 /* rseq_migrate() requires preemption to be disabled. */
rseq_migrate(struct task_struct * t)2052 static inline void rseq_migrate(struct task_struct *t)
2053 {
2054 	__set_bit(RSEQ_EVENT_MIGRATE_BIT, &t->rseq_event_mask);
2055 	rseq_set_notify_resume(t);
2056 }
2057 
2058 /*
2059  * If parent process has a registered restartable sequences area, the
2060  * child inherits. Unregister rseq for a clone with CLONE_VM set.
2061  */
rseq_fork(struct task_struct * t,unsigned long clone_flags)2062 static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
2063 {
2064 	if (clone_flags & CLONE_VM) {
2065 		t->rseq = NULL;
2066 		t->rseq_sig = 0;
2067 		t->rseq_event_mask = 0;
2068 	} else {
2069 		t->rseq = current->rseq;
2070 		t->rseq_sig = current->rseq_sig;
2071 		t->rseq_event_mask = current->rseq_event_mask;
2072 	}
2073 }
2074 
rseq_execve(struct task_struct * t)2075 static inline void rseq_execve(struct task_struct *t)
2076 {
2077 	t->rseq = NULL;
2078 	t->rseq_sig = 0;
2079 	t->rseq_event_mask = 0;
2080 }
2081 
2082 #else
2083 
rseq_set_notify_resume(struct task_struct * t)2084 static inline void rseq_set_notify_resume(struct task_struct *t)
2085 {
2086 }
rseq_handle_notify_resume(struct ksignal * ksig,struct pt_regs * regs)2087 static inline void rseq_handle_notify_resume(struct ksignal *ksig,
2088 					     struct pt_regs *regs)
2089 {
2090 }
rseq_signal_deliver(struct ksignal * ksig,struct pt_regs * regs)2091 static inline void rseq_signal_deliver(struct ksignal *ksig,
2092 				       struct pt_regs *regs)
2093 {
2094 }
rseq_preempt(struct task_struct * t)2095 static inline void rseq_preempt(struct task_struct *t)
2096 {
2097 }
rseq_migrate(struct task_struct * t)2098 static inline void rseq_migrate(struct task_struct *t)
2099 {
2100 }
rseq_fork(struct task_struct * t,unsigned long clone_flags)2101 static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
2102 {
2103 }
rseq_execve(struct task_struct * t)2104 static inline void rseq_execve(struct task_struct *t)
2105 {
2106 }
2107 
2108 #endif
2109 
2110 #ifdef CONFIG_DEBUG_RSEQ
2111 
2112 void rseq_syscall(struct pt_regs *regs);
2113 
2114 #else
2115 
rseq_syscall(struct pt_regs * regs)2116 static inline void rseq_syscall(struct pt_regs *regs)
2117 {
2118 }
2119 
2120 #endif
2121 
2122 const struct sched_avg *sched_trace_cfs_rq_avg(struct cfs_rq *cfs_rq);
2123 char *sched_trace_cfs_rq_path(struct cfs_rq *cfs_rq, char *str, int len);
2124 int sched_trace_cfs_rq_cpu(struct cfs_rq *cfs_rq);
2125 
2126 const struct sched_avg *sched_trace_rq_avg_rt(struct rq *rq);
2127 const struct sched_avg *sched_trace_rq_avg_dl(struct rq *rq);
2128 const struct sched_avg *sched_trace_rq_avg_irq(struct rq *rq);
2129 
2130 int sched_trace_rq_cpu(struct rq *rq);
2131 int sched_trace_rq_cpu_capacity(struct rq *rq);
2132 int sched_trace_rq_nr_running(struct rq *rq);
2133 
2134 const struct cpumask *sched_trace_rd_span(struct root_domain *rd);
2135 
2136 #endif
2137