• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1  /* SPDX-License-Identifier: GPL-2.0 */
2  #ifndef _LINUX_SCHED_H
3  #define _LINUX_SCHED_H
4  
5  /*
6   * Define 'struct task_struct' and provide the main scheduler
7   * APIs (schedule(), wakeup variants, etc.)
8   */
9  
10  #include <uapi/linux/sched.h>
11  
12  #include <asm/current.h>
13  
14  #include <linux/pid.h>
15  #include <linux/sem.h>
16  #include <linux/shm.h>
17  #include <linux/mutex.h>
18  #include <linux/plist.h>
19  #include <linux/hrtimer.h>
20  #include <linux/irqflags.h>
21  #include <linux/seccomp.h>
22  #include <linux/nodemask.h>
23  #include <linux/rcupdate.h>
24  #include <linux/refcount.h>
25  #include <linux/resource.h>
26  #include <linux/latencytop.h>
27  #include <linux/sched/prio.h>
28  #include <linux/sched/types.h>
29  #include <linux/signal_types.h>
30  #include <linux/syscall_user_dispatch.h>
31  #include <linux/mm_types_task.h>
32  #include <linux/task_io_accounting.h>
33  #include <linux/posix-timers.h>
34  #include <linux/rseq.h>
35  #include <linux/seqlock.h>
36  #include <linux/kcsan.h>
37  #include <linux/android_vendor.h>
38  #include <asm/kmap_size.h>
39  #include <linux/android_kabi.h>
40  
41  /* task_struct member predeclarations (sorted alphabetically): */
42  struct audit_context;
43  struct backing_dev_info;
44  struct bio_list;
45  struct blk_plug;
46  struct bpf_local_storage;
47  struct bpf_run_ctx;
48  struct capture_control;
49  struct cfs_rq;
50  struct fs_struct;
51  struct futex_pi_state;
52  struct io_context;
53  struct io_uring_task;
54  struct mempolicy;
55  struct nameidata;
56  struct nsproxy;
57  struct perf_event_context;
58  struct pid_namespace;
59  struct pipe_inode_info;
60  struct rcu_node;
61  struct reclaim_state;
62  struct robust_list_head;
63  struct root_domain;
64  struct rq;
65  struct sched_attr;
66  struct sched_param;
67  struct seq_file;
68  struct sighand_struct;
69  struct signal_struct;
70  struct task_delay_info;
71  struct task_group;
72  
73  /*
74   * Task state bitmask. NOTE! These bits are also
75   * encoded in fs/proc/array.c: get_task_state().
76   *
77   * We have two separate sets of flags: task->state
78   * is about runnability, while task->exit_state are
79   * about the task exiting. Confusing, but this way
80   * modifying one set can't modify the other one by
81   * mistake.
82   */
83  
84  /* Used in tsk->state: */
85  #define TASK_RUNNING			0x0000
86  #define TASK_INTERRUPTIBLE		0x0001
87  #define TASK_UNINTERRUPTIBLE		0x0002
88  #define __TASK_STOPPED			0x0004
89  #define __TASK_TRACED			0x0008
90  /* Used in tsk->exit_state: */
91  #define EXIT_DEAD			0x0010
92  #define EXIT_ZOMBIE			0x0020
93  #define EXIT_TRACE			(EXIT_ZOMBIE | EXIT_DEAD)
94  /* Used in tsk->state again: */
95  #define TASK_PARKED			0x0040
96  #define TASK_DEAD			0x0080
97  #define TASK_WAKEKILL			0x0100
98  #define TASK_WAKING			0x0200
99  #define TASK_NOLOAD			0x0400
100  #define TASK_NEW			0x0800
101  /* RT specific auxilliary flag to mark RT lock waiters */
102  #define TASK_RTLOCK_WAIT		0x1000
103  #define TASK_STATE_MAX			0x2000
104  
105  /* Convenience macros for the sake of set_current_state: */
106  #define TASK_KILLABLE			(TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
107  #define TASK_STOPPED			(TASK_WAKEKILL | __TASK_STOPPED)
108  #define TASK_TRACED			(TASK_WAKEKILL | __TASK_TRACED)
109  
110  #define TASK_IDLE			(TASK_UNINTERRUPTIBLE | TASK_NOLOAD)
111  
112  /* Convenience macros for the sake of wake_up(): */
113  #define TASK_NORMAL			(TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
114  
115  /* get_task_state(): */
116  #define TASK_REPORT			(TASK_RUNNING | TASK_INTERRUPTIBLE | \
117  					 TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
118  					 __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \
119  					 TASK_PARKED)
120  
121  #define task_is_running(task)		(READ_ONCE((task)->__state) == TASK_RUNNING)
122  
123  #define task_is_traced(task)		((READ_ONCE(task->__state) & __TASK_TRACED) != 0)
124  
125  #define task_is_stopped(task)		((READ_ONCE(task->__state) & __TASK_STOPPED) != 0)
126  
127  #define task_is_stopped_or_traced(task)	((READ_ONCE(task->__state) & (__TASK_STOPPED | __TASK_TRACED)) != 0)
128  
129  /*
130   * Special states are those that do not use the normal wait-loop pattern. See
131   * the comment with set_special_state().
132   */
133  #define is_special_task_state(state)				\
134  	((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | TASK_DEAD))
135  
136  #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
137  # define debug_normal_state_change(state_value)				\
138  	do {								\
139  		WARN_ON_ONCE(is_special_task_state(state_value));	\
140  		current->task_state_change = _THIS_IP_;			\
141  	} while (0)
142  
143  # define debug_special_state_change(state_value)			\
144  	do {								\
145  		WARN_ON_ONCE(!is_special_task_state(state_value));	\
146  		current->task_state_change = _THIS_IP_;			\
147  	} while (0)
148  
149  # define debug_rtlock_wait_set_state()					\
150  	do {								 \
151  		current->saved_state_change = current->task_state_change;\
152  		current->task_state_change = _THIS_IP_;			 \
153  	} while (0)
154  
155  # define debug_rtlock_wait_restore_state()				\
156  	do {								 \
157  		current->task_state_change = current->saved_state_change;\
158  	} while (0)
159  
160  #else
161  # define debug_normal_state_change(cond)	do { } while (0)
162  # define debug_special_state_change(cond)	do { } while (0)
163  # define debug_rtlock_wait_set_state()		do { } while (0)
164  # define debug_rtlock_wait_restore_state()	do { } while (0)
165  #endif
166  
167  /*
168   * set_current_state() includes a barrier so that the write of current->state
169   * is correctly serialised wrt the caller's subsequent test of whether to
170   * actually sleep:
171   *
172   *   for (;;) {
173   *	set_current_state(TASK_UNINTERRUPTIBLE);
174   *	if (CONDITION)
175   *	   break;
176   *
177   *	schedule();
178   *   }
179   *   __set_current_state(TASK_RUNNING);
180   *
181   * If the caller does not need such serialisation (because, for instance, the
182   * CONDITION test and condition change and wakeup are under the same lock) then
183   * use __set_current_state().
184   *
185   * The above is typically ordered against the wakeup, which does:
186   *
187   *   CONDITION = 1;
188   *   wake_up_state(p, TASK_UNINTERRUPTIBLE);
189   *
190   * where wake_up_state()/try_to_wake_up() executes a full memory barrier before
191   * accessing p->state.
192   *
193   * Wakeup will do: if (@state & p->state) p->state = TASK_RUNNING, that is,
194   * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a
195   * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING).
196   *
197   * However, with slightly different timing the wakeup TASK_RUNNING store can
198   * also collide with the TASK_UNINTERRUPTIBLE store. Losing that store is not
199   * a problem either because that will result in one extra go around the loop
200   * and our @cond test will save the day.
201   *
202   * Also see the comments of try_to_wake_up().
203   */
204  #define __set_current_state(state_value)				\
205  	do {								\
206  		debug_normal_state_change((state_value));		\
207  		WRITE_ONCE(current->__state, (state_value));		\
208  	} while (0)
209  
210  #define set_current_state(state_value)					\
211  	do {								\
212  		debug_normal_state_change((state_value));		\
213  		smp_store_mb(current->__state, (state_value));		\
214  	} while (0)
215  
216  /*
217   * set_special_state() should be used for those states when the blocking task
218   * can not use the regular condition based wait-loop. In that case we must
219   * serialize against wakeups such that any possible in-flight TASK_RUNNING
220   * stores will not collide with our state change.
221   */
222  #define set_special_state(state_value)					\
223  	do {								\
224  		unsigned long flags; /* may shadow */			\
225  									\
226  		raw_spin_lock_irqsave(&current->pi_lock, flags);	\
227  		debug_special_state_change((state_value));		\
228  		WRITE_ONCE(current->__state, (state_value));		\
229  		raw_spin_unlock_irqrestore(&current->pi_lock, flags);	\
230  	} while (0)
231  
232  /*
233   * PREEMPT_RT specific variants for "sleeping" spin/rwlocks
234   *
235   * RT's spin/rwlock substitutions are state preserving. The state of the
236   * task when blocking on the lock is saved in task_struct::saved_state and
237   * restored after the lock has been acquired.  These operations are
238   * serialized by task_struct::pi_lock against try_to_wake_up(). Any non RT
239   * lock related wakeups while the task is blocked on the lock are
240   * redirected to operate on task_struct::saved_state to ensure that these
241   * are not dropped. On restore task_struct::saved_state is set to
242   * TASK_RUNNING so any wakeup attempt redirected to saved_state will fail.
243   *
244   * The lock operation looks like this:
245   *
246   *	current_save_and_set_rtlock_wait_state();
247   *	for (;;) {
248   *		if (try_lock())
249   *			break;
250   *		raw_spin_unlock_irq(&lock->wait_lock);
251   *		schedule_rtlock();
252   *		raw_spin_lock_irq(&lock->wait_lock);
253   *		set_current_state(TASK_RTLOCK_WAIT);
254   *	}
255   *	current_restore_rtlock_saved_state();
256   */
257  #define current_save_and_set_rtlock_wait_state()			\
258  	do {								\
259  		lockdep_assert_irqs_disabled();				\
260  		raw_spin_lock(&current->pi_lock);			\
261  		current->saved_state = current->__state;		\
262  		debug_rtlock_wait_set_state();				\
263  		WRITE_ONCE(current->__state, TASK_RTLOCK_WAIT);		\
264  		raw_spin_unlock(&current->pi_lock);			\
265  	} while (0);
266  
267  #define current_restore_rtlock_saved_state()				\
268  	do {								\
269  		lockdep_assert_irqs_disabled();				\
270  		raw_spin_lock(&current->pi_lock);			\
271  		debug_rtlock_wait_restore_state();			\
272  		WRITE_ONCE(current->__state, current->saved_state);	\
273  		current->saved_state = TASK_RUNNING;			\
274  		raw_spin_unlock(&current->pi_lock);			\
275  	} while (0);
276  
277  #define get_current_state()	READ_ONCE(current->__state)
278  
279  /* Task command name length: */
280  #define TASK_COMM_LEN			16
281  
282  extern void scheduler_tick(void);
283  
284  #define	MAX_SCHEDULE_TIMEOUT		LONG_MAX
285  
286  extern long schedule_timeout(long timeout);
287  extern long schedule_timeout_interruptible(long timeout);
288  extern long schedule_timeout_killable(long timeout);
289  extern long schedule_timeout_uninterruptible(long timeout);
290  extern long schedule_timeout_idle(long timeout);
291  asmlinkage void schedule(void);
292  extern void schedule_preempt_disabled(void);
293  asmlinkage void preempt_schedule_irq(void);
294  #ifdef CONFIG_PREEMPT_RT
295   extern void schedule_rtlock(void);
296  #endif
297  
298  extern int __must_check io_schedule_prepare(void);
299  extern void io_schedule_finish(int token);
300  extern long io_schedule_timeout(long timeout);
301  extern void io_schedule(void);
302  extern struct task_struct *pick_migrate_task(struct rq *rq);
303  extern int select_fallback_rq(int cpu, struct task_struct *p);
304  
305  /**
306   * struct prev_cputime - snapshot of system and user cputime
307   * @utime: time spent in user mode
308   * @stime: time spent in system mode
309   * @lock: protects the above two fields
310   *
311   * Stores previous user/system time values such that we can guarantee
312   * monotonicity.
313   */
314  struct prev_cputime {
315  #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
316  	u64				utime;
317  	u64				stime;
318  	raw_spinlock_t			lock;
319  #endif
320  };
321  
322  enum vtime_state {
323  	/* Task is sleeping or running in a CPU with VTIME inactive: */
324  	VTIME_INACTIVE = 0,
325  	/* Task is idle */
326  	VTIME_IDLE,
327  	/* Task runs in kernelspace in a CPU with VTIME active: */
328  	VTIME_SYS,
329  	/* Task runs in userspace in a CPU with VTIME active: */
330  	VTIME_USER,
331  	/* Task runs as guests in a CPU with VTIME active: */
332  	VTIME_GUEST,
333  };
334  
335  struct vtime {
336  	seqcount_t		seqcount;
337  	unsigned long long	starttime;
338  	enum vtime_state	state;
339  	unsigned int		cpu;
340  	u64			utime;
341  	u64			stime;
342  	u64			gtime;
343  };
344  
345  /*
346   * Utilization clamp constraints.
347   * @UCLAMP_MIN:	Minimum utilization
348   * @UCLAMP_MAX:	Maximum utilization
349   * @UCLAMP_CNT:	Utilization clamp constraints count
350   */
351  enum uclamp_id {
352  	UCLAMP_MIN = 0,
353  	UCLAMP_MAX,
354  	UCLAMP_CNT
355  };
356  
357  #ifdef CONFIG_SMP
358  extern struct root_domain def_root_domain;
359  extern struct mutex sched_domains_mutex;
360  #endif
361  
362  struct sched_info {
363  #ifdef CONFIG_SCHED_INFO
364  	/* Cumulative counters: */
365  
366  	/* # of times we have run on this CPU: */
367  	unsigned long			pcount;
368  
369  	/* Time spent waiting on a runqueue: */
370  	unsigned long long		run_delay;
371  
372  	/* Timestamps: */
373  
374  	/* When did we last run on a CPU? */
375  	unsigned long long		last_arrival;
376  
377  	/* When were we last queued to run? */
378  	unsigned long long		last_queued;
379  
380  #endif /* CONFIG_SCHED_INFO */
381  };
382  
383  /*
384   * Integer metrics need fixed point arithmetic, e.g., sched/fair
385   * has a few: load, load_avg, util_avg, freq, and capacity.
386   *
387   * We define a basic fixed point arithmetic range, and then formalize
388   * all these metrics based on that basic range.
389   */
390  # define SCHED_FIXEDPOINT_SHIFT		10
391  # define SCHED_FIXEDPOINT_SCALE		(1L << SCHED_FIXEDPOINT_SHIFT)
392  
393  /* Increase resolution of cpu_capacity calculations */
394  # define SCHED_CAPACITY_SHIFT		SCHED_FIXEDPOINT_SHIFT
395  # define SCHED_CAPACITY_SCALE		(1L << SCHED_CAPACITY_SHIFT)
396  
397  struct load_weight {
398  	unsigned long			weight;
399  	u32				inv_weight;
400  };
401  
402  /**
403   * struct util_est - Estimation utilization of FAIR tasks
404   * @enqueued: instantaneous estimated utilization of a task/cpu
405   * @ewma:     the Exponential Weighted Moving Average (EWMA)
406   *            utilization of a task
407   *
408   * Support data structure to track an Exponential Weighted Moving Average
409   * (EWMA) of a FAIR task's utilization. New samples are added to the moving
410   * average each time a task completes an activation. Sample's weight is chosen
411   * so that the EWMA will be relatively insensitive to transient changes to the
412   * task's workload.
413   *
414   * The enqueued attribute has a slightly different meaning for tasks and cpus:
415   * - task:   the task's util_avg at last task dequeue time
416   * - cfs_rq: the sum of util_est.enqueued for each RUNNABLE task on that CPU
417   * Thus, the util_est.enqueued of a task represents the contribution on the
418   * estimated utilization of the CPU where that task is currently enqueued.
419   *
420   * Only for tasks we track a moving average of the past instantaneous
421   * estimated utilization. This allows to absorb sporadic drops in utilization
422   * of an otherwise almost periodic task.
423   *
424   * The UTIL_AVG_UNCHANGED flag is used to synchronize util_est with util_avg
425   * updates. When a task is dequeued, its util_est should not be updated if its
426   * util_avg has not been updated in the meantime.
427   * This information is mapped into the MSB bit of util_est.enqueued at dequeue
428   * time. Since max value of util_est.enqueued for a task is 1024 (PELT util_avg
429   * for a task) it is safe to use MSB.
430   */
431  struct util_est {
432  	unsigned int			enqueued;
433  	unsigned int			ewma;
434  #define UTIL_EST_WEIGHT_SHIFT		2
435  #define UTIL_AVG_UNCHANGED		0x80000000
436  } __attribute__((__aligned__(sizeof(u64))));
437  
438  /*
439   * The load/runnable/util_avg accumulates an infinite geometric series
440   * (see __update_load_avg_cfs_rq() in kernel/sched/pelt.c).
441   *
442   * [load_avg definition]
443   *
444   *   load_avg = runnable% * scale_load_down(load)
445   *
446   * [runnable_avg definition]
447   *
448   *   runnable_avg = runnable% * SCHED_CAPACITY_SCALE
449   *
450   * [util_avg definition]
451   *
452   *   util_avg = running% * SCHED_CAPACITY_SCALE
453   *
454   * where runnable% is the time ratio that a sched_entity is runnable and
455   * running% the time ratio that a sched_entity is running.
456   *
457   * For cfs_rq, they are the aggregated values of all runnable and blocked
458   * sched_entities.
459   *
460   * The load/runnable/util_avg doesn't directly factor frequency scaling and CPU
461   * capacity scaling. The scaling is done through the rq_clock_pelt that is used
462   * for computing those signals (see update_rq_clock_pelt())
463   *
464   * N.B., the above ratios (runnable% and running%) themselves are in the
465   * range of [0, 1]. To do fixed point arithmetics, we therefore scale them
466   * to as large a range as necessary. This is for example reflected by
467   * util_avg's SCHED_CAPACITY_SCALE.
468   *
469   * [Overflow issue]
470   *
471   * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities
472   * with the highest load (=88761), always runnable on a single cfs_rq,
473   * and should not overflow as the number already hits PID_MAX_LIMIT.
474   *
475   * For all other cases (including 32-bit kernels), struct load_weight's
476   * weight will overflow first before we do, because:
477   *
478   *    Max(load_avg) <= Max(load.weight)
479   *
480   * Then it is the load_weight's responsibility to consider overflow
481   * issues.
482   */
483  struct sched_avg {
484  	u64				last_update_time;
485  	u64				load_sum;
486  	u64				runnable_sum;
487  	u32				util_sum;
488  	u32				period_contrib;
489  	unsigned long			load_avg;
490  	unsigned long			runnable_avg;
491  	unsigned long			util_avg;
492  	struct util_est			util_est;
493  } ____cacheline_aligned;
494  
495  struct sched_statistics {
496  #ifdef CONFIG_SCHEDSTATS
497  	u64				wait_start;
498  	u64				wait_max;
499  	u64				wait_count;
500  	u64				wait_sum;
501  	u64				iowait_count;
502  	u64				iowait_sum;
503  
504  	u64				sleep_start;
505  	u64				sleep_max;
506  	s64				sum_sleep_runtime;
507  
508  	u64				block_start;
509  	u64				block_max;
510  	u64				exec_max;
511  	u64				slice_max;
512  
513  	u64				nr_migrations_cold;
514  	u64				nr_failed_migrations_affine;
515  	u64				nr_failed_migrations_running;
516  	u64				nr_failed_migrations_hot;
517  	u64				nr_forced_migrations;
518  
519  	u64				nr_wakeups;
520  	u64				nr_wakeups_sync;
521  	u64				nr_wakeups_migrate;
522  	u64				nr_wakeups_local;
523  	u64				nr_wakeups_remote;
524  	u64				nr_wakeups_affine;
525  	u64				nr_wakeups_affine_attempts;
526  	u64				nr_wakeups_passive;
527  	u64				nr_wakeups_idle;
528  #endif
529  };
530  
531  struct sched_entity {
532  	/* For load-balancing: */
533  	struct load_weight		load;
534  	struct rb_node			run_node;
535  	struct list_head		group_node;
536  	unsigned int			on_rq;
537  
538  	u64				exec_start;
539  	u64				sum_exec_runtime;
540  	u64				vruntime;
541  	u64				prev_sum_exec_runtime;
542  
543  	u64				nr_migrations;
544  
545  	struct sched_statistics		statistics;
546  
547  #ifdef CONFIG_FAIR_GROUP_SCHED
548  	int				depth;
549  	struct sched_entity		*parent;
550  	/* rq on which this entity is (to be) queued: */
551  	struct cfs_rq			*cfs_rq;
552  	/* rq "owned" by this entity/group: */
553  	struct cfs_rq			*my_q;
554  	/* cached value of my_q->h_nr_running */
555  	unsigned long			runnable_weight;
556  #endif
557  
558  #ifdef CONFIG_SMP
559  	/*
560  	 * Per entity load average tracking.
561  	 *
562  	 * Put into separate cache line so it does not
563  	 * collide with read-mostly values above.
564  	 */
565  	struct sched_avg		avg;
566  #endif
567  
568  	ANDROID_KABI_RESERVE(1);
569  	ANDROID_KABI_RESERVE(2);
570  	ANDROID_KABI_RESERVE(3);
571  	ANDROID_KABI_RESERVE(4);
572  };
573  
574  struct sched_rt_entity {
575  	struct list_head		run_list;
576  	unsigned long			timeout;
577  	unsigned long			watchdog_stamp;
578  	unsigned int			time_slice;
579  	unsigned short			on_rq;
580  	unsigned short			on_list;
581  
582  	struct sched_rt_entity		*back;
583  #ifdef CONFIG_RT_GROUP_SCHED
584  	struct sched_rt_entity		*parent;
585  	/* rq on which this entity is (to be) queued: */
586  	struct rt_rq			*rt_rq;
587  	/* rq "owned" by this entity/group: */
588  	struct rt_rq			*my_q;
589  #endif
590  
591  	ANDROID_KABI_RESERVE(1);
592  	ANDROID_KABI_RESERVE(2);
593  	ANDROID_KABI_RESERVE(3);
594  	ANDROID_KABI_RESERVE(4);
595  } __randomize_layout;
596  
597  struct sched_dl_entity {
598  	struct rb_node			rb_node;
599  
600  	/*
601  	 * Original scheduling parameters. Copied here from sched_attr
602  	 * during sched_setattr(), they will remain the same until
603  	 * the next sched_setattr().
604  	 */
605  	u64				dl_runtime;	/* Maximum runtime for each instance	*/
606  	u64				dl_deadline;	/* Relative deadline of each instance	*/
607  	u64				dl_period;	/* Separation of two instances (period) */
608  	u64				dl_bw;		/* dl_runtime / dl_period		*/
609  	u64				dl_density;	/* dl_runtime / dl_deadline		*/
610  
611  	/*
612  	 * Actual scheduling parameters. Initialized with the values above,
613  	 * they are continuously updated during task execution. Note that
614  	 * the remaining runtime could be < 0 in case we are in overrun.
615  	 */
616  	s64				runtime;	/* Remaining runtime for this instance	*/
617  	u64				deadline;	/* Absolute deadline for this instance	*/
618  	unsigned int			flags;		/* Specifying the scheduler behaviour	*/
619  
620  	/*
621  	 * Some bool flags:
622  	 *
623  	 * @dl_throttled tells if we exhausted the runtime. If so, the
624  	 * task has to wait for a replenishment to be performed at the
625  	 * next firing of dl_timer.
626  	 *
627  	 * @dl_boosted tells if we are boosted due to DI. If so we are
628  	 * outside bandwidth enforcement mechanism (but only until we
629  	 * exit the critical section);
630  	 *
631  	 * @dl_yielded tells if task gave up the CPU before consuming
632  	 * all its available runtime during the last job.
633  	 *
634  	 * @dl_non_contending tells if the task is inactive while still
635  	 * contributing to the active utilization. In other words, it
636  	 * indicates if the inactive timer has been armed and its handler
637  	 * has not been executed yet. This flag is useful to avoid race
638  	 * conditions between the inactive timer handler and the wakeup
639  	 * code.
640  	 *
641  	 * @dl_overrun tells if the task asked to be informed about runtime
642  	 * overruns.
643  	 */
644  	unsigned int			dl_throttled      : 1;
645  	unsigned int			dl_yielded        : 1;
646  	unsigned int			dl_non_contending : 1;
647  	unsigned int			dl_overrun	  : 1;
648  
649  	/*
650  	 * Bandwidth enforcement timer. Each -deadline task has its
651  	 * own bandwidth to be enforced, thus we need one timer per task.
652  	 */
653  	struct hrtimer			dl_timer;
654  
655  	/*
656  	 * Inactive timer, responsible for decreasing the active utilization
657  	 * at the "0-lag time". When a -deadline task blocks, it contributes
658  	 * to GRUB's active utilization until the "0-lag time", hence a
659  	 * timer is needed to decrease the active utilization at the correct
660  	 * time.
661  	 */
662  	struct hrtimer inactive_timer;
663  
664  #ifdef CONFIG_RT_MUTEXES
665  	/*
666  	 * Priority Inheritance. When a DEADLINE scheduling entity is boosted
667  	 * pi_se points to the donor, otherwise points to the dl_se it belongs
668  	 * to (the original one/itself).
669  	 */
670  	struct sched_dl_entity *pi_se;
671  #endif
672  };
673  
674  #ifdef CONFIG_UCLAMP_TASK
675  /* Number of utilization clamp buckets (shorter alias) */
676  #define UCLAMP_BUCKETS CONFIG_UCLAMP_BUCKETS_COUNT
677  
678  /*
679   * Utilization clamp for a scheduling entity
680   * @value:		clamp value "assigned" to a se
681   * @bucket_id:		bucket index corresponding to the "assigned" value
682   * @active:		the se is currently refcounted in a rq's bucket
683   * @user_defined:	the requested clamp value comes from user-space
684   *
685   * The bucket_id is the index of the clamp bucket matching the clamp value
686   * which is pre-computed and stored to avoid expensive integer divisions from
687   * the fast path.
688   *
689   * The active bit is set whenever a task has got an "effective" value assigned,
690   * which can be different from the clamp value "requested" from user-space.
691   * This allows to know a task is refcounted in the rq's bucket corresponding
692   * to the "effective" bucket_id.
693   *
694   * The user_defined bit is set whenever a task has got a task-specific clamp
695   * value requested from userspace, i.e. the system defaults apply to this task
696   * just as a restriction. This allows to relax default clamps when a less
697   * restrictive task-specific value has been requested, thus allowing to
698   * implement a "nice" semantic. For example, a task running with a 20%
699   * default boost can still drop its own boosting to 0%.
700   */
701  struct uclamp_se {
702  	unsigned int value		: bits_per(SCHED_CAPACITY_SCALE);
703  	unsigned int bucket_id		: bits_per(UCLAMP_BUCKETS);
704  	unsigned int active		: 1;
705  	unsigned int user_defined	: 1;
706  };
707  #endif /* CONFIG_UCLAMP_TASK */
708  
709  union rcu_special {
710  	struct {
711  		u8			blocked;
712  		u8			need_qs;
713  		u8			exp_hint; /* Hint for performance. */
714  		u8			need_mb; /* Readers need smp_mb(). */
715  	} b; /* Bits. */
716  	u32 s; /* Set of bits. */
717  };
718  
719  enum perf_event_task_context {
720  	perf_invalid_context = -1,
721  	perf_hw_context = 0,
722  	perf_sw_context,
723  	perf_nr_task_contexts,
724  };
725  
726  struct wake_q_node {
727  	struct wake_q_node *next;
728  };
729  
730  struct kmap_ctrl {
731  #ifdef CONFIG_KMAP_LOCAL
732  	int				idx;
733  	pte_t				pteval[KM_MAX_IDX];
734  #endif
735  };
736  
737  struct task_struct {
738  #ifdef CONFIG_THREAD_INFO_IN_TASK
739  	/*
740  	 * For reasons of header soup (see current_thread_info()), this
741  	 * must be the first element of task_struct.
742  	 */
743  	struct thread_info		thread_info;
744  #endif
745  	unsigned int			__state;
746  
747  #ifdef CONFIG_PREEMPT_RT
748  	/* saved state for "spinlock sleepers" */
749  	unsigned int			saved_state;
750  #endif
751  
752  	/*
753  	 * This begins the randomizable portion of task_struct. Only
754  	 * scheduling-critical items should be added above here.
755  	 */
756  	randomized_struct_fields_start
757  
758  	void				*stack;
759  	refcount_t			usage;
760  	/* Per task flags (PF_*), defined further below: */
761  	unsigned int			flags;
762  	unsigned int			ptrace;
763  
764  #ifdef CONFIG_SMP
765  	int				on_cpu;
766  	struct __call_single_node	wake_entry;
767  #ifdef CONFIG_THREAD_INFO_IN_TASK
768  	/* Current CPU: */
769  	unsigned int			cpu;
770  #endif
771  	unsigned int			wakee_flips;
772  	unsigned long			wakee_flip_decay_ts;
773  	struct task_struct		*last_wakee;
774  
775  	/*
776  	 * recent_used_cpu is initially set as the last CPU used by a task
777  	 * that wakes affine another task. Waker/wakee relationships can
778  	 * push tasks around a CPU where each wakeup moves to the next one.
779  	 * Tracking a recently used CPU allows a quick search for a recently
780  	 * used CPU that may be idle.
781  	 */
782  	int				recent_used_cpu;
783  	int				wake_cpu;
784  #endif
785  	int				on_rq;
786  
787  	int				prio;
788  	int				static_prio;
789  	int				normal_prio;
790  	unsigned int			rt_priority;
791  
792  	const struct sched_class	*sched_class;
793  	struct sched_entity		se;
794  	struct sched_rt_entity		rt;
795  	struct sched_dl_entity		dl;
796  
797  #ifdef CONFIG_SCHED_CORE
798  	struct rb_node			core_node;
799  	unsigned long			core_cookie;
800  	unsigned int			core_occupation;
801  #endif
802  
803  #ifdef CONFIG_CGROUP_SCHED
804  	struct task_group		*sched_task_group;
805  #endif
806  
807  #ifdef CONFIG_UCLAMP_TASK
808  	/*
809  	 * Clamp values requested for a scheduling entity.
810  	 * Must be updated with task_rq_lock() held.
811  	 */
812  	struct uclamp_se		uclamp_req[UCLAMP_CNT];
813  	/*
814  	 * Effective clamp values used for a scheduling entity.
815  	 * Must be updated with task_rq_lock() held.
816  	 */
817  	struct uclamp_se		uclamp[UCLAMP_CNT];
818  #endif
819  
820  #ifdef CONFIG_PREEMPT_NOTIFIERS
821  	/* List of struct preempt_notifier: */
822  	struct hlist_head		preempt_notifiers;
823  #endif
824  
825  #ifdef CONFIG_BLK_DEV_IO_TRACE
826  	unsigned int			btrace_seq;
827  #endif
828  
829  	unsigned int			policy;
830  	int				nr_cpus_allowed;
831  	const cpumask_t			*cpus_ptr;
832  	cpumask_t			*user_cpus_ptr;
833  	cpumask_t			cpus_mask;
834  	void				*migration_pending;
835  #ifdef CONFIG_SMP
836  	unsigned short			migration_disabled;
837  #endif
838  	unsigned short			migration_flags;
839  
840  #ifdef CONFIG_PREEMPT_RCU
841  	int				rcu_read_lock_nesting;
842  	union rcu_special		rcu_read_unlock_special;
843  	struct list_head		rcu_node_entry;
844  	struct rcu_node			*rcu_blocked_node;
845  #endif /* #ifdef CONFIG_PREEMPT_RCU */
846  
847  #ifdef CONFIG_TASKS_RCU
848  	unsigned long			rcu_tasks_nvcsw;
849  	u8				rcu_tasks_holdout;
850  	u8				rcu_tasks_idx;
851  	int				rcu_tasks_idle_cpu;
852  	struct list_head		rcu_tasks_holdout_list;
853  #endif /* #ifdef CONFIG_TASKS_RCU */
854  
855  #ifdef CONFIG_TASKS_TRACE_RCU
856  	int				trc_reader_nesting;
857  	int				trc_ipi_to_cpu;
858  	union rcu_special		trc_reader_special;
859  	bool				trc_reader_checked;
860  	struct list_head		trc_holdout_list;
861  #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
862  
863  	struct sched_info		sched_info;
864  
865  	struct list_head		tasks;
866  #ifdef CONFIG_SMP
867  	struct plist_node		pushable_tasks;
868  	struct rb_node			pushable_dl_tasks;
869  #endif
870  
871  	struct mm_struct		*mm;
872  	struct mm_struct		*active_mm;
873  
874  	/* Per-thread vma caching: */
875  	struct vmacache			vmacache;
876  
877  #ifdef SPLIT_RSS_COUNTING
878  	struct task_rss_stat		rss_stat;
879  #endif
880  	int				exit_state;
881  	int				exit_code;
882  	int				exit_signal;
883  	/* The signal sent when the parent dies: */
884  	int				pdeath_signal;
885  	/* JOBCTL_*, siglock protected: */
886  	unsigned long			jobctl;
887  
888  	/* Used for emulating ABI behavior of previous Linux versions: */
889  	unsigned int			personality;
890  
891  	/* Scheduler bits, serialized by scheduler locks: */
892  	unsigned			sched_reset_on_fork:1;
893  	unsigned			sched_contributes_to_load:1;
894  	unsigned			sched_migrated:1;
895  #ifdef CONFIG_PSI
896  	unsigned			sched_psi_wake_requeue:1;
897  #endif
898  
899  	/* Force alignment to the next boundary: */
900  	unsigned			:0;
901  
902  	/* Unserialized, strictly 'current' */
903  
904  	/*
905  	 * This field must not be in the scheduler word above due to wakelist
906  	 * queueing no longer being serialized by p->on_cpu. However:
907  	 *
908  	 * p->XXX = X;			ttwu()
909  	 * schedule()			  if (p->on_rq && ..) // false
910  	 *   smp_mb__after_spinlock();	  if (smp_load_acquire(&p->on_cpu) && //true
911  	 *   deactivate_task()		      ttwu_queue_wakelist())
912  	 *     p->on_rq = 0;			p->sched_remote_wakeup = Y;
913  	 *
914  	 * guarantees all stores of 'current' are visible before
915  	 * ->sched_remote_wakeup gets used, so it can be in this word.
916  	 */
917  	unsigned			sched_remote_wakeup:1;
918  
919  	/* Bit to tell LSMs we're in execve(): */
920  	unsigned			in_execve:1;
921  	unsigned			in_iowait:1;
922  #ifndef TIF_RESTORE_SIGMASK
923  	unsigned			restore_sigmask:1;
924  #endif
925  #ifdef CONFIG_MEMCG
926  	unsigned			in_user_fault:1;
927  #endif
928  #ifdef CONFIG_LRU_GEN
929  	/* whether the LRU algorithm may apply to this access */
930  	unsigned			in_lru_fault:1;
931  #endif
932  #ifdef CONFIG_COMPAT_BRK
933  	unsigned			brk_randomized:1;
934  #endif
935  #ifdef CONFIG_CGROUPS
936  	/* disallow userland-initiated cgroup migration */
937  	unsigned			no_cgroup_migration:1;
938  	/* task is frozen/stopped (used by the cgroup freezer) */
939  	unsigned			frozen:1;
940  #endif
941  #ifdef CONFIG_BLK_CGROUP
942  	unsigned			use_memdelay:1;
943  #endif
944  #ifdef CONFIG_PSI
945  	/* Stalled due to lack of memory */
946  	unsigned			in_memstall:1;
947  #endif
948  #ifdef CONFIG_PAGE_OWNER
949  	/* Used by page_owner=on to detect recursion in page tracking. */
950  	unsigned			in_page_owner:1;
951  #endif
952  #ifdef CONFIG_EVENTFD
953  	/* Recursion prevention for eventfd_signal() */
954  	unsigned			in_eventfd_signal:1;
955  #endif
956  
957  	unsigned long			atomic_flags; /* Flags requiring atomic access. */
958  
959  	struct restart_block		restart_block;
960  
961  	pid_t				pid;
962  	pid_t				tgid;
963  
964  #ifdef CONFIG_STACKPROTECTOR
965  	/* Canary value for the -fstack-protector GCC feature: */
966  	unsigned long			stack_canary;
967  #endif
968  	/*
969  	 * Pointers to the (original) parent process, youngest child, younger sibling,
970  	 * older sibling, respectively.  (p->father can be replaced with
971  	 * p->real_parent->pid)
972  	 */
973  
974  	/* Real parent process: */
975  	struct task_struct __rcu	*real_parent;
976  
977  	/* Recipient of SIGCHLD, wait4() reports: */
978  	struct task_struct __rcu	*parent;
979  
980  	/*
981  	 * Children/sibling form the list of natural children:
982  	 */
983  	struct list_head		children;
984  	struct list_head		sibling;
985  	struct task_struct		*group_leader;
986  
987  	/*
988  	 * 'ptraced' is the list of tasks this task is using ptrace() on.
989  	 *
990  	 * This includes both natural children and PTRACE_ATTACH targets.
991  	 * 'ptrace_entry' is this task's link on the p->parent->ptraced list.
992  	 */
993  	struct list_head		ptraced;
994  	struct list_head		ptrace_entry;
995  
996  	/* PID/PID hash table linkage. */
997  	struct pid			*thread_pid;
998  	struct hlist_node		pid_links[PIDTYPE_MAX];
999  	struct list_head		thread_group;
1000  	struct list_head		thread_node;
1001  
1002  	struct completion		*vfork_done;
1003  
1004  	/* CLONE_CHILD_SETTID: */
1005  	int __user			*set_child_tid;
1006  
1007  	/* CLONE_CHILD_CLEARTID: */
1008  	int __user			*clear_child_tid;
1009  
1010  	/* PF_IO_WORKER */
1011  	void				*pf_io_worker;
1012  
1013  	u64				utime;
1014  	u64				stime;
1015  #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
1016  	u64				utimescaled;
1017  	u64				stimescaled;
1018  #endif
1019  	u64				gtime;
1020  #ifdef CONFIG_CPU_FREQ_TIMES
1021  	u64				*time_in_state;
1022  	unsigned int			max_state;
1023  #endif
1024  	struct prev_cputime		prev_cputime;
1025  #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
1026  	struct vtime			vtime;
1027  #endif
1028  
1029  #ifdef CONFIG_NO_HZ_FULL
1030  	atomic_t			tick_dep_mask;
1031  #endif
1032  	/* Context switch counts: */
1033  	unsigned long			nvcsw;
1034  	unsigned long			nivcsw;
1035  
1036  	/* Monotonic time in nsecs: */
1037  	u64				start_time;
1038  
1039  	/* Boot based time in nsecs: */
1040  	u64				start_boottime;
1041  
1042  	/* MM fault and swap info: this can arguably be seen as either mm-specific or thread-specific: */
1043  	unsigned long			min_flt;
1044  	unsigned long			maj_flt;
1045  
1046  	/* Empty if CONFIG_POSIX_CPUTIMERS=n */
1047  	struct posix_cputimers		posix_cputimers;
1048  
1049  #ifdef CONFIG_POSIX_CPU_TIMERS_TASK_WORK
1050  	struct posix_cputimers_work	posix_cputimers_work;
1051  #endif
1052  
1053  	/* Process credentials: */
1054  
1055  	/* Tracer's credentials at attach: */
1056  	const struct cred __rcu		*ptracer_cred;
1057  
1058  	/* Objective and real subjective task credentials (COW): */
1059  	const struct cred __rcu		*real_cred;
1060  
1061  	/* Effective (overridable) subjective task credentials (COW): */
1062  	const struct cred __rcu		*cred;
1063  
1064  #ifdef CONFIG_KEYS
1065  	/* Cached requested key. */
1066  	struct key			*cached_requested_key;
1067  #endif
1068  
1069  	/*
1070  	 * executable name, excluding path.
1071  	 *
1072  	 * - normally initialized setup_new_exec()
1073  	 * - access it with [gs]et_task_comm()
1074  	 * - lock it with task_lock()
1075  	 */
1076  	char				comm[TASK_COMM_LEN];
1077  
1078  	struct nameidata		*nameidata;
1079  
1080  #ifdef CONFIG_SYSVIPC
1081  	struct sysv_sem			sysvsem;
1082  	struct sysv_shm			sysvshm;
1083  #endif
1084  #ifdef CONFIG_DETECT_HUNG_TASK
1085  	unsigned long			last_switch_count;
1086  	unsigned long			last_switch_time;
1087  #endif
1088  	/* Filesystem information: */
1089  	struct fs_struct		*fs;
1090  
1091  	/* Open file information: */
1092  	struct files_struct		*files;
1093  
1094  #ifdef CONFIG_IO_URING
1095  	struct io_uring_task		*io_uring;
1096  #endif
1097  
1098  	/* Namespaces: */
1099  	struct nsproxy			*nsproxy;
1100  
1101  	/* Signal handlers: */
1102  	struct signal_struct		*signal;
1103  	struct sighand_struct __rcu		*sighand;
1104  	sigset_t			blocked;
1105  	sigset_t			real_blocked;
1106  	/* Restored if set_restore_sigmask() was used: */
1107  	sigset_t			saved_sigmask;
1108  	struct sigpending		pending;
1109  	unsigned long			sas_ss_sp;
1110  	size_t				sas_ss_size;
1111  	unsigned int			sas_ss_flags;
1112  
1113  	struct callback_head		*task_works;
1114  
1115  #ifdef CONFIG_AUDIT
1116  #ifdef CONFIG_AUDITSYSCALL
1117  	struct audit_context		*audit_context;
1118  #endif
1119  	kuid_t				loginuid;
1120  	unsigned int			sessionid;
1121  #endif
1122  	struct seccomp			seccomp;
1123  	struct syscall_user_dispatch	syscall_dispatch;
1124  
1125  	/* Thread group tracking: */
1126  	u64				parent_exec_id;
1127  	u64				self_exec_id;
1128  
1129  	/* Protection against (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, mempolicy: */
1130  	spinlock_t			alloc_lock;
1131  
1132  	/* Protection of the PI data structures: */
1133  	raw_spinlock_t			pi_lock;
1134  
1135  	struct wake_q_node		wake_q;
1136  	int				wake_q_count;
1137  
1138  #ifdef CONFIG_RT_MUTEXES
1139  	/* PI waiters blocked on a rt_mutex held by this task: */
1140  	struct rb_root_cached		pi_waiters;
1141  	/* Updated under owner's pi_lock and rq lock */
1142  	struct task_struct		*pi_top_task;
1143  	/* Deadlock detection and priority inheritance handling: */
1144  	struct rt_mutex_waiter		*pi_blocked_on;
1145  #endif
1146  
1147  #ifdef CONFIG_DEBUG_MUTEXES
1148  	/* Mutex deadlock detection: */
1149  	struct mutex_waiter		*blocked_on;
1150  #endif
1151  
1152  #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
1153  	int				non_block_count;
1154  #endif
1155  
1156  #ifdef CONFIG_TRACE_IRQFLAGS
1157  	struct irqtrace_events		irqtrace;
1158  	unsigned int			hardirq_threaded;
1159  	u64				hardirq_chain_key;
1160  	int				softirqs_enabled;
1161  	int				softirq_context;
1162  	int				irq_config;
1163  #endif
1164  #ifdef CONFIG_PREEMPT_RT
1165  	int				softirq_disable_cnt;
1166  #endif
1167  
1168  #ifdef CONFIG_LOCKDEP
1169  # define MAX_LOCK_DEPTH			48UL
1170  	u64				curr_chain_key;
1171  	int				lockdep_depth;
1172  	unsigned int			lockdep_recursion;
1173  	struct held_lock		held_locks[MAX_LOCK_DEPTH];
1174  #endif
1175  
1176  #if defined(CONFIG_UBSAN) && !defined(CONFIG_UBSAN_TRAP)
1177  	unsigned int			in_ubsan;
1178  #endif
1179  
1180  	/* Journalling filesystem info: */
1181  	void				*journal_info;
1182  
1183  	/* Stacked block device info: */
1184  	struct bio_list			*bio_list;
1185  
1186  #ifdef CONFIG_BLOCK
1187  	/* Stack plugging: */
1188  	struct blk_plug			*plug;
1189  #endif
1190  
1191  	/* VM state: */
1192  	struct reclaim_state		*reclaim_state;
1193  
1194  	struct backing_dev_info		*backing_dev_info;
1195  
1196  	struct io_context		*io_context;
1197  
1198  #ifdef CONFIG_COMPACTION
1199  	struct capture_control		*capture_control;
1200  #endif
1201  	/* Ptrace state: */
1202  	unsigned long			ptrace_message;
1203  	kernel_siginfo_t		*last_siginfo;
1204  
1205  	struct task_io_accounting	ioac;
1206  #ifdef CONFIG_PSI
1207  	/* Pressure stall state */
1208  	unsigned int			psi_flags;
1209  #endif
1210  #ifdef CONFIG_TASK_XACCT
1211  	/* Accumulated RSS usage: */
1212  	u64				acct_rss_mem1;
1213  	/* Accumulated virtual memory usage: */
1214  	u64				acct_vm_mem1;
1215  	/* stime + utime since last update: */
1216  	u64				acct_timexpd;
1217  #endif
1218  #ifdef CONFIG_CPUSETS
1219  	/* Protected by ->alloc_lock: */
1220  	nodemask_t			mems_allowed;
1221  	/* Sequence number to catch updates: */
1222  	seqcount_spinlock_t		mems_allowed_seq;
1223  	int				cpuset_mem_spread_rotor;
1224  	int				cpuset_slab_spread_rotor;
1225  #endif
1226  #ifdef CONFIG_CGROUPS
1227  	/* Control Group info protected by css_set_lock: */
1228  	struct css_set __rcu		*cgroups;
1229  	/* cg_list protected by css_set_lock and tsk->alloc_lock: */
1230  	struct list_head		cg_list;
1231  #endif
1232  #ifdef CONFIG_X86_CPU_RESCTRL
1233  	u32				closid;
1234  	u32				rmid;
1235  #endif
1236  #ifdef CONFIG_FUTEX
1237  	struct robust_list_head __user	*robust_list;
1238  #ifdef CONFIG_COMPAT
1239  	struct compat_robust_list_head __user *compat_robust_list;
1240  #endif
1241  	struct list_head		pi_state_list;
1242  	struct futex_pi_state		*pi_state_cache;
1243  	struct mutex			futex_exit_mutex;
1244  	unsigned int			futex_state;
1245  #endif
1246  #ifdef CONFIG_PERF_EVENTS
1247  	struct perf_event_context	*perf_event_ctxp[perf_nr_task_contexts];
1248  	struct mutex			perf_event_mutex;
1249  	struct list_head		perf_event_list;
1250  #endif
1251  #ifdef CONFIG_DEBUG_PREEMPT
1252  	unsigned long			preempt_disable_ip;
1253  #endif
1254  #ifdef CONFIG_NUMA
1255  	/* Protected by alloc_lock: */
1256  	struct mempolicy		*mempolicy;
1257  	short				il_prev;
1258  	short				pref_node_fork;
1259  #endif
1260  #ifdef CONFIG_NUMA_BALANCING
1261  	int				numa_scan_seq;
1262  	unsigned int			numa_scan_period;
1263  	unsigned int			numa_scan_period_max;
1264  	int				numa_preferred_nid;
1265  	unsigned long			numa_migrate_retry;
1266  	/* Migration stamp: */
1267  	u64				node_stamp;
1268  	u64				last_task_numa_placement;
1269  	u64				last_sum_exec_runtime;
1270  	struct callback_head		numa_work;
1271  
1272  	/*
1273  	 * This pointer is only modified for current in syscall and
1274  	 * pagefault context (and for tasks being destroyed), so it can be read
1275  	 * from any of the following contexts:
1276  	 *  - RCU read-side critical section
1277  	 *  - current->numa_group from everywhere
1278  	 *  - task's runqueue locked, task not running
1279  	 */
1280  	struct numa_group __rcu		*numa_group;
1281  
1282  	/*
1283  	 * numa_faults is an array split into four regions:
1284  	 * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer
1285  	 * in this precise order.
1286  	 *
1287  	 * faults_memory: Exponential decaying average of faults on a per-node
1288  	 * basis. Scheduling placement decisions are made based on these
1289  	 * counts. The values remain static for the duration of a PTE scan.
1290  	 * faults_cpu: Track the nodes the process was running on when a NUMA
1291  	 * hinting fault was incurred.
1292  	 * faults_memory_buffer and faults_cpu_buffer: Record faults per node
1293  	 * during the current scan window. When the scan completes, the counts
1294  	 * in faults_memory and faults_cpu decay and these values are copied.
1295  	 */
1296  	unsigned long			*numa_faults;
1297  	unsigned long			total_numa_faults;
1298  
1299  	/*
1300  	 * numa_faults_locality tracks if faults recorded during the last
1301  	 * scan window were remote/local or failed to migrate. The task scan
1302  	 * period is adapted based on the locality of the faults with different
1303  	 * weights depending on whether they were shared or private faults
1304  	 */
1305  	unsigned long			numa_faults_locality[3];
1306  
1307  	unsigned long			numa_pages_migrated;
1308  #endif /* CONFIG_NUMA_BALANCING */
1309  
1310  #ifdef CONFIG_RSEQ
1311  	struct rseq __user *rseq;
1312  	u32 rseq_sig;
1313  	/*
1314  	 * RmW on rseq_event_mask must be performed atomically
1315  	 * with respect to preemption.
1316  	 */
1317  	unsigned long rseq_event_mask;
1318  #endif
1319  
1320  	struct tlbflush_unmap_batch	tlb_ubc;
1321  
1322  	union {
1323  		refcount_t		rcu_users;
1324  		struct rcu_head		rcu;
1325  	};
1326  
1327  	/* Cache last used pipe for splice(): */
1328  	struct pipe_inode_info		*splice_pipe;
1329  
1330  	struct page_frag		task_frag;
1331  
1332  #ifdef CONFIG_TASK_DELAY_ACCT
1333  	struct task_delay_info		*delays;
1334  #endif
1335  
1336  #ifdef CONFIG_FAULT_INJECTION
1337  	int				make_it_fail;
1338  	unsigned int			fail_nth;
1339  #endif
1340  	/*
1341  	 * When (nr_dirtied >= nr_dirtied_pause), it's time to call
1342  	 * balance_dirty_pages() for a dirty throttling pause:
1343  	 */
1344  	int				nr_dirtied;
1345  	int				nr_dirtied_pause;
1346  	/* Start of a write-and-pause period: */
1347  	unsigned long			dirty_paused_when;
1348  
1349  #ifdef CONFIG_LATENCYTOP
1350  	int				latency_record_count;
1351  	struct latency_record		latency_record[LT_SAVECOUNT];
1352  #endif
1353  	/*
1354  	 * Time slack values; these are used to round up poll() and
1355  	 * select() etc timeout values. These are in nanoseconds.
1356  	 */
1357  	u64				timer_slack_ns;
1358  	u64				default_timer_slack_ns;
1359  
1360  #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
1361  	unsigned int			kasan_depth;
1362  #endif
1363  
1364  #ifdef CONFIG_KCSAN
1365  	struct kcsan_ctx		kcsan_ctx;
1366  #ifdef CONFIG_TRACE_IRQFLAGS
1367  	struct irqtrace_events		kcsan_save_irqtrace;
1368  #endif
1369  #endif
1370  
1371  	struct kunit			*kunit_test;
1372  
1373  #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1374  	/* Index of current stored address in ret_stack: */
1375  	int				curr_ret_stack;
1376  	int				curr_ret_depth;
1377  
1378  	/* Stack of return addresses for return function tracing: */
1379  	struct ftrace_ret_stack		*ret_stack;
1380  
1381  	/* Timestamp for last schedule: */
1382  	unsigned long long		ftrace_timestamp;
1383  
1384  	/*
1385  	 * Number of functions that haven't been traced
1386  	 * because of depth overrun:
1387  	 */
1388  	atomic_t			trace_overrun;
1389  
1390  	/* Pause tracing: */
1391  	atomic_t			tracing_graph_pause;
1392  #endif
1393  
1394  #ifdef CONFIG_TRACING
1395  	/* State flags for use by tracers: */
1396  	unsigned long			trace;
1397  
1398  	/* Bitmask and counter of trace recursion: */
1399  	unsigned long			trace_recursion;
1400  #endif /* CONFIG_TRACING */
1401  
1402  #ifdef CONFIG_KCOV
1403  	/* See kernel/kcov.c for more details. */
1404  
1405  	/* Coverage collection mode enabled for this task (0 if disabled): */
1406  	unsigned int			kcov_mode;
1407  
1408  	/* Size of the kcov_area: */
1409  	unsigned int			kcov_size;
1410  
1411  	/* Buffer for coverage collection: */
1412  	void				*kcov_area;
1413  
1414  	/* KCOV descriptor wired with this task or NULL: */
1415  	struct kcov			*kcov;
1416  
1417  	/* KCOV common handle for remote coverage collection: */
1418  	u64				kcov_handle;
1419  
1420  	/* KCOV sequence number: */
1421  	int				kcov_sequence;
1422  
1423  	/* Collect coverage from softirq context: */
1424  	unsigned int			kcov_softirq;
1425  #endif
1426  
1427  #ifdef CONFIG_MEMCG
1428  	struct mem_cgroup		*memcg_in_oom;
1429  	gfp_t				memcg_oom_gfp_mask;
1430  	int				memcg_oom_order;
1431  
1432  	/* Number of pages to reclaim on returning to userland: */
1433  	unsigned int			memcg_nr_pages_over_high;
1434  
1435  	/* Used by memcontrol for targeted memcg charge: */
1436  	struct mem_cgroup		*active_memcg;
1437  #endif
1438  
1439  #ifdef CONFIG_BLK_CGROUP
1440  	struct request_queue		*throttle_queue;
1441  #endif
1442  
1443  #ifdef CONFIG_UPROBES
1444  	struct uprobe_task		*utask;
1445  #endif
1446  #if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE)
1447  	unsigned int			sequential_io;
1448  	unsigned int			sequential_io_avg;
1449  #endif
1450  	struct kmap_ctrl		kmap_ctrl;
1451  #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
1452  	unsigned long			task_state_change;
1453  # ifdef CONFIG_PREEMPT_RT
1454  	unsigned long			saved_state_change;
1455  # endif
1456  #endif
1457  	int				pagefault_disabled;
1458  #ifdef CONFIG_MMU
1459  	struct task_struct		*oom_reaper_list;
1460  	struct timer_list		oom_reaper_timer;
1461  #endif
1462  #ifdef CONFIG_VMAP_STACK
1463  	struct vm_struct		*stack_vm_area;
1464  #endif
1465  #ifdef CONFIG_THREAD_INFO_IN_TASK
1466  	/* A live task holds one reference: */
1467  	refcount_t			stack_refcount;
1468  #endif
1469  #ifdef CONFIG_LIVEPATCH
1470  	int patch_state;
1471  #endif
1472  #ifdef CONFIG_SECURITY
1473  	/* Used by LSM modules for access restriction: */
1474  	void				*security;
1475  #endif
1476  #ifdef CONFIG_BPF_SYSCALL
1477  	/* Used by BPF task local storage */
1478  	struct bpf_local_storage __rcu	*bpf_storage;
1479  	/* Used for BPF run context */
1480  	struct bpf_run_ctx		*bpf_ctx;
1481  #endif
1482  
1483  #ifdef CONFIG_GCC_PLUGIN_STACKLEAK
1484  	unsigned long			lowest_stack;
1485  	unsigned long			prev_lowest_stack;
1486  #endif
1487  
1488  #ifdef CONFIG_X86_MCE
1489  	void __user			*mce_vaddr;
1490  	__u64				mce_kflags;
1491  	u64				mce_addr;
1492  	__u64				mce_ripv : 1,
1493  					mce_whole_page : 1,
1494  					__mce_reserved : 62;
1495  	struct callback_head		mce_kill_me;
1496  	int				mce_count;
1497  #endif
1498  	ANDROID_VENDOR_DATA_ARRAY(1, 64);
1499  	ANDROID_OEM_DATA_ARRAY(1, 6);
1500  
1501  #ifdef CONFIG_KRETPROBES
1502  	struct llist_head               kretprobe_instances;
1503  #endif
1504  
1505  #ifdef CONFIG_ARCH_HAS_PARANOID_L1D_FLUSH
1506  	/*
1507  	 * If L1D flush is supported on mm context switch
1508  	 * then we use this callback head to queue kill work
1509  	 * to kill tasks that are not running on SMT disabled
1510  	 * cores
1511  	 */
1512  	struct callback_head		l1d_flush_kill;
1513  #endif
1514  
1515  	ANDROID_KABI_RESERVE(1);
1516  	ANDROID_KABI_RESERVE(2);
1517  	ANDROID_KABI_RESERVE(3);
1518  	ANDROID_KABI_RESERVE(4);
1519  	ANDROID_KABI_RESERVE(5);
1520  	ANDROID_KABI_RESERVE(6);
1521  	ANDROID_KABI_RESERVE(7);
1522  	ANDROID_KABI_RESERVE(8);
1523  
1524  	/*
1525  	 * New fields for task_struct should be added above here, so that
1526  	 * they are included in the randomized portion of task_struct.
1527  	 */
1528  	randomized_struct_fields_end
1529  
1530  	/* CPU-specific state of this task: */
1531  	struct thread_struct		thread;
1532  
1533  	/*
1534  	 * WARNING: on x86, 'thread_struct' contains a variable-sized
1535  	 * structure.  It *MUST* be at the end of 'task_struct'.
1536  	 *
1537  	 * Do not put anything below here!
1538  	 */
1539  };
1540  
task_pid(struct task_struct * task)1541  static inline struct pid *task_pid(struct task_struct *task)
1542  {
1543  	return task->thread_pid;
1544  }
1545  
1546  /*
1547   * the helpers to get the task's different pids as they are seen
1548   * from various namespaces
1549   *
1550   * task_xid_nr()     : global id, i.e. the id seen from the init namespace;
1551   * task_xid_vnr()    : virtual id, i.e. the id seen from the pid namespace of
1552   *                     current.
1553   * task_xid_nr_ns()  : id seen from the ns specified;
1554   *
1555   * see also pid_nr() etc in include/linux/pid.h
1556   */
1557  pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, struct pid_namespace *ns);
1558  
task_pid_nr(struct task_struct * tsk)1559  static inline pid_t task_pid_nr(struct task_struct *tsk)
1560  {
1561  	return tsk->pid;
1562  }
1563  
task_pid_nr_ns(struct task_struct * tsk,struct pid_namespace * ns)1564  static inline pid_t task_pid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1565  {
1566  	return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns);
1567  }
1568  
task_pid_vnr(struct task_struct * tsk)1569  static inline pid_t task_pid_vnr(struct task_struct *tsk)
1570  {
1571  	return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL);
1572  }
1573  
1574  
task_tgid_nr(struct task_struct * tsk)1575  static inline pid_t task_tgid_nr(struct task_struct *tsk)
1576  {
1577  	return tsk->tgid;
1578  }
1579  
1580  /**
1581   * pid_alive - check that a task structure is not stale
1582   * @p: Task structure to be checked.
1583   *
1584   * Test if a process is not yet dead (at most zombie state)
1585   * If pid_alive fails, then pointers within the task structure
1586   * can be stale and must not be dereferenced.
1587   *
1588   * Return: 1 if the process is alive. 0 otherwise.
1589   */
pid_alive(const struct task_struct * p)1590  static inline int pid_alive(const struct task_struct *p)
1591  {
1592  	return p->thread_pid != NULL;
1593  }
1594  
task_pgrp_nr_ns(struct task_struct * tsk,struct pid_namespace * ns)1595  static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1596  {
1597  	return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns);
1598  }
1599  
task_pgrp_vnr(struct task_struct * tsk)1600  static inline pid_t task_pgrp_vnr(struct task_struct *tsk)
1601  {
1602  	return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL);
1603  }
1604  
1605  
task_session_nr_ns(struct task_struct * tsk,struct pid_namespace * ns)1606  static inline pid_t task_session_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1607  {
1608  	return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns);
1609  }
1610  
task_session_vnr(struct task_struct * tsk)1611  static inline pid_t task_session_vnr(struct task_struct *tsk)
1612  {
1613  	return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL);
1614  }
1615  
task_tgid_nr_ns(struct task_struct * tsk,struct pid_namespace * ns)1616  static inline pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
1617  {
1618  	return __task_pid_nr_ns(tsk, PIDTYPE_TGID, ns);
1619  }
1620  
task_tgid_vnr(struct task_struct * tsk)1621  static inline pid_t task_tgid_vnr(struct task_struct *tsk)
1622  {
1623  	return __task_pid_nr_ns(tsk, PIDTYPE_TGID, NULL);
1624  }
1625  
task_ppid_nr_ns(const struct task_struct * tsk,struct pid_namespace * ns)1626  static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns)
1627  {
1628  	pid_t pid = 0;
1629  
1630  	rcu_read_lock();
1631  	if (pid_alive(tsk))
1632  		pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns);
1633  	rcu_read_unlock();
1634  
1635  	return pid;
1636  }
1637  
task_ppid_nr(const struct task_struct * tsk)1638  static inline pid_t task_ppid_nr(const struct task_struct *tsk)
1639  {
1640  	return task_ppid_nr_ns(tsk, &init_pid_ns);
1641  }
1642  
1643  /* Obsolete, do not use: */
task_pgrp_nr(struct task_struct * tsk)1644  static inline pid_t task_pgrp_nr(struct task_struct *tsk)
1645  {
1646  	return task_pgrp_nr_ns(tsk, &init_pid_ns);
1647  }
1648  
1649  #define TASK_REPORT_IDLE	(TASK_REPORT + 1)
1650  #define TASK_REPORT_MAX		(TASK_REPORT_IDLE << 1)
1651  
task_state_index(struct task_struct * tsk)1652  static inline unsigned int task_state_index(struct task_struct *tsk)
1653  {
1654  	unsigned int tsk_state = READ_ONCE(tsk->__state);
1655  	unsigned int state = (tsk_state | tsk->exit_state) & TASK_REPORT;
1656  
1657  	BUILD_BUG_ON_NOT_POWER_OF_2(TASK_REPORT_MAX);
1658  
1659  	if (tsk_state == TASK_IDLE)
1660  		state = TASK_REPORT_IDLE;
1661  
1662  	/*
1663  	 * We're lying here, but rather than expose a completely new task state
1664  	 * to userspace, we can make this appear as if the task has gone through
1665  	 * a regular rt_mutex_lock() call.
1666  	 */
1667  	if (tsk_state == TASK_RTLOCK_WAIT)
1668  		state = TASK_UNINTERRUPTIBLE;
1669  
1670  	return fls(state);
1671  }
1672  
task_index_to_char(unsigned int state)1673  static inline char task_index_to_char(unsigned int state)
1674  {
1675  	static const char state_char[] = "RSDTtXZPI";
1676  
1677  	BUILD_BUG_ON(1 + ilog2(TASK_REPORT_MAX) != sizeof(state_char) - 1);
1678  
1679  	return state_char[state];
1680  }
1681  
task_state_to_char(struct task_struct * tsk)1682  static inline char task_state_to_char(struct task_struct *tsk)
1683  {
1684  	return task_index_to_char(task_state_index(tsk));
1685  }
1686  
1687  /**
1688   * is_global_init - check if a task structure is init. Since init
1689   * is free to have sub-threads we need to check tgid.
1690   * @tsk: Task structure to be checked.
1691   *
1692   * Check if a task structure is the first user space task the kernel created.
1693   *
1694   * Return: 1 if the task structure is init. 0 otherwise.
1695   */
is_global_init(struct task_struct * tsk)1696  static inline int is_global_init(struct task_struct *tsk)
1697  {
1698  	return task_tgid_nr(tsk) == 1;
1699  }
1700  
1701  extern struct pid *cad_pid;
1702  
1703  /*
1704   * Per process flags
1705   */
1706  #define PF_VCPU			0x00000001	/* I'm a virtual CPU */
1707  #define PF_IDLE			0x00000002	/* I am an IDLE thread */
1708  #define PF_EXITING		0x00000004	/* Getting shut down */
1709  #define PF_IO_WORKER		0x00000010	/* Task is an IO worker */
1710  #define PF_WQ_WORKER		0x00000020	/* I'm a workqueue worker */
1711  #define PF_FORKNOEXEC		0x00000040	/* Forked but didn't exec */
1712  #define PF_MCE_PROCESS		0x00000080      /* Process policy on mce errors */
1713  #define PF_SUPERPRIV		0x00000100	/* Used super-user privileges */
1714  #define PF_DUMPCORE		0x00000200	/* Dumped core */
1715  #define PF_SIGNALED		0x00000400	/* Killed by a signal */
1716  #define PF_MEMALLOC		0x00000800	/* Allocating memory */
1717  #define PF_NPROC_EXCEEDED	0x00001000	/* set_user() noticed that RLIMIT_NPROC was exceeded */
1718  #define PF_USED_MATH		0x00002000	/* If unset the fpu must be initialized before use */
1719  #define PF_NOFREEZE		0x00008000	/* This thread should not be frozen */
1720  #define PF_FROZEN		0x00010000	/* Frozen for system suspend */
1721  #define PF_KSWAPD		0x00020000	/* I am kswapd */
1722  #define PF_MEMALLOC_NOFS	0x00040000	/* All allocation requests will inherit GFP_NOFS */
1723  #define PF_MEMALLOC_NOIO	0x00080000	/* All allocation requests will inherit GFP_NOIO */
1724  #define PF_LOCAL_THROTTLE	0x00100000	/* Throttle writes only against the bdi I write to,
1725  						 * I am cleaning dirty pages from some other bdi. */
1726  #define PF_KTHREAD		0x00200000	/* I am a kernel thread */
1727  #define PF_RANDOMIZE		0x00400000	/* Randomize virtual address space */
1728  #define PF_SWAPWRITE		0x00800000	/* Allowed to write to swap */
1729  #define PF_NO_SETAFFINITY	0x04000000	/* Userland is not allowed to meddle with cpus_mask */
1730  #define PF_MCE_EARLY		0x08000000      /* Early kill for mce process policy */
1731  #define PF_MEMALLOC_PIN		0x10000000	/* Allocation context constrained to zones which allow long term pinning. */
1732  #define PF_FREEZER_SKIP		0x40000000	/* Freezer should not count it as freezable */
1733  #define PF_SUSPEND_TASK		0x80000000      /* This thread called freeze_processes() and should not be frozen */
1734  
1735  /*
1736   * Only the _current_ task can read/write to tsk->flags, but other
1737   * tasks can access tsk->flags in readonly mode for example
1738   * with tsk_used_math (like during threaded core dumping).
1739   * There is however an exception to this rule during ptrace
1740   * or during fork: the ptracer task is allowed to write to the
1741   * child->flags of its traced child (same goes for fork, the parent
1742   * can write to the child->flags), because we're guaranteed the
1743   * child is not running and in turn not changing child->flags
1744   * at the same time the parent does it.
1745   */
1746  #define clear_stopped_child_used_math(child)	do { (child)->flags &= ~PF_USED_MATH; } while (0)
1747  #define set_stopped_child_used_math(child)	do { (child)->flags |= PF_USED_MATH; } while (0)
1748  #define clear_used_math()			clear_stopped_child_used_math(current)
1749  #define set_used_math()				set_stopped_child_used_math(current)
1750  
1751  #define conditional_stopped_child_used_math(condition, child) \
1752  	do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0)
1753  
1754  #define conditional_used_math(condition)	conditional_stopped_child_used_math(condition, current)
1755  
1756  #define copy_to_stopped_child_used_math(child) \
1757  	do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0)
1758  
1759  /* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */
1760  #define tsk_used_math(p)			((p)->flags & PF_USED_MATH)
1761  #define used_math()				tsk_used_math(current)
1762  
is_percpu_thread(void)1763  static __always_inline bool is_percpu_thread(void)
1764  {
1765  #ifdef CONFIG_SMP
1766  	return (current->flags & PF_NO_SETAFFINITY) &&
1767  		(current->nr_cpus_allowed  == 1);
1768  #else
1769  	return true;
1770  #endif
1771  }
1772  
1773  /* Per-process atomic flags. */
1774  #define PFA_NO_NEW_PRIVS		0	/* May not gain new privileges. */
1775  #define PFA_SPREAD_PAGE			1	/* Spread page cache over cpuset */
1776  #define PFA_SPREAD_SLAB			2	/* Spread some slab caches over cpuset */
1777  #define PFA_SPEC_SSB_DISABLE		3	/* Speculative Store Bypass disabled */
1778  #define PFA_SPEC_SSB_FORCE_DISABLE	4	/* Speculative Store Bypass force disabled*/
1779  #define PFA_SPEC_IB_DISABLE		5	/* Indirect branch speculation restricted */
1780  #define PFA_SPEC_IB_FORCE_DISABLE	6	/* Indirect branch speculation permanently restricted */
1781  #define PFA_SPEC_SSB_NOEXEC		7	/* Speculative Store Bypass clear on execve() */
1782  
1783  #define TASK_PFA_TEST(name, func)					\
1784  	static inline bool task_##func(struct task_struct *p)		\
1785  	{ return test_bit(PFA_##name, &p->atomic_flags); }
1786  
1787  #define TASK_PFA_SET(name, func)					\
1788  	static inline void task_set_##func(struct task_struct *p)	\
1789  	{ set_bit(PFA_##name, &p->atomic_flags); }
1790  
1791  #define TASK_PFA_CLEAR(name, func)					\
1792  	static inline void task_clear_##func(struct task_struct *p)	\
1793  	{ clear_bit(PFA_##name, &p->atomic_flags); }
1794  
TASK_PFA_TEST(NO_NEW_PRIVS,no_new_privs)1795  TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs)
1796  TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs)
1797  
1798  TASK_PFA_TEST(SPREAD_PAGE, spread_page)
1799  TASK_PFA_SET(SPREAD_PAGE, spread_page)
1800  TASK_PFA_CLEAR(SPREAD_PAGE, spread_page)
1801  
1802  TASK_PFA_TEST(SPREAD_SLAB, spread_slab)
1803  TASK_PFA_SET(SPREAD_SLAB, spread_slab)
1804  TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)
1805  
1806  TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable)
1807  TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable)
1808  TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable)
1809  
1810  TASK_PFA_TEST(SPEC_SSB_NOEXEC, spec_ssb_noexec)
1811  TASK_PFA_SET(SPEC_SSB_NOEXEC, spec_ssb_noexec)
1812  TASK_PFA_CLEAR(SPEC_SSB_NOEXEC, spec_ssb_noexec)
1813  
1814  TASK_PFA_TEST(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
1815  TASK_PFA_SET(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
1816  
1817  TASK_PFA_TEST(SPEC_IB_DISABLE, spec_ib_disable)
1818  TASK_PFA_SET(SPEC_IB_DISABLE, spec_ib_disable)
1819  TASK_PFA_CLEAR(SPEC_IB_DISABLE, spec_ib_disable)
1820  
1821  TASK_PFA_TEST(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
1822  TASK_PFA_SET(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
1823  
1824  static inline void
1825  current_restore_flags(unsigned long orig_flags, unsigned long flags)
1826  {
1827  	current->flags &= ~flags;
1828  	current->flags |= orig_flags & flags;
1829  }
1830  
1831  extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
1832  extern int task_can_attach(struct task_struct *p);
1833  extern int dl_bw_alloc(int cpu, u64 dl_bw);
1834  extern void dl_bw_free(int cpu, u64 dl_bw);
1835  
1836  #ifdef CONFIG_RT_SOFTINT_OPTIMIZATION
1837  extern bool cpupri_check_rt(void);
1838  #else
cpupri_check_rt(void)1839  static inline bool cpupri_check_rt(void)
1840  {
1841  	return false;
1842  }
1843  #endif
1844  
1845  #ifdef CONFIG_SMP
1846  extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask);
1847  extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask);
1848  extern int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, int node);
1849  extern void release_user_cpus_ptr(struct task_struct *p);
1850  extern int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask);
1851  extern void force_compatible_cpus_allowed_ptr(struct task_struct *p);
1852  extern void relax_compatible_cpus_allowed_ptr(struct task_struct *p);
1853  #else
do_set_cpus_allowed(struct task_struct * p,const struct cpumask * new_mask)1854  static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
1855  {
1856  }
set_cpus_allowed_ptr(struct task_struct * p,const struct cpumask * new_mask)1857  static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
1858  {
1859  	if (!cpumask_test_cpu(0, new_mask))
1860  		return -EINVAL;
1861  	return 0;
1862  }
dup_user_cpus_ptr(struct task_struct * dst,struct task_struct * src,int node)1863  static inline int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, int node)
1864  {
1865  	if (src->user_cpus_ptr)
1866  		return -EINVAL;
1867  	return 0;
1868  }
release_user_cpus_ptr(struct task_struct * p)1869  static inline void release_user_cpus_ptr(struct task_struct *p)
1870  {
1871  	WARN_ON(p->user_cpus_ptr);
1872  }
1873  
dl_task_check_affinity(struct task_struct * p,const struct cpumask * mask)1874  static inline int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask)
1875  {
1876  	return 0;
1877  }
1878  #endif
1879  
1880  extern int yield_to(struct task_struct *p, bool preempt);
1881  extern void set_user_nice(struct task_struct *p, long nice);
1882  extern int task_prio(const struct task_struct *p);
1883  
1884  /**
1885   * task_nice - return the nice value of a given task.
1886   * @p: the task in question.
1887   *
1888   * Return: The nice value [ -20 ... 0 ... 19 ].
1889   */
task_nice(const struct task_struct * p)1890  static inline int task_nice(const struct task_struct *p)
1891  {
1892  	return PRIO_TO_NICE((p)->static_prio);
1893  }
1894  
1895  extern int can_nice(const struct task_struct *p, const int nice);
1896  extern int task_curr(const struct task_struct *p);
1897  extern int idle_cpu(int cpu);
1898  extern int available_idle_cpu(int cpu);
1899  extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *);
1900  extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *);
1901  extern void sched_set_fifo(struct task_struct *p);
1902  extern void sched_set_fifo_low(struct task_struct *p);
1903  extern void sched_set_normal(struct task_struct *p, int nice);
1904  extern int sched_setattr(struct task_struct *, const struct sched_attr *);
1905  extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *);
1906  extern struct task_struct *idle_task(int cpu);
1907  
1908  /**
1909   * is_idle_task - is the specified task an idle task?
1910   * @p: the task in question.
1911   *
1912   * Return: 1 if @p is an idle task. 0 otherwise.
1913   */
is_idle_task(const struct task_struct * p)1914  static __always_inline bool is_idle_task(const struct task_struct *p)
1915  {
1916  	return !!(p->flags & PF_IDLE);
1917  }
1918  
1919  extern struct task_struct *curr_task(int cpu);
1920  extern void ia64_set_curr_task(int cpu, struct task_struct *p);
1921  
1922  void yield(void);
1923  
1924  union thread_union {
1925  #ifndef CONFIG_ARCH_TASK_STRUCT_ON_STACK
1926  	struct task_struct task;
1927  #endif
1928  #ifndef CONFIG_THREAD_INFO_IN_TASK
1929  	struct thread_info thread_info;
1930  #endif
1931  	unsigned long stack[THREAD_SIZE/sizeof(long)];
1932  };
1933  
1934  #ifndef CONFIG_THREAD_INFO_IN_TASK
1935  extern struct thread_info init_thread_info;
1936  #endif
1937  
1938  extern unsigned long init_stack[THREAD_SIZE / sizeof(unsigned long)];
1939  
1940  #ifdef CONFIG_THREAD_INFO_IN_TASK
task_thread_info(struct task_struct * task)1941  static inline struct thread_info *task_thread_info(struct task_struct *task)
1942  {
1943  	return &task->thread_info;
1944  }
1945  #elif !defined(__HAVE_THREAD_FUNCTIONS)
1946  # define task_thread_info(task)	((struct thread_info *)(task)->stack)
1947  #endif
1948  
1949  /*
1950   * find a task by one of its numerical ids
1951   *
1952   * find_task_by_pid_ns():
1953   *      finds a task by its pid in the specified namespace
1954   * find_task_by_vpid():
1955   *      finds a task by its virtual pid
1956   *
1957   * see also find_vpid() etc in include/linux/pid.h
1958   */
1959  
1960  extern struct task_struct *find_task_by_vpid(pid_t nr);
1961  extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns);
1962  
1963  /*
1964   * find a task by its virtual pid and get the task struct
1965   */
1966  extern struct task_struct *find_get_task_by_vpid(pid_t nr);
1967  
1968  extern int wake_up_state(struct task_struct *tsk, unsigned int state);
1969  extern int wake_up_process(struct task_struct *tsk);
1970  extern void wake_up_new_task(struct task_struct *tsk);
1971  
1972  #ifdef CONFIG_SMP
1973  extern void kick_process(struct task_struct *tsk);
1974  #else
kick_process(struct task_struct * tsk)1975  static inline void kick_process(struct task_struct *tsk) { }
1976  #endif
1977  
1978  extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec);
1979  
set_task_comm(struct task_struct * tsk,const char * from)1980  static inline void set_task_comm(struct task_struct *tsk, const char *from)
1981  {
1982  	__set_task_comm(tsk, from, false);
1983  }
1984  
1985  extern char *__get_task_comm(char *to, size_t len, struct task_struct *tsk);
1986  #define get_task_comm(buf, tsk) ({			\
1987  	BUILD_BUG_ON(sizeof(buf) != TASK_COMM_LEN);	\
1988  	__get_task_comm(buf, sizeof(buf), tsk);		\
1989  })
1990  
1991  #ifdef CONFIG_SMP
scheduler_ipi(void)1992  static __always_inline void scheduler_ipi(void)
1993  {
1994  	/*
1995  	 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
1996  	 * TIF_NEED_RESCHED remotely (for the first time) will also send
1997  	 * this IPI.
1998  	 */
1999  	preempt_fold_need_resched();
2000  }
2001  extern unsigned long wait_task_inactive(struct task_struct *, unsigned int match_state);
2002  #else
scheduler_ipi(void)2003  static inline void scheduler_ipi(void) { }
wait_task_inactive(struct task_struct * p,unsigned int match_state)2004  static inline unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state)
2005  {
2006  	return 1;
2007  }
2008  #endif
2009  
2010  /*
2011   * Set thread flags in other task's structures.
2012   * See asm/thread_info.h for TIF_xxxx flags available:
2013   */
set_tsk_thread_flag(struct task_struct * tsk,int flag)2014  static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag)
2015  {
2016  	set_ti_thread_flag(task_thread_info(tsk), flag);
2017  }
2018  
clear_tsk_thread_flag(struct task_struct * tsk,int flag)2019  static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag)
2020  {
2021  	clear_ti_thread_flag(task_thread_info(tsk), flag);
2022  }
2023  
update_tsk_thread_flag(struct task_struct * tsk,int flag,bool value)2024  static inline void update_tsk_thread_flag(struct task_struct *tsk, int flag,
2025  					  bool value)
2026  {
2027  	update_ti_thread_flag(task_thread_info(tsk), flag, value);
2028  }
2029  
test_and_set_tsk_thread_flag(struct task_struct * tsk,int flag)2030  static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
2031  {
2032  	return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
2033  }
2034  
test_and_clear_tsk_thread_flag(struct task_struct * tsk,int flag)2035  static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
2036  {
2037  	return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
2038  }
2039  
test_tsk_thread_flag(struct task_struct * tsk,int flag)2040  static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
2041  {
2042  	return test_ti_thread_flag(task_thread_info(tsk), flag);
2043  }
2044  
set_tsk_need_resched(struct task_struct * tsk)2045  static inline void set_tsk_need_resched(struct task_struct *tsk)
2046  {
2047  	set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
2048  }
2049  
clear_tsk_need_resched(struct task_struct * tsk)2050  static inline void clear_tsk_need_resched(struct task_struct *tsk)
2051  {
2052  	clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
2053  }
2054  
test_tsk_need_resched(struct task_struct * tsk)2055  static inline int test_tsk_need_resched(struct task_struct *tsk)
2056  {
2057  	return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
2058  }
2059  
2060  /*
2061   * cond_resched() and cond_resched_lock(): latency reduction via
2062   * explicit rescheduling in places that are safe. The return
2063   * value indicates whether a reschedule was done in fact.
2064   * cond_resched_lock() will drop the spinlock before scheduling,
2065   */
2066  #if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC)
2067  extern int __cond_resched(void);
2068  
2069  #ifdef CONFIG_PREEMPT_DYNAMIC
2070  
2071  DECLARE_STATIC_CALL(cond_resched, __cond_resched);
2072  
_cond_resched(void)2073  static __always_inline int _cond_resched(void)
2074  {
2075  	return static_call_mod(cond_resched)();
2076  }
2077  
2078  #else
2079  
_cond_resched(void)2080  static inline int _cond_resched(void)
2081  {
2082  	return __cond_resched();
2083  }
2084  
2085  #endif /* CONFIG_PREEMPT_DYNAMIC */
2086  
2087  #else
2088  
_cond_resched(void)2089  static inline int _cond_resched(void) { return 0; }
2090  
2091  #endif /* !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) */
2092  
2093  #define cond_resched() ({			\
2094  	___might_sleep(__FILE__, __LINE__, 0);	\
2095  	_cond_resched();			\
2096  })
2097  
2098  extern int __cond_resched_lock(spinlock_t *lock);
2099  extern int __cond_resched_rwlock_read(rwlock_t *lock);
2100  extern int __cond_resched_rwlock_write(rwlock_t *lock);
2101  
2102  #define cond_resched_lock(lock) ({				\
2103  	___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\
2104  	__cond_resched_lock(lock);				\
2105  })
2106  
2107  #define cond_resched_rwlock_read(lock) ({			\
2108  	__might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);	\
2109  	__cond_resched_rwlock_read(lock);			\
2110  })
2111  
2112  #define cond_resched_rwlock_write(lock) ({			\
2113  	__might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);	\
2114  	__cond_resched_rwlock_write(lock);			\
2115  })
2116  
cond_resched_rcu(void)2117  static inline void cond_resched_rcu(void)
2118  {
2119  #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU)
2120  	rcu_read_unlock();
2121  	cond_resched();
2122  	rcu_read_lock();
2123  #endif
2124  }
2125  
2126  /*
2127   * Does a critical section need to be broken due to another
2128   * task waiting?: (technically does not depend on CONFIG_PREEMPTION,
2129   * but a general need for low latency)
2130   */
spin_needbreak(spinlock_t * lock)2131  static inline int spin_needbreak(spinlock_t *lock)
2132  {
2133  #ifdef CONFIG_PREEMPTION
2134  	return spin_is_contended(lock);
2135  #else
2136  	return 0;
2137  #endif
2138  }
2139  
2140  /*
2141   * Check if a rwlock is contended.
2142   * Returns non-zero if there is another task waiting on the rwlock.
2143   * Returns zero if the lock is not contended or the system / underlying
2144   * rwlock implementation does not support contention detection.
2145   * Technically does not depend on CONFIG_PREEMPTION, but a general need
2146   * for low latency.
2147   */
rwlock_needbreak(rwlock_t * lock)2148  static inline int rwlock_needbreak(rwlock_t *lock)
2149  {
2150  #ifdef CONFIG_PREEMPTION
2151  	return rwlock_is_contended(lock);
2152  #else
2153  	return 0;
2154  #endif
2155  }
2156  
need_resched(void)2157  static __always_inline bool need_resched(void)
2158  {
2159  	return unlikely(tif_need_resched());
2160  }
2161  
2162  /*
2163   * Wrappers for p->thread_info->cpu access. No-op on UP.
2164   */
2165  #ifdef CONFIG_SMP
2166  
task_cpu(const struct task_struct * p)2167  static inline unsigned int task_cpu(const struct task_struct *p)
2168  {
2169  #ifdef CONFIG_THREAD_INFO_IN_TASK
2170  	return READ_ONCE(p->cpu);
2171  #else
2172  	return READ_ONCE(task_thread_info(p)->cpu);
2173  #endif
2174  }
2175  
2176  extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
2177  
2178  #else
2179  
task_cpu(const struct task_struct * p)2180  static inline unsigned int task_cpu(const struct task_struct *p)
2181  {
2182  	return 0;
2183  }
2184  
set_task_cpu(struct task_struct * p,unsigned int cpu)2185  static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
2186  {
2187  }
2188  
2189  #endif /* CONFIG_SMP */
2190  
2191  extern bool sched_task_on_rq(struct task_struct *p);
2192  
2193  /*
2194   * In order to reduce various lock holder preemption latencies provide an
2195   * interface to see if a vCPU is currently running or not.
2196   *
2197   * This allows us to terminate optimistic spin loops and block, analogous to
2198   * the native optimistic spin heuristic of testing if the lock owner task is
2199   * running or not.
2200   */
2201  #ifndef vcpu_is_preempted
vcpu_is_preempted(int cpu)2202  static inline bool vcpu_is_preempted(int cpu)
2203  {
2204  	return false;
2205  }
2206  #endif
2207  
2208  extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
2209  extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
2210  
2211  #ifndef TASK_SIZE_OF
2212  #define TASK_SIZE_OF(tsk)	TASK_SIZE
2213  #endif
2214  
2215  #ifdef CONFIG_SMP
2216  /* Returns effective CPU energy utilization, as seen by the scheduler */
2217  unsigned long sched_cpu_util(int cpu, unsigned long max);
2218  #endif /* CONFIG_SMP */
2219  
2220  #ifdef CONFIG_RSEQ
2221  
2222  /*
2223   * Map the event mask on the user-space ABI enum rseq_cs_flags
2224   * for direct mask checks.
2225   */
2226  enum rseq_event_mask_bits {
2227  	RSEQ_EVENT_PREEMPT_BIT	= RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT,
2228  	RSEQ_EVENT_SIGNAL_BIT	= RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT,
2229  	RSEQ_EVENT_MIGRATE_BIT	= RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT,
2230  };
2231  
2232  enum rseq_event_mask {
2233  	RSEQ_EVENT_PREEMPT	= (1U << RSEQ_EVENT_PREEMPT_BIT),
2234  	RSEQ_EVENT_SIGNAL	= (1U << RSEQ_EVENT_SIGNAL_BIT),
2235  	RSEQ_EVENT_MIGRATE	= (1U << RSEQ_EVENT_MIGRATE_BIT),
2236  };
2237  
rseq_set_notify_resume(struct task_struct * t)2238  static inline void rseq_set_notify_resume(struct task_struct *t)
2239  {
2240  	if (t->rseq)
2241  		set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);
2242  }
2243  
2244  void __rseq_handle_notify_resume(struct ksignal *sig, struct pt_regs *regs);
2245  
rseq_handle_notify_resume(struct ksignal * ksig,struct pt_regs * regs)2246  static inline void rseq_handle_notify_resume(struct ksignal *ksig,
2247  					     struct pt_regs *regs)
2248  {
2249  	if (current->rseq)
2250  		__rseq_handle_notify_resume(ksig, regs);
2251  }
2252  
rseq_signal_deliver(struct ksignal * ksig,struct pt_regs * regs)2253  static inline void rseq_signal_deliver(struct ksignal *ksig,
2254  				       struct pt_regs *regs)
2255  {
2256  	preempt_disable();
2257  	__set_bit(RSEQ_EVENT_SIGNAL_BIT, &current->rseq_event_mask);
2258  	preempt_enable();
2259  	rseq_handle_notify_resume(ksig, regs);
2260  }
2261  
2262  /* rseq_preempt() requires preemption to be disabled. */
rseq_preempt(struct task_struct * t)2263  static inline void rseq_preempt(struct task_struct *t)
2264  {
2265  	__set_bit(RSEQ_EVENT_PREEMPT_BIT, &t->rseq_event_mask);
2266  	rseq_set_notify_resume(t);
2267  }
2268  
2269  /* rseq_migrate() requires preemption to be disabled. */
rseq_migrate(struct task_struct * t)2270  static inline void rseq_migrate(struct task_struct *t)
2271  {
2272  	__set_bit(RSEQ_EVENT_MIGRATE_BIT, &t->rseq_event_mask);
2273  	rseq_set_notify_resume(t);
2274  }
2275  
2276  /*
2277   * If parent process has a registered restartable sequences area, the
2278   * child inherits. Unregister rseq for a clone with CLONE_VM set.
2279   */
rseq_fork(struct task_struct * t,unsigned long clone_flags)2280  static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
2281  {
2282  	if (clone_flags & CLONE_VM) {
2283  		t->rseq = NULL;
2284  		t->rseq_sig = 0;
2285  		t->rseq_event_mask = 0;
2286  	} else {
2287  		t->rseq = current->rseq;
2288  		t->rseq_sig = current->rseq_sig;
2289  		t->rseq_event_mask = current->rseq_event_mask;
2290  	}
2291  }
2292  
rseq_execve(struct task_struct * t)2293  static inline void rseq_execve(struct task_struct *t)
2294  {
2295  	t->rseq = NULL;
2296  	t->rseq_sig = 0;
2297  	t->rseq_event_mask = 0;
2298  }
2299  
2300  #else
2301  
rseq_set_notify_resume(struct task_struct * t)2302  static inline void rseq_set_notify_resume(struct task_struct *t)
2303  {
2304  }
rseq_handle_notify_resume(struct ksignal * ksig,struct pt_regs * regs)2305  static inline void rseq_handle_notify_resume(struct ksignal *ksig,
2306  					     struct pt_regs *regs)
2307  {
2308  }
rseq_signal_deliver(struct ksignal * ksig,struct pt_regs * regs)2309  static inline void rseq_signal_deliver(struct ksignal *ksig,
2310  				       struct pt_regs *regs)
2311  {
2312  }
rseq_preempt(struct task_struct * t)2313  static inline void rseq_preempt(struct task_struct *t)
2314  {
2315  }
rseq_migrate(struct task_struct * t)2316  static inline void rseq_migrate(struct task_struct *t)
2317  {
2318  }
rseq_fork(struct task_struct * t,unsigned long clone_flags)2319  static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
2320  {
2321  }
rseq_execve(struct task_struct * t)2322  static inline void rseq_execve(struct task_struct *t)
2323  {
2324  }
2325  
2326  #endif
2327  
2328  #ifdef CONFIG_DEBUG_RSEQ
2329  
2330  void rseq_syscall(struct pt_regs *regs);
2331  
2332  #else
2333  
rseq_syscall(struct pt_regs * regs)2334  static inline void rseq_syscall(struct pt_regs *regs)
2335  {
2336  }
2337  
2338  #endif
2339  
2340  const struct sched_avg *sched_trace_cfs_rq_avg(struct cfs_rq *cfs_rq);
2341  char *sched_trace_cfs_rq_path(struct cfs_rq *cfs_rq, char *str, int len);
2342  int sched_trace_cfs_rq_cpu(struct cfs_rq *cfs_rq);
2343  
2344  const struct sched_avg *sched_trace_rq_avg_rt(struct rq *rq);
2345  const struct sched_avg *sched_trace_rq_avg_dl(struct rq *rq);
2346  const struct sched_avg *sched_trace_rq_avg_irq(struct rq *rq);
2347  
2348  int sched_trace_rq_cpu(struct rq *rq);
2349  int sched_trace_rq_cpu_capacity(struct rq *rq);
2350  int sched_trace_rq_nr_running(struct rq *rq);
2351  
2352  const struct cpumask *sched_trace_rd_span(struct root_domain *rd);
2353  
2354  #ifdef CONFIG_SCHED_CORE
2355  extern void sched_core_free(struct task_struct *tsk);
2356  extern void sched_core_fork(struct task_struct *p);
2357  extern int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
2358  				unsigned long uaddr);
2359  #else
sched_core_free(struct task_struct * tsk)2360  static inline void sched_core_free(struct task_struct *tsk) { }
sched_core_fork(struct task_struct * p)2361  static inline void sched_core_fork(struct task_struct *p) { }
2362  #endif
2363  
2364  #endif
2365