| /kernel/sched/ |
| D | sched.h | 89 struct rq;
101 extern void calc_global_load_tick(struct rq *this_rq);
102 extern long calc_load_fold_active(struct rq *this_rq, long adjust);
104 extern void call_trace_sched_update_nr_running(struct rq *rq, int count);
592 struct rq *rq; /* CPU runqueue to which this cfs_rq is attached */ member
663 struct rq *rq; member
856 extern void rq_attach_root(struct rq *rq, struct root_domain *rd);
863 extern struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu);
917 struct rq { struct
1092 static inline struct rq *rq_of(struct cfs_rq *cfs_rq) in rq_of() argument
[all …]
|
| D | pelt.h | 7 int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
8 int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
11 int update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity);
13 static inline u64 thermal_load_avg(struct rq *rq) in thermal_load_avg() argument
15 return READ_ONCE(rq->avg_thermal.load_avg); in thermal_load_avg()
19 update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity) in update_thermal_load_avg() argument
24 static inline u64 thermal_load_avg(struct rq *rq) in thermal_load_avg() argument
31 int update_irq_load_avg(struct rq *rq, u64 running);
34 update_irq_load_avg(struct rq *rq, u64 running) in update_irq_load_avg() argument
78 static inline void update_rq_clock_pelt(struct rq *rq, s64 delta) in update_rq_clock_pelt() argument
[all …]
|
| D | deadline.c | 29 static inline struct rq *rq_of_dl_rq(struct dl_rq *dl_rq) in rq_of_dl_rq()
31 return container_of(dl_rq, struct rq, dl); in rq_of_dl_rq()
37 struct rq *rq = task_rq(p); in dl_rq_of_se() local
39 return &rq->dl; in dl_rq_of_se()
220 struct rq *rq; in dl_change_utilization() local
227 rq = task_rq(p); in dl_change_utilization()
229 sub_running_bw(&p->dl, &rq->dl); in dl_change_utilization()
241 __sub_rq_bw(p->dl.dl_bw, &rq->dl); in dl_change_utilization()
242 __add_rq_bw(new_bw, &rq->dl); in dl_change_utilization()
304 struct rq *rq = rq_of_dl_rq(dl_rq); in task_non_contending() local
[all …]
|
| D | stats.h | 9 rq_sched_info_arrive(struct rq *rq, unsigned long long delta) in rq_sched_info_arrive() argument
11 if (rq) { in rq_sched_info_arrive()
12 rq->rq_sched_info.run_delay += delta; in rq_sched_info_arrive()
13 rq->rq_sched_info.pcount++; in rq_sched_info_arrive()
21 rq_sched_info_depart(struct rq *rq, unsigned long long delta) in rq_sched_info_depart() argument
23 if (rq) in rq_sched_info_depart()
24 rq->rq_cpu_time += delta; in rq_sched_info_depart()
28 rq_sched_info_dequeued(struct rq *rq, unsigned long long delta) in rq_sched_info_dequeued() argument
30 if (rq) in rq_sched_info_dequeued()
31 rq->rq_sched_info.run_delay += delta; in rq_sched_info_dequeued()
[all …]
|
| D | rt.c | 130 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) in rq_of_rt_rq()
132 return rt_rq->rq; in rq_of_rt_rq()
140 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) in rq_of_rt_se()
144 return rt_rq->rq; in rq_of_rt_se()
169 struct rq *rq = cpu_rq(cpu); in init_tg_rt_entry() local
173 rt_rq->rq = rq; in init_tg_rt_entry()
183 rt_se->rt_rq = &rq->rt; in init_tg_rt_entry()
241 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) in rq_of_rt_rq()
243 return container_of(rt_rq, struct rq, rt); in rq_of_rt_rq()
246 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) in rq_of_rt_se()
[all …]
|
| D | core.c | 57 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
194 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) in __task_rq_lock()
195 __acquires(rq->lock) in __task_rq_lock()
197 struct rq *rq; in __task_rq_lock() local
202 rq = task_rq(p); in __task_rq_lock()
203 raw_spin_lock(&rq->lock); in __task_rq_lock()
204 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { in __task_rq_lock()
205 rq_pin_lock(rq, rf); in __task_rq_lock()
206 return rq; in __task_rq_lock()
208 raw_spin_unlock(&rq->lock); in __task_rq_lock()
[all …]
|
| D | stop_task.c | 20 balance_stop(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) in balance_stop() argument
22 return sched_stop_runnable(rq); in balance_stop()
27 check_preempt_curr_stop(struct rq *rq, struct task_struct *p, int flags) in check_preempt_curr_stop() argument
32 static void set_next_task_stop(struct rq *rq, struct task_struct *stop, bool first) in set_next_task_stop() argument
34 stop->se.exec_start = rq_clock_task(rq); in set_next_task_stop()
37 static struct task_struct *pick_next_task_stop(struct rq *rq) in pick_next_task_stop() argument
39 if (!sched_stop_runnable(rq)) in pick_next_task_stop()
42 set_next_task_stop(rq, rq->stop, true); in pick_next_task_stop()
43 return rq->stop; in pick_next_task_stop()
47 enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags) in enqueue_task_stop() argument
[all …]
|
| D | fair.c | 309 struct rq *rq = rq_of(cfs_rq); in list_add_leaf_cfs_rq() local
310 int cpu = cpu_of(rq); in list_add_leaf_cfs_rq()
313 return rq->tmp_alone_branch == &rq->leaf_cfs_rq_list; in list_add_leaf_cfs_rq()
341 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; in list_add_leaf_cfs_rq()
351 &rq->leaf_cfs_rq_list); in list_add_leaf_cfs_rq()
356 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; in list_add_leaf_cfs_rq()
366 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, rq->tmp_alone_branch); in list_add_leaf_cfs_rq()
371 rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list; in list_add_leaf_cfs_rq()
378 struct rq *rq = rq_of(cfs_rq); in list_del_leaf_cfs_rq() local
387 if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list) in list_del_leaf_cfs_rq()
[all …]
|
| D | idle.c | 413 balance_idle(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) in balance_idle() argument
422 static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int flags) in check_preempt_curr_idle() argument
424 resched_curr(rq); in check_preempt_curr_idle()
427 static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) in put_prev_task_idle() argument
431 static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first) in set_next_task_idle() argument
433 update_idle_core(rq); in set_next_task_idle()
434 schedstat_inc(rq->sched_goidle); in set_next_task_idle()
437 struct task_struct *pick_next_task_idle(struct rq *rq) in pick_next_task_idle() argument
439 struct task_struct *next = rq->idle; in pick_next_task_idle()
441 set_next_task_idle(rq, next, true); in pick_next_task_idle()
[all …]
|
| D | pelt.c | 410 int update_rt_rq_load_avg(u64 now, struct rq *rq, int running) in update_rt_rq_load_avg() argument
412 if (___update_load_sum(now, &rq->avg_rt, in update_rt_rq_load_avg()
417 ___update_load_avg(&rq->avg_rt, 1); in update_rt_rq_load_avg()
418 trace_pelt_rt_tp(rq); in update_rt_rq_load_avg()
436 int update_dl_rq_load_avg(u64 now, struct rq *rq, int running) in update_dl_rq_load_avg() argument
438 if (___update_load_sum(now, &rq->avg_dl, in update_dl_rq_load_avg()
443 ___update_load_avg(&rq->avg_dl, 1); in update_dl_rq_load_avg()
444 trace_pelt_dl_tp(rq); in update_dl_rq_load_avg()
467 int update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity) in update_thermal_load_avg() argument
469 if (___update_load_sum(now, &rq->avg_thermal, in update_thermal_load_avg()
[all …]
|
| D | stats.c | 23 struct rq *rq; in show_schedstat() local
29 rq = cpu_rq(cpu); in show_schedstat()
34 cpu, rq->yld_count, in show_schedstat()
35 rq->sched_count, rq->sched_goidle, in show_schedstat()
36 rq->ttwu_count, rq->ttwu_local, in show_schedstat()
37 rq->rq_cpu_time, in show_schedstat()
38 rq->rq_sched_info.run_delay, rq->rq_sched_info.pcount); in show_schedstat()
|
| D | loadavg.c | 79 long calc_load_fold_active(struct rq *this_rq, long adjust) in calc_load_fold_active()
234 static void calc_load_nohz_fold(struct rq *rq) in calc_load_nohz_fold() argument
238 delta = calc_load_fold_active(rq, 0); in calc_load_nohz_fold()
259 void calc_load_nohz_remote(struct rq *rq) in calc_load_nohz_remote() argument
261 calc_load_nohz_fold(rq); in calc_load_nohz_remote()
266 struct rq *this_rq = this_rq(); in calc_load_nohz_stop()
386 void calc_global_load_tick(struct rq *this_rq) in calc_global_load_tick()
|
| D | debug.c | 508 print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) in print_task() argument
510 if (rq->curr == p) in print_task()
536 static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) in print_rq() argument
552 print_task(m, rq, p); in print_rq()
561 struct rq *rq = cpu_rq(cpu); in print_cfs_rq() local
575 raw_spin_lock_irqsave(&rq->lock, flags); in print_cfs_rq()
583 raw_spin_unlock_irqrestore(&rq->lock, flags); in print_cfs_rq()
689 struct rq *rq = cpu_rq(cpu); in print_cpu() local
704 if (sizeof(rq->x) == 4) \ in print_cpu()
705 SEQ_printf(m, " .%-30s: %ld\n", #x, (long)(rq->x)); \ in print_cpu()
[all …]
|
| D | cputime.c | 232 struct rq *rq = this_rq(); in account_idle_time() local
234 if (atomic_read(&rq->nr_iowait) > 0) in account_idle_time()
290 struct rq *rq; in read_sum_exec_runtime() local
292 rq = task_rq_lock(t, &rf); in read_sum_exec_runtime()
294 task_rq_unlock(rq, t, &rf); in read_sum_exec_runtime()
979 struct rq *rq; in kcpustat_field() local
985 rq = cpu_rq(cpu); in kcpustat_field()
991 curr = rcu_dereference(rq->curr); in kcpustat_field()
1064 struct rq *rq; in kcpustat_cpu_fetch() local
1072 rq = cpu_rq(cpu); in kcpustat_cpu_fetch()
[all …]
|
| D | cpufreq_schedutil.c | 207 struct rq *rq = cpu_rq(cpu); in schedutil_cpu_util() local
210 type == FREQUENCY_UTIL && rt_rq_is_runnable(&rq->rt)) { in schedutil_cpu_util()
219 irq = cpu_util_irq(rq); in schedutil_cpu_util()
235 util = util_cfs + cpu_util_rt(rq); in schedutil_cpu_util()
237 util = uclamp_rq_util_with(rq, util, p); in schedutil_cpu_util()
239 dl_util = cpu_util_dl(rq); in schedutil_cpu_util()
283 util += cpu_bw_dl(rq); in schedutil_cpu_util()
291 struct rq *rq = cpu_rq(sg_cpu->cpu); in sugov_get_util() local
292 unsigned long util = cpu_util_cfs(rq); in sugov_get_util()
296 sg_cpu->bw_dl = cpu_bw_dl(rq); in sugov_get_util()
|
| D | topology.c | 445 void rq_attach_root(struct rq *rq, struct root_domain *rd) in rq_attach_root() argument
450 raw_spin_lock_irqsave(&rq->lock, flags); in rq_attach_root()
452 if (rq->rd) { in rq_attach_root()
453 old_rd = rq->rd; in rq_attach_root()
455 if (cpumask_test_cpu(rq->cpu, old_rd->online)) in rq_attach_root()
456 set_rq_offline(rq); in rq_attach_root()
458 cpumask_clear_cpu(rq->cpu, old_rd->span); in rq_attach_root()
470 rq->rd = rd; in rq_attach_root()
472 cpumask_set_cpu(rq->cpu, rd->span); in rq_attach_root()
473 if (cpumask_test_cpu(rq->cpu, cpu_active_mask)) in rq_attach_root()
[all …]
|
| D | psi.c | 935 struct rq *rq; in psi_memstall_enter() local
948 rq = this_rq_lock_irq(&rf); in psi_memstall_enter()
953 rq_unlock_irq(rq, &rf); in psi_memstall_enter()
965 struct rq *rq; in psi_memstall_leave() local
977 rq = this_rq_lock_irq(&rf); in psi_memstall_leave()
982 rq_unlock_irq(rq, &rf); in psi_memstall_leave()
1025 struct rq *rq; in cgroup_move_task() local
1036 rq = task_rq_lock(task, &rf); in cgroup_move_task()
1073 task_rq_unlock(rq, task, &rf); in cgroup_move_task()
|
| D | membarrier.c | 335 struct rq *rq = cpu_rq(cpu); in sync_runqueues_membarrier_state() local
338 p = rcu_dereference(rq->curr); in sync_runqueues_membarrier_state()
|
| /kernel/trace/ |
| D | blktrace.c | 803 blk_trace_request_get_cgid(struct request_queue *q, struct request *rq) in blk_trace_request_get_cgid() argument
805 if (!rq->bio) in blk_trace_request_get_cgid()
808 return blk_trace_bio_get_cgid(q, rq->bio); in blk_trace_request_get_cgid()
827 static void blk_add_trace_rq(struct request *rq, int error, in blk_add_trace_rq() argument
833 bt = rcu_dereference(rq->q->blk_trace); in blk_add_trace_rq()
839 if (blk_rq_is_passthrough(rq)) in blk_add_trace_rq()
844 __blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, req_op(rq), in blk_add_trace_rq()
845 rq->cmd_flags, what, error, 0, NULL, cgid); in blk_add_trace_rq()
850 struct request_queue *q, struct request *rq) in blk_add_trace_rq_insert() argument
852 blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_INSERT, in blk_add_trace_rq_insert()
[all …]
|
| /kernel/livepatch/ |
| D | transition.c | 284 struct rq *rq; in klp_try_switch_task() local
307 rq = task_rq_lock(task, &flags); in klp_try_switch_task()
309 if (task_running(rq, task) && task != current) { in klp_try_switch_task()
326 task_rq_unlock(rq, task, &flags); in klp_try_switch_task()
|