1 /*
2 * kernel/sched/debug.c
3 *
4 * Print the CFS rbtree
5 *
6 * Copyright(C) 2007, Red Hat, Inc., Ingo Molnar
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13 #include <linux/proc_fs.h>
14 #include <linux/sched/mm.h>
15 #include <linux/sched/task.h>
16 #include <linux/seq_file.h>
17 #include <linux/kallsyms.h>
18 #include <linux/utsname.h>
19 #include <linux/mempolicy.h>
20 #include <linux/debugfs.h>
21
22 #include "sched.h"
23
24 static DEFINE_SPINLOCK(sched_debug_lock);
25
26 /*
27 * This allows printing both to /proc/sched_debug and
28 * to the console
29 */
30 #define SEQ_printf(m, x...) \
31 do { \
32 if (m) \
33 seq_printf(m, x); \
34 else \
35 printk(x); \
36 } while (0)
37
38 /*
39 * Ease the printing of nsec fields:
40 */
nsec_high(unsigned long long nsec)41 static long long nsec_high(unsigned long long nsec)
42 {
43 if ((long long)nsec < 0) {
44 nsec = -nsec;
45 do_div(nsec, 1000000);
46 return -nsec;
47 }
48 do_div(nsec, 1000000);
49
50 return nsec;
51 }
52
nsec_low(unsigned long long nsec)53 static unsigned long nsec_low(unsigned long long nsec)
54 {
55 if ((long long)nsec < 0)
56 nsec = -nsec;
57
58 return do_div(nsec, 1000000);
59 }
60
61 #define SPLIT_NS(x) nsec_high(x), nsec_low(x)
62
63 #define SCHED_FEAT(name, enabled) \
64 #name ,
65
66 static const char * const sched_feat_names[] = {
67 #include "features.h"
68 };
69
70 #undef SCHED_FEAT
71
sched_feat_show(struct seq_file * m,void * v)72 static int sched_feat_show(struct seq_file *m, void *v)
73 {
74 int i;
75
76 for (i = 0; i < __SCHED_FEAT_NR; i++) {
77 if (!(sysctl_sched_features & (1UL << i)))
78 seq_puts(m, "NO_");
79 seq_printf(m, "%s ", sched_feat_names[i]);
80 }
81 seq_puts(m, "\n");
82
83 return 0;
84 }
85
86 #ifdef HAVE_JUMP_LABEL
87
88 #define jump_label_key__true STATIC_KEY_INIT_TRUE
89 #define jump_label_key__false STATIC_KEY_INIT_FALSE
90
91 #define SCHED_FEAT(name, enabled) \
92 jump_label_key__##enabled ,
93
94 struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
95 #include "features.h"
96 };
97
98 #undef SCHED_FEAT
99
sched_feat_disable(int i)100 static void sched_feat_disable(int i)
101 {
102 static_key_disable(&sched_feat_keys[i]);
103 }
104
sched_feat_enable(int i)105 static void sched_feat_enable(int i)
106 {
107 static_key_enable(&sched_feat_keys[i]);
108 }
109 #else
sched_feat_disable(int i)110 static void sched_feat_disable(int i) { };
sched_feat_enable(int i)111 static void sched_feat_enable(int i) { };
112 #endif /* HAVE_JUMP_LABEL */
113
sched_feat_set(char * cmp)114 static int sched_feat_set(char *cmp)
115 {
116 int i;
117 int neg = 0;
118
119 if (strncmp(cmp, "NO_", 3) == 0) {
120 neg = 1;
121 cmp += 3;
122 }
123
124 for (i = 0; i < __SCHED_FEAT_NR; i++) {
125 if (strcmp(cmp, sched_feat_names[i]) == 0) {
126 if (neg) {
127 sysctl_sched_features &= ~(1UL << i);
128 sched_feat_disable(i);
129 } else {
130 sysctl_sched_features |= (1UL << i);
131 sched_feat_enable(i);
132 }
133 break;
134 }
135 }
136
137 return i;
138 }
139
140 static ssize_t
sched_feat_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)141 sched_feat_write(struct file *filp, const char __user *ubuf,
142 size_t cnt, loff_t *ppos)
143 {
144 char buf[64];
145 char *cmp;
146 int i;
147 struct inode *inode;
148
149 if (cnt > 63)
150 cnt = 63;
151
152 if (copy_from_user(&buf, ubuf, cnt))
153 return -EFAULT;
154
155 buf[cnt] = 0;
156 cmp = strstrip(buf);
157
158 /* Ensure the static_key remains in a consistent state */
159 inode = file_inode(filp);
160 inode_lock(inode);
161 i = sched_feat_set(cmp);
162 inode_unlock(inode);
163 if (i == __SCHED_FEAT_NR)
164 return -EINVAL;
165
166 *ppos += cnt;
167
168 return cnt;
169 }
170
sched_feat_open(struct inode * inode,struct file * filp)171 static int sched_feat_open(struct inode *inode, struct file *filp)
172 {
173 return single_open(filp, sched_feat_show, NULL);
174 }
175
176 static const struct file_operations sched_feat_fops = {
177 .open = sched_feat_open,
178 .write = sched_feat_write,
179 .read = seq_read,
180 .llseek = seq_lseek,
181 .release = single_release,
182 };
183
184 __read_mostly bool sched_debug_enabled;
185
sched_init_debug(void)186 static __init int sched_init_debug(void)
187 {
188 debugfs_create_file("sched_features", 0644, NULL, NULL,
189 &sched_feat_fops);
190
191 debugfs_create_bool("sched_debug", 0644, NULL,
192 &sched_debug_enabled);
193
194 return 0;
195 }
196 late_initcall(sched_init_debug);
197
198 #ifdef CONFIG_SMP
199
200 #ifdef CONFIG_SYSCTL
201
202 static struct ctl_table sd_ctl_dir[] = {
203 {
204 .procname = "sched_domain",
205 .mode = 0555,
206 },
207 {}
208 };
209
210 static struct ctl_table sd_ctl_root[] = {
211 {
212 .procname = "kernel",
213 .mode = 0555,
214 .child = sd_ctl_dir,
215 },
216 {}
217 };
218
sd_alloc_ctl_entry(int n)219 static struct ctl_table *sd_alloc_ctl_entry(int n)
220 {
221 struct ctl_table *entry =
222 kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
223
224 return entry;
225 }
226
sd_free_ctl_entry(struct ctl_table ** tablep)227 static void sd_free_ctl_entry(struct ctl_table **tablep)
228 {
229 struct ctl_table *entry;
230
231 /*
232 * In the intermediate directories, both the child directory and
233 * procname are dynamically allocated and could fail but the mode
234 * will always be set. In the lowest directory the names are
235 * static strings and all have proc handlers.
236 */
237 for (entry = *tablep; entry->mode; entry++) {
238 if (entry->child)
239 sd_free_ctl_entry(&entry->child);
240 if (entry->proc_handler == NULL)
241 kfree(entry->procname);
242 }
243
244 kfree(*tablep);
245 *tablep = NULL;
246 }
247
248 static int min_load_idx = 0;
249 static int max_load_idx = CPU_LOAD_IDX_MAX-1;
250
251 static void
set_table_entry(struct ctl_table * entry,const char * procname,void * data,int maxlen,umode_t mode,proc_handler * proc_handler,bool load_idx)252 set_table_entry(struct ctl_table *entry,
253 const char *procname, void *data, int maxlen,
254 umode_t mode, proc_handler *proc_handler,
255 bool load_idx)
256 {
257 entry->procname = procname;
258 entry->data = data;
259 entry->maxlen = maxlen;
260 entry->mode = mode;
261 entry->proc_handler = proc_handler;
262
263 if (load_idx) {
264 entry->extra1 = &min_load_idx;
265 entry->extra2 = &max_load_idx;
266 }
267 }
268
269 static struct ctl_table *
sd_alloc_ctl_energy_table(struct sched_group_energy * sge)270 sd_alloc_ctl_energy_table(struct sched_group_energy *sge)
271 {
272 struct ctl_table *table = sd_alloc_ctl_entry(5);
273
274 if (table == NULL)
275 return NULL;
276
277 set_table_entry(&table[0], "nr_idle_states", &sge->nr_idle_states,
278 sizeof(int), 0444, proc_dointvec_minmax, false);
279 set_table_entry(&table[1], "idle_states", &sge->idle_states[0].power,
280 sge->nr_idle_states*sizeof(struct idle_state), 0444,
281 proc_doulongvec_minmax, false);
282 set_table_entry(&table[2], "nr_cap_states", &sge->nr_cap_states,
283 sizeof(int), 0444, proc_dointvec_minmax, false);
284 set_table_entry(&table[3], "cap_states", &sge->cap_states[0].cap,
285 sge->nr_cap_states*sizeof(struct capacity_state), 0444,
286 proc_doulongvec_minmax, false);
287
288 return table;
289 }
290
291 static struct ctl_table *
sd_alloc_ctl_group_table(struct sched_group * sg)292 sd_alloc_ctl_group_table(struct sched_group *sg)
293 {
294 struct ctl_table *table = sd_alloc_ctl_entry(2);
295
296 if (table == NULL)
297 return NULL;
298
299 table->procname = kstrdup("energy", GFP_KERNEL);
300 table->mode = 0555;
301 table->child = sd_alloc_ctl_energy_table((struct sched_group_energy *)sg->sge);
302
303 return table;
304 }
305
306 static struct ctl_table *
sd_alloc_ctl_domain_table(struct sched_domain * sd)307 sd_alloc_ctl_domain_table(struct sched_domain *sd)
308 {
309 struct ctl_table *table;
310 unsigned int nr_entries = 14;
311
312 int i = 0;
313 struct sched_group *sg = sd->groups;
314
315 if (sg->sge) {
316 int nr_sgs = 0;
317
318 do {} while (nr_sgs++, sg = sg->next, sg != sd->groups);
319
320 nr_entries += nr_sgs;
321 }
322
323 table = sd_alloc_ctl_entry(nr_entries);
324
325 if (table == NULL)
326 return NULL;
327
328 set_table_entry(&table[0], "min_interval", &sd->min_interval,
329 sizeof(long), 0644, proc_doulongvec_minmax, false);
330 set_table_entry(&table[1], "max_interval", &sd->max_interval,
331 sizeof(long), 0644, proc_doulongvec_minmax, false);
332 set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
333 sizeof(int), 0644, proc_dointvec_minmax, true);
334 set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
335 sizeof(int), 0644, proc_dointvec_minmax, true);
336 set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
337 sizeof(int), 0644, proc_dointvec_minmax, true);
338 set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
339 sizeof(int), 0644, proc_dointvec_minmax, true);
340 set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
341 sizeof(int), 0644, proc_dointvec_minmax, true);
342 set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
343 sizeof(int), 0644, proc_dointvec_minmax, false);
344 set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
345 sizeof(int), 0644, proc_dointvec_minmax, false);
346 set_table_entry(&table[9], "cache_nice_tries",
347 &sd->cache_nice_tries,
348 sizeof(int), 0644, proc_dointvec_minmax, false);
349 set_table_entry(&table[10], "flags", &sd->flags,
350 sizeof(int), 0644, proc_dointvec_minmax, false);
351 set_table_entry(&table[11], "max_newidle_lb_cost",
352 &sd->max_newidle_lb_cost,
353 sizeof(long), 0644, proc_doulongvec_minmax, false);
354 set_table_entry(&table[12], "name", sd->name,
355 CORENAME_MAX_SIZE, 0444, proc_dostring, false);
356 sg = sd->groups;
357 if (sg->sge) {
358 char buf[32];
359 struct ctl_table *entry = &table[13];
360
361 do {
362 snprintf(buf, 32, "group%d", i);
363 entry->procname = kstrdup(buf, GFP_KERNEL);
364 entry->mode = 0555;
365 entry->child = sd_alloc_ctl_group_table(sg);
366 } while (entry++, i++, sg = sg->next, sg != sd->groups);
367 }
368 /* &table[nr_entries-1] is terminator */
369
370 return table;
371 }
372
sd_alloc_ctl_cpu_table(int cpu)373 static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu)
374 {
375 struct ctl_table *entry, *table;
376 struct sched_domain *sd;
377 int domain_num = 0, i;
378 char buf[32];
379
380 for_each_domain(cpu, sd)
381 domain_num++;
382 entry = table = sd_alloc_ctl_entry(domain_num + 1);
383 if (table == NULL)
384 return NULL;
385
386 i = 0;
387 for_each_domain(cpu, sd) {
388 snprintf(buf, 32, "domain%d", i);
389 entry->procname = kstrdup(buf, GFP_KERNEL);
390 entry->mode = 0555;
391 entry->child = sd_alloc_ctl_domain_table(sd);
392 entry++;
393 i++;
394 }
395 return table;
396 }
397
398 static cpumask_var_t sd_sysctl_cpus;
399 static struct ctl_table_header *sd_sysctl_header;
400
register_sched_domain_sysctl(void)401 void register_sched_domain_sysctl(void)
402 {
403 static struct ctl_table *cpu_entries;
404 static struct ctl_table **cpu_idx;
405 static bool init_done = false;
406 char buf[32];
407 int i;
408
409 if (!cpu_entries) {
410 cpu_entries = sd_alloc_ctl_entry(num_possible_cpus() + 1);
411 if (!cpu_entries)
412 return;
413
414 WARN_ON(sd_ctl_dir[0].child);
415 sd_ctl_dir[0].child = cpu_entries;
416 }
417
418 if (!cpu_idx) {
419 struct ctl_table *e = cpu_entries;
420
421 cpu_idx = kcalloc(nr_cpu_ids, sizeof(struct ctl_table*), GFP_KERNEL);
422 if (!cpu_idx)
423 return;
424
425 /* deal with sparse possible map */
426 for_each_possible_cpu(i) {
427 cpu_idx[i] = e;
428 e++;
429 }
430 }
431
432 if (!cpumask_available(sd_sysctl_cpus)) {
433 if (!alloc_cpumask_var(&sd_sysctl_cpus, GFP_KERNEL))
434 return;
435 }
436
437 if (!init_done) {
438 init_done = true;
439 /* init to possible to not have holes in @cpu_entries */
440 cpumask_copy(sd_sysctl_cpus, cpu_possible_mask);
441 }
442
443 for_each_cpu(i, sd_sysctl_cpus) {
444 struct ctl_table *e = cpu_idx[i];
445
446 if (e->child)
447 sd_free_ctl_entry(&e->child);
448
449 if (!e->procname) {
450 snprintf(buf, 32, "cpu%d", i);
451 e->procname = kstrdup(buf, GFP_KERNEL);
452 }
453 e->mode = 0555;
454 e->child = sd_alloc_ctl_cpu_table(i);
455
456 __cpumask_clear_cpu(i, sd_sysctl_cpus);
457 }
458
459 WARN_ON(sd_sysctl_header);
460 sd_sysctl_header = register_sysctl_table(sd_ctl_root);
461 }
462
dirty_sched_domain_sysctl(int cpu)463 void dirty_sched_domain_sysctl(int cpu)
464 {
465 if (cpumask_available(sd_sysctl_cpus))
466 __cpumask_set_cpu(cpu, sd_sysctl_cpus);
467 }
468
469 /* may be called multiple times per register */
unregister_sched_domain_sysctl(void)470 void unregister_sched_domain_sysctl(void)
471 {
472 unregister_sysctl_table(sd_sysctl_header);
473 sd_sysctl_header = NULL;
474 }
475 #endif /* CONFIG_SYSCTL */
476 #endif /* CONFIG_SMP */
477
478 #ifdef CONFIG_FAIR_GROUP_SCHED
print_cfs_group_stats(struct seq_file * m,int cpu,struct task_group * tg)479 static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group *tg)
480 {
481 struct sched_entity *se = tg->se[cpu];
482
483 #define P(F) \
484 SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F)
485 #define P_SCHEDSTAT(F) \
486 SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)schedstat_val(F))
487 #define PN(F) \
488 SEQ_printf(m, " .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)F))
489 #define PN_SCHEDSTAT(F) \
490 SEQ_printf(m, " .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)schedstat_val(F)))
491
492 if (!se)
493 return;
494
495 PN(se->exec_start);
496 PN(se->vruntime);
497 PN(se->sum_exec_runtime);
498 if (schedstat_enabled()) {
499 PN_SCHEDSTAT(se->statistics.wait_start);
500 PN_SCHEDSTAT(se->statistics.sleep_start);
501 PN_SCHEDSTAT(se->statistics.block_start);
502 PN_SCHEDSTAT(se->statistics.sleep_max);
503 PN_SCHEDSTAT(se->statistics.block_max);
504 PN_SCHEDSTAT(se->statistics.exec_max);
505 PN_SCHEDSTAT(se->statistics.slice_max);
506 PN_SCHEDSTAT(se->statistics.wait_max);
507 PN_SCHEDSTAT(se->statistics.wait_sum);
508 P_SCHEDSTAT(se->statistics.wait_count);
509 }
510 P(se->load.weight);
511 #ifdef CONFIG_SMP
512 P(se->avg.load_avg);
513 P(se->avg.util_avg);
514 #endif
515
516 #undef PN_SCHEDSTAT
517 #undef PN
518 #undef P_SCHEDSTAT
519 #undef P
520 }
521 #endif
522
523 #ifdef CONFIG_CGROUP_SCHED
524 static char group_path[PATH_MAX];
525
task_group_path(struct task_group * tg)526 static char *task_group_path(struct task_group *tg)
527 {
528 if (autogroup_path(tg, group_path, PATH_MAX))
529 return group_path;
530
531 cgroup_path(tg->css.cgroup, group_path, PATH_MAX);
532 return group_path;
533 }
534 #endif
535
536 static void
print_task(struct seq_file * m,struct rq * rq,struct task_struct * p)537 print_task(struct seq_file *m, struct rq *rq, struct task_struct *p)
538 {
539 if (rq->curr == p)
540 SEQ_printf(m, ">R");
541 else
542 SEQ_printf(m, " %c", task_state_to_char(p));
543
544 SEQ_printf(m, "%15s %5d %9Ld.%06ld %9Ld %5d ",
545 p->comm, task_pid_nr(p),
546 SPLIT_NS(p->se.vruntime),
547 (long long)(p->nvcsw + p->nivcsw),
548 p->prio);
549
550 SEQ_printf(m, "%9Ld.%06ld %9Ld.%06ld %9Ld.%06ld",
551 SPLIT_NS(schedstat_val_or_zero(p->se.statistics.wait_sum)),
552 SPLIT_NS(p->se.sum_exec_runtime),
553 SPLIT_NS(schedstat_val_or_zero(p->se.statistics.sum_sleep_runtime)));
554
555 #ifdef CONFIG_NUMA_BALANCING
556 SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));
557 #endif
558 #ifdef CONFIG_CGROUP_SCHED
559 SEQ_printf(m, " %s", task_group_path(task_group(p)));
560 #endif
561
562 SEQ_printf(m, "\n");
563 }
564
print_rq(struct seq_file * m,struct rq * rq,int rq_cpu)565 static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu)
566 {
567 struct task_struct *g, *p;
568
569 SEQ_printf(m,
570 "\nrunnable tasks:\n"
571 " S task PID tree-key switches prio"
572 " wait-time sum-exec sum-sleep\n"
573 "-------------------------------------------------------"
574 "----------------------------------------------------\n");
575
576 rcu_read_lock();
577 for_each_process_thread(g, p) {
578 if (task_cpu(p) != rq_cpu)
579 continue;
580
581 print_task(m, rq, p);
582 }
583 rcu_read_unlock();
584 }
585
print_cfs_rq(struct seq_file * m,int cpu,struct cfs_rq * cfs_rq)586 void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
587 {
588 s64 MIN_vruntime = -1, min_vruntime, max_vruntime = -1,
589 spread, rq0_min_vruntime, spread0;
590 struct rq *rq = cpu_rq(cpu);
591 struct sched_entity *last;
592 unsigned long flags;
593
594 #ifdef CONFIG_FAIR_GROUP_SCHED
595 SEQ_printf(m, "\ncfs_rq[%d]:%s\n", cpu, task_group_path(cfs_rq->tg));
596 #else
597 SEQ_printf(m, "\ncfs_rq[%d]:\n", cpu);
598 #endif
599 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock",
600 SPLIT_NS(cfs_rq->exec_clock));
601
602 raw_spin_lock_irqsave(&rq->lock, flags);
603 if (rb_first_cached(&cfs_rq->tasks_timeline))
604 MIN_vruntime = (__pick_first_entity(cfs_rq))->vruntime;
605 last = __pick_last_entity(cfs_rq);
606 if (last)
607 max_vruntime = last->vruntime;
608 min_vruntime = cfs_rq->min_vruntime;
609 rq0_min_vruntime = cpu_rq(0)->cfs.min_vruntime;
610 raw_spin_unlock_irqrestore(&rq->lock, flags);
611 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "MIN_vruntime",
612 SPLIT_NS(MIN_vruntime));
613 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "min_vruntime",
614 SPLIT_NS(min_vruntime));
615 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "max_vruntime",
616 SPLIT_NS(max_vruntime));
617 spread = max_vruntime - MIN_vruntime;
618 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread",
619 SPLIT_NS(spread));
620 spread0 = min_vruntime - rq0_min_vruntime;
621 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread0",
622 SPLIT_NS(spread0));
623 SEQ_printf(m, " .%-30s: %d\n", "nr_spread_over",
624 cfs_rq->nr_spread_over);
625 SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running);
626 SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight);
627 #ifdef CONFIG_SMP
628 SEQ_printf(m, " .%-30s: %lu\n", "load_avg",
629 cfs_rq->avg.load_avg);
630 SEQ_printf(m, " .%-30s: %lu\n", "runnable_load_avg",
631 cfs_rq->runnable_load_avg);
632 SEQ_printf(m, " .%-30s: %lu\n", "util_avg",
633 cfs_rq->avg.util_avg);
634 SEQ_printf(m, " .%-30s: %u\n", "util_est_enqueued",
635 cfs_rq->avg.util_est.enqueued);
636 SEQ_printf(m, " .%-30s: %ld\n", "removed_load_avg",
637 atomic_long_read(&cfs_rq->removed_load_avg));
638 SEQ_printf(m, " .%-30s: %ld\n", "removed_util_avg",
639 atomic_long_read(&cfs_rq->removed_util_avg));
640 #ifdef CONFIG_FAIR_GROUP_SCHED
641 SEQ_printf(m, " .%-30s: %lu\n", "tg_load_avg_contrib",
642 cfs_rq->tg_load_avg_contrib);
643 SEQ_printf(m, " .%-30s: %ld\n", "tg_load_avg",
644 atomic_long_read(&cfs_rq->tg->load_avg));
645 #endif
646 #endif
647 #ifdef CONFIG_CFS_BANDWIDTH
648 SEQ_printf(m, " .%-30s: %d\n", "throttled",
649 cfs_rq->throttled);
650 SEQ_printf(m, " .%-30s: %d\n", "throttle_count",
651 cfs_rq->throttle_count);
652 #endif
653
654 #ifdef CONFIG_FAIR_GROUP_SCHED
655 print_cfs_group_stats(m, cpu, cfs_rq->tg);
656 #endif
657 }
658
print_rt_rq(struct seq_file * m,int cpu,struct rt_rq * rt_rq)659 void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq)
660 {
661 #ifdef CONFIG_RT_GROUP_SCHED
662 SEQ_printf(m, "\nrt_rq[%d]:%s\n", cpu, task_group_path(rt_rq->tg));
663 #else
664 SEQ_printf(m, "\nrt_rq[%d]:\n", cpu);
665 #endif
666
667 #define P(x) \
668 SEQ_printf(m, " .%-30s: %Ld\n", #x, (long long)(rt_rq->x))
669 #define PU(x) \
670 SEQ_printf(m, " .%-30s: %lu\n", #x, (unsigned long)(rt_rq->x))
671 #define PN(x) \
672 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rt_rq->x))
673
674 PU(rt_nr_running);
675 #ifdef CONFIG_SMP
676 PU(rt_nr_migratory);
677 #endif
678 P(rt_throttled);
679 PN(rt_time);
680 PN(rt_runtime);
681
682 #undef PN
683 #undef PU
684 #undef P
685 }
686
print_dl_rq(struct seq_file * m,int cpu,struct dl_rq * dl_rq)687 void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq)
688 {
689 struct dl_bw *dl_bw;
690
691 SEQ_printf(m, "\ndl_rq[%d]:\n", cpu);
692
693 #define PU(x) \
694 SEQ_printf(m, " .%-30s: %lu\n", #x, (unsigned long)(dl_rq->x))
695
696 PU(dl_nr_running);
697 #ifdef CONFIG_SMP
698 PU(dl_nr_migratory);
699 dl_bw = &cpu_rq(cpu)->rd->dl_bw;
700 #else
701 dl_bw = &dl_rq->dl_bw;
702 #endif
703 SEQ_printf(m, " .%-30s: %lld\n", "dl_bw->bw", dl_bw->bw);
704 SEQ_printf(m, " .%-30s: %lld\n", "dl_bw->total_bw", dl_bw->total_bw);
705
706 #undef PU
707 }
708
709 extern __read_mostly int sched_clock_running;
710
print_cpu(struct seq_file * m,int cpu)711 static void print_cpu(struct seq_file *m, int cpu)
712 {
713 struct rq *rq = cpu_rq(cpu);
714 unsigned long flags;
715
716 #ifdef CONFIG_X86
717 {
718 unsigned int freq = cpu_khz ? : 1;
719
720 SEQ_printf(m, "cpu#%d, %u.%03u MHz\n",
721 cpu, freq / 1000, (freq % 1000));
722 }
723 #else
724 SEQ_printf(m, "cpu#%d\n", cpu);
725 #endif
726
727 #define P(x) \
728 do { \
729 if (sizeof(rq->x) == 4) \
730 SEQ_printf(m, " .%-30s: %ld\n", #x, (long)(rq->x)); \
731 else \
732 SEQ_printf(m, " .%-30s: %Ld\n", #x, (long long)(rq->x));\
733 } while (0)
734
735 #define PN(x) \
736 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rq->x))
737
738 P(nr_running);
739 SEQ_printf(m, " .%-30s: %lu\n", "load",
740 rq->load.weight);
741 P(nr_switches);
742 P(nr_load_updates);
743 P(nr_uninterruptible);
744 PN(next_balance);
745 SEQ_printf(m, " .%-30s: %ld\n", "curr->pid", (long)(task_pid_nr(rq->curr)));
746 PN(clock);
747 PN(clock_task);
748 P(cpu_load[0]);
749 P(cpu_load[1]);
750 P(cpu_load[2]);
751 P(cpu_load[3]);
752 P(cpu_load[4]);
753 #undef P
754 #undef PN
755
756 #ifdef CONFIG_SMP
757 #define P64(n) SEQ_printf(m, " .%-30s: %Ld\n", #n, rq->n);
758 P64(avg_idle);
759 P64(max_idle_balance_cost);
760 #undef P64
761 #endif
762
763 #define P(n) SEQ_printf(m, " .%-30s: %d\n", #n, schedstat_val(rq->n));
764 if (schedstat_enabled()) {
765 P(yld_count);
766 P(sched_count);
767 P(sched_goidle);
768 P(ttwu_count);
769 P(ttwu_local);
770 }
771 #undef P
772
773 spin_lock_irqsave(&sched_debug_lock, flags);
774 print_cfs_stats(m, cpu);
775 print_rt_stats(m, cpu);
776 print_dl_stats(m, cpu);
777
778 print_rq(m, rq, cpu);
779 spin_unlock_irqrestore(&sched_debug_lock, flags);
780 SEQ_printf(m, "\n");
781 }
782
783 static const char *sched_tunable_scaling_names[] = {
784 "none",
785 "logaritmic",
786 "linear"
787 };
788
sched_debug_header(struct seq_file * m)789 static void sched_debug_header(struct seq_file *m)
790 {
791 u64 ktime, sched_clk, cpu_clk;
792 unsigned long flags;
793
794 local_irq_save(flags);
795 ktime = ktime_to_ns(ktime_get());
796 sched_clk = sched_clock();
797 cpu_clk = local_clock();
798 local_irq_restore(flags);
799
800 SEQ_printf(m, "Sched Debug Version: v0.11, %s %.*s\n",
801 init_utsname()->release,
802 (int)strcspn(init_utsname()->version, " "),
803 init_utsname()->version);
804
805 #define P(x) \
806 SEQ_printf(m, "%-40s: %Ld\n", #x, (long long)(x))
807 #define PN(x) \
808 SEQ_printf(m, "%-40s: %Ld.%06ld\n", #x, SPLIT_NS(x))
809 PN(ktime);
810 PN(sched_clk);
811 PN(cpu_clk);
812 P(jiffies);
813 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
814 P(sched_clock_stable());
815 #endif
816 #undef PN
817 #undef P
818
819 SEQ_printf(m, "\n");
820 SEQ_printf(m, "sysctl_sched\n");
821
822 #define P(x) \
823 SEQ_printf(m, " .%-40s: %Ld\n", #x, (long long)(x))
824 #define PN(x) \
825 SEQ_printf(m, " .%-40s: %Ld.%06ld\n", #x, SPLIT_NS(x))
826 PN(sysctl_sched_latency);
827 PN(sysctl_sched_min_granularity);
828 PN(sysctl_sched_wakeup_granularity);
829 P(sysctl_sched_child_runs_first);
830 P(sysctl_sched_features);
831 #undef PN
832 #undef P
833
834 SEQ_printf(m, " .%-40s: %d (%s)\n",
835 "sysctl_sched_tunable_scaling",
836 sysctl_sched_tunable_scaling,
837 sched_tunable_scaling_names[sysctl_sched_tunable_scaling]);
838 SEQ_printf(m, "\n");
839 }
840
sched_debug_show(struct seq_file * m,void * v)841 static int sched_debug_show(struct seq_file *m, void *v)
842 {
843 int cpu = (unsigned long)(v - 2);
844
845 if (cpu != -1)
846 print_cpu(m, cpu);
847 else
848 sched_debug_header(m);
849
850 return 0;
851 }
852
sysrq_sched_debug_show(void)853 void sysrq_sched_debug_show(void)
854 {
855 int cpu;
856
857 sched_debug_header(NULL);
858 for_each_online_cpu(cpu)
859 print_cpu(NULL, cpu);
860
861 }
862
863 /*
864 * This itererator needs some explanation.
865 * It returns 1 for the header position.
866 * This means 2 is cpu 0.
867 * In a hotplugged system some cpus, including cpu 0, may be missing so we have
868 * to use cpumask_* to iterate over the cpus.
869 */
sched_debug_start(struct seq_file * file,loff_t * offset)870 static void *sched_debug_start(struct seq_file *file, loff_t *offset)
871 {
872 unsigned long n = *offset;
873
874 if (n == 0)
875 return (void *) 1;
876
877 n--;
878
879 if (n > 0)
880 n = cpumask_next(n - 1, cpu_online_mask);
881 else
882 n = cpumask_first(cpu_online_mask);
883
884 *offset = n + 1;
885
886 if (n < nr_cpu_ids)
887 return (void *)(unsigned long)(n + 2);
888 return NULL;
889 }
890
sched_debug_next(struct seq_file * file,void * data,loff_t * offset)891 static void *sched_debug_next(struct seq_file *file, void *data, loff_t *offset)
892 {
893 (*offset)++;
894 return sched_debug_start(file, offset);
895 }
896
sched_debug_stop(struct seq_file * file,void * data)897 static void sched_debug_stop(struct seq_file *file, void *data)
898 {
899 }
900
901 static const struct seq_operations sched_debug_sops = {
902 .start = sched_debug_start,
903 .next = sched_debug_next,
904 .stop = sched_debug_stop,
905 .show = sched_debug_show,
906 };
907
sched_debug_release(struct inode * inode,struct file * file)908 static int sched_debug_release(struct inode *inode, struct file *file)
909 {
910 seq_release(inode, file);
911
912 return 0;
913 }
914
sched_debug_open(struct inode * inode,struct file * filp)915 static int sched_debug_open(struct inode *inode, struct file *filp)
916 {
917 int ret = 0;
918
919 ret = seq_open(filp, &sched_debug_sops);
920
921 return ret;
922 }
923
924 static const struct file_operations sched_debug_fops = {
925 .open = sched_debug_open,
926 .read = seq_read,
927 .llseek = seq_lseek,
928 .release = sched_debug_release,
929 };
930
init_sched_debug_procfs(void)931 static int __init init_sched_debug_procfs(void)
932 {
933 struct proc_dir_entry *pe;
934
935 pe = proc_create("sched_debug", 0444, NULL, &sched_debug_fops);
936 if (!pe)
937 return -ENOMEM;
938 return 0;
939 }
940
941 __initcall(init_sched_debug_procfs);
942
943 #define __P(F) \
944 SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)F)
945 #define P(F) \
946 SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)p->F)
947 #define __PN(F) \
948 SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F))
949 #define PN(F) \
950 SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F))
951
952
953 #ifdef CONFIG_NUMA_BALANCING
print_numa_stats(struct seq_file * m,int node,unsigned long tsf,unsigned long tpf,unsigned long gsf,unsigned long gpf)954 void print_numa_stats(struct seq_file *m, int node, unsigned long tsf,
955 unsigned long tpf, unsigned long gsf, unsigned long gpf)
956 {
957 SEQ_printf(m, "numa_faults node=%d ", node);
958 SEQ_printf(m, "task_private=%lu task_shared=%lu ", tsf, tpf);
959 SEQ_printf(m, "group_private=%lu group_shared=%lu\n", gsf, gpf);
960 }
961 #endif
962
963
sched_show_numa(struct task_struct * p,struct seq_file * m)964 static void sched_show_numa(struct task_struct *p, struct seq_file *m)
965 {
966 #ifdef CONFIG_NUMA_BALANCING
967 struct mempolicy *pol;
968
969 if (p->mm)
970 P(mm->numa_scan_seq);
971
972 task_lock(p);
973 pol = p->mempolicy;
974 if (pol && !(pol->flags & MPOL_F_MORON))
975 pol = NULL;
976 mpol_get(pol);
977 task_unlock(p);
978
979 P(numa_pages_migrated);
980 P(numa_preferred_nid);
981 P(total_numa_faults);
982 SEQ_printf(m, "current_node=%d, numa_group_id=%d\n",
983 task_node(p), task_numa_group_id(p));
984 show_numa_stats(p, m);
985 mpol_put(pol);
986 #endif
987 }
988
proc_sched_show_task(struct task_struct * p,struct pid_namespace * ns,struct seq_file * m)989 void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
990 struct seq_file *m)
991 {
992 unsigned long nr_switches;
993
994 SEQ_printf(m, "%s (%d, #threads: %d)\n", p->comm, task_pid_nr_ns(p, ns),
995 get_nr_threads(p));
996 SEQ_printf(m,
997 "---------------------------------------------------------"
998 "----------\n");
999 #define __P(F) \
1000 SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)F)
1001 #define P(F) \
1002 SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)p->F)
1003 #define P_SCHEDSTAT(F) \
1004 SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)schedstat_val(p->F))
1005 #define __PN(F) \
1006 SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F))
1007 #define PN(F) \
1008 SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F))
1009 #define PN_SCHEDSTAT(F) \
1010 SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)schedstat_val(p->F)))
1011
1012 PN(se.exec_start);
1013 PN(se.vruntime);
1014 PN(se.sum_exec_runtime);
1015
1016 nr_switches = p->nvcsw + p->nivcsw;
1017
1018 P(se.nr_migrations);
1019
1020 if (schedstat_enabled()) {
1021 u64 avg_atom, avg_per_cpu;
1022
1023 PN_SCHEDSTAT(se.statistics.sum_sleep_runtime);
1024 PN_SCHEDSTAT(se.statistics.wait_start);
1025 PN_SCHEDSTAT(se.statistics.sleep_start);
1026 PN_SCHEDSTAT(se.statistics.block_start);
1027 PN_SCHEDSTAT(se.statistics.sleep_max);
1028 PN_SCHEDSTAT(se.statistics.block_max);
1029 PN_SCHEDSTAT(se.statistics.exec_max);
1030 PN_SCHEDSTAT(se.statistics.slice_max);
1031 PN_SCHEDSTAT(se.statistics.wait_max);
1032 PN_SCHEDSTAT(se.statistics.wait_sum);
1033 P_SCHEDSTAT(se.statistics.wait_count);
1034 PN_SCHEDSTAT(se.statistics.iowait_sum);
1035 P_SCHEDSTAT(se.statistics.iowait_count);
1036 P_SCHEDSTAT(se.statistics.nr_migrations_cold);
1037 P_SCHEDSTAT(se.statistics.nr_failed_migrations_affine);
1038 P_SCHEDSTAT(se.statistics.nr_failed_migrations_running);
1039 P_SCHEDSTAT(se.statistics.nr_failed_migrations_hot);
1040 P_SCHEDSTAT(se.statistics.nr_forced_migrations);
1041 P_SCHEDSTAT(se.statistics.nr_wakeups);
1042 P_SCHEDSTAT(se.statistics.nr_wakeups_sync);
1043 P_SCHEDSTAT(se.statistics.nr_wakeups_migrate);
1044 P_SCHEDSTAT(se.statistics.nr_wakeups_local);
1045 P_SCHEDSTAT(se.statistics.nr_wakeups_remote);
1046 P_SCHEDSTAT(se.statistics.nr_wakeups_affine);
1047 P_SCHEDSTAT(se.statistics.nr_wakeups_affine_attempts);
1048 P_SCHEDSTAT(se.statistics.nr_wakeups_passive);
1049 P_SCHEDSTAT(se.statistics.nr_wakeups_idle);
1050
1051 avg_atom = p->se.sum_exec_runtime;
1052 if (nr_switches)
1053 avg_atom = div64_ul(avg_atom, nr_switches);
1054 else
1055 avg_atom = -1LL;
1056
1057 avg_per_cpu = p->se.sum_exec_runtime;
1058 if (p->se.nr_migrations) {
1059 avg_per_cpu = div64_u64(avg_per_cpu,
1060 p->se.nr_migrations);
1061 } else {
1062 avg_per_cpu = -1LL;
1063 }
1064
1065 __PN(avg_atom);
1066 __PN(avg_per_cpu);
1067 }
1068
1069 __P(nr_switches);
1070 SEQ_printf(m, "%-45s:%21Ld\n",
1071 "nr_voluntary_switches", (long long)p->nvcsw);
1072 SEQ_printf(m, "%-45s:%21Ld\n",
1073 "nr_involuntary_switches", (long long)p->nivcsw);
1074
1075 P(se.load.weight);
1076 #ifdef CONFIG_SMP
1077 P(se.avg.load_sum);
1078 P(se.avg.util_sum);
1079 P(se.avg.load_avg);
1080 P(se.avg.util_avg);
1081 P(se.avg.last_update_time);
1082 P(se.avg.util_est.ewma);
1083 P(se.avg.util_est.enqueued);
1084 #endif
1085 P(policy);
1086 P(prio);
1087 if (p->policy == SCHED_DEADLINE) {
1088 P(dl.runtime);
1089 P(dl.deadline);
1090 }
1091 #undef PN_SCHEDSTAT
1092 #undef PN
1093 #undef __PN
1094 #undef P_SCHEDSTAT
1095 #undef P
1096 #undef __P
1097
1098 {
1099 unsigned int this_cpu = raw_smp_processor_id();
1100 u64 t0, t1;
1101
1102 t0 = cpu_clock(this_cpu);
1103 t1 = cpu_clock(this_cpu);
1104 SEQ_printf(m, "%-45s:%21Ld\n",
1105 "clock-delta", (long long)(t1-t0));
1106 }
1107
1108 sched_show_numa(p, m);
1109 }
1110
proc_sched_set_task(struct task_struct * p)1111 void proc_sched_set_task(struct task_struct *p)
1112 {
1113 #ifdef CONFIG_SCHEDSTATS
1114 memset(&p->se.statistics, 0, sizeof(p->se.statistics));
1115 #endif
1116 }
1117