1 /*
2 * kernel/sched/debug.c
3 *
4 * Print the CFS rbtree
5 *
6 * Copyright(C) 2007, Red Hat, Inc., Ingo Molnar
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13 #include <linux/proc_fs.h>
14 #include <linux/sched.h>
15 #include <linux/seq_file.h>
16 #include <linux/kallsyms.h>
17 #include <linux/utsname.h>
18 #include <linux/mempolicy.h>
19 #include <linux/debugfs.h>
20
21 #include "sched.h"
22
23 static DEFINE_SPINLOCK(sched_debug_lock);
24
25 /*
26 * This allows printing both to /proc/sched_debug and
27 * to the console
28 */
29 #define SEQ_printf(m, x...) \
30 do { \
31 if (m) \
32 seq_printf(m, x); \
33 else \
34 printk(x); \
35 } while (0)
36
37 /*
38 * Ease the printing of nsec fields:
39 */
nsec_high(unsigned long long nsec)40 static long long nsec_high(unsigned long long nsec)
41 {
42 if ((long long)nsec < 0) {
43 nsec = -nsec;
44 do_div(nsec, 1000000);
45 return -nsec;
46 }
47 do_div(nsec, 1000000);
48
49 return nsec;
50 }
51
nsec_low(unsigned long long nsec)52 static unsigned long nsec_low(unsigned long long nsec)
53 {
54 if ((long long)nsec < 0)
55 nsec = -nsec;
56
57 return do_div(nsec, 1000000);
58 }
59
60 #define SPLIT_NS(x) nsec_high(x), nsec_low(x)
61
62 #define SCHED_FEAT(name, enabled) \
63 #name ,
64
65 static const char * const sched_feat_names[] = {
66 #include "features.h"
67 };
68
69 #undef SCHED_FEAT
70
sched_feat_show(struct seq_file * m,void * v)71 static int sched_feat_show(struct seq_file *m, void *v)
72 {
73 int i;
74
75 for (i = 0; i < __SCHED_FEAT_NR; i++) {
76 if (!(sysctl_sched_features & (1UL << i)))
77 seq_puts(m, "NO_");
78 seq_printf(m, "%s ", sched_feat_names[i]);
79 }
80 seq_puts(m, "\n");
81
82 return 0;
83 }
84
85 #ifdef HAVE_JUMP_LABEL
86
87 #define jump_label_key__true STATIC_KEY_INIT_TRUE
88 #define jump_label_key__false STATIC_KEY_INIT_FALSE
89
90 #define SCHED_FEAT(name, enabled) \
91 jump_label_key__##enabled ,
92
93 struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
94 #include "features.h"
95 };
96
97 #undef SCHED_FEAT
98
sched_feat_disable(int i)99 static void sched_feat_disable(int i)
100 {
101 static_key_disable(&sched_feat_keys[i]);
102 }
103
sched_feat_enable(int i)104 static void sched_feat_enable(int i)
105 {
106 static_key_enable(&sched_feat_keys[i]);
107 }
108 #else
sched_feat_disable(int i)109 static void sched_feat_disable(int i) { };
sched_feat_enable(int i)110 static void sched_feat_enable(int i) { };
111 #endif /* HAVE_JUMP_LABEL */
112
sched_feat_set(char * cmp)113 static int sched_feat_set(char *cmp)
114 {
115 int i;
116 int neg = 0;
117
118 if (strncmp(cmp, "NO_", 3) == 0) {
119 neg = 1;
120 cmp += 3;
121 }
122
123 for (i = 0; i < __SCHED_FEAT_NR; i++) {
124 if (strcmp(cmp, sched_feat_names[i]) == 0) {
125 if (neg) {
126 sysctl_sched_features &= ~(1UL << i);
127 sched_feat_disable(i);
128 } else {
129 sysctl_sched_features |= (1UL << i);
130 sched_feat_enable(i);
131 }
132 break;
133 }
134 }
135
136 return i;
137 }
138
139 static ssize_t
sched_feat_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)140 sched_feat_write(struct file *filp, const char __user *ubuf,
141 size_t cnt, loff_t *ppos)
142 {
143 char buf[64];
144 char *cmp;
145 int i;
146 struct inode *inode;
147
148 if (cnt > 63)
149 cnt = 63;
150
151 if (copy_from_user(&buf, ubuf, cnt))
152 return -EFAULT;
153
154 buf[cnt] = 0;
155 cmp = strstrip(buf);
156
157 /* Ensure the static_key remains in a consistent state */
158 inode = file_inode(filp);
159 inode_lock(inode);
160 i = sched_feat_set(cmp);
161 inode_unlock(inode);
162 if (i == __SCHED_FEAT_NR)
163 return -EINVAL;
164
165 *ppos += cnt;
166
167 return cnt;
168 }
169
sched_feat_open(struct inode * inode,struct file * filp)170 static int sched_feat_open(struct inode *inode, struct file *filp)
171 {
172 return single_open(filp, sched_feat_show, NULL);
173 }
174
175 static const struct file_operations sched_feat_fops = {
176 .open = sched_feat_open,
177 .write = sched_feat_write,
178 .read = seq_read,
179 .llseek = seq_lseek,
180 .release = single_release,
181 };
182
sched_init_debug(void)183 static __init int sched_init_debug(void)
184 {
185 debugfs_create_file("sched_features", 0644, NULL, NULL,
186 &sched_feat_fops);
187
188 return 0;
189 }
190 late_initcall(sched_init_debug);
191
192 #ifdef CONFIG_SMP
193
194 #ifdef CONFIG_SYSCTL
195
196 static struct ctl_table sd_ctl_dir[] = {
197 {
198 .procname = "sched_domain",
199 .mode = 0555,
200 },
201 {}
202 };
203
204 static struct ctl_table sd_ctl_root[] = {
205 {
206 .procname = "kernel",
207 .mode = 0555,
208 .child = sd_ctl_dir,
209 },
210 {}
211 };
212
sd_alloc_ctl_entry(int n)213 static struct ctl_table *sd_alloc_ctl_entry(int n)
214 {
215 struct ctl_table *entry =
216 kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
217
218 return entry;
219 }
220
sd_free_ctl_entry(struct ctl_table ** tablep)221 static void sd_free_ctl_entry(struct ctl_table **tablep)
222 {
223 struct ctl_table *entry;
224
225 /*
226 * In the intermediate directories, both the child directory and
227 * procname are dynamically allocated and could fail but the mode
228 * will always be set. In the lowest directory the names are
229 * static strings and all have proc handlers.
230 */
231 for (entry = *tablep; entry->mode; entry++) {
232 if (entry->child)
233 sd_free_ctl_entry(&entry->child);
234 if (entry->proc_handler == NULL)
235 kfree(entry->procname);
236 }
237
238 kfree(*tablep);
239 *tablep = NULL;
240 }
241
242 static int min_load_idx = 0;
243 static int max_load_idx = CPU_LOAD_IDX_MAX-1;
244
245 static void
set_table_entry(struct ctl_table * entry,const char * procname,void * data,int maxlen,umode_t mode,proc_handler * proc_handler,bool load_idx)246 set_table_entry(struct ctl_table *entry,
247 const char *procname, void *data, int maxlen,
248 umode_t mode, proc_handler *proc_handler,
249 bool load_idx)
250 {
251 entry->procname = procname;
252 entry->data = data;
253 entry->maxlen = maxlen;
254 entry->mode = mode;
255 entry->proc_handler = proc_handler;
256
257 if (load_idx) {
258 entry->extra1 = &min_load_idx;
259 entry->extra2 = &max_load_idx;
260 }
261 }
262
263 static struct ctl_table *
sd_alloc_ctl_energy_table(struct sched_group_energy * sge)264 sd_alloc_ctl_energy_table(struct sched_group_energy *sge)
265 {
266 struct ctl_table *table = sd_alloc_ctl_entry(5);
267
268 if (table == NULL)
269 return NULL;
270
271 set_table_entry(&table[0], "nr_idle_states", &sge->nr_idle_states,
272 sizeof(int), 0644, proc_dointvec_minmax, false);
273 set_table_entry(&table[1], "idle_states", &sge->idle_states[0].power,
274 sge->nr_idle_states*sizeof(struct idle_state), 0644,
275 proc_doulongvec_minmax, false);
276 set_table_entry(&table[2], "nr_cap_states", &sge->nr_cap_states,
277 sizeof(int), 0644, proc_dointvec_minmax, false);
278 set_table_entry(&table[3], "cap_states", &sge->cap_states[0].cap,
279 sge->nr_cap_states*sizeof(struct capacity_state), 0644,
280 proc_doulongvec_minmax, false);
281
282 return table;
283 }
284
285 static struct ctl_table *
sd_alloc_ctl_group_table(struct sched_group * sg)286 sd_alloc_ctl_group_table(struct sched_group *sg)
287 {
288 struct ctl_table *table = sd_alloc_ctl_entry(2);
289
290 if (table == NULL)
291 return NULL;
292
293 table->procname = kstrdup("energy", GFP_KERNEL);
294 table->mode = 0555;
295 table->child = sd_alloc_ctl_energy_table((struct sched_group_energy *)sg->sge);
296
297 return table;
298 }
299
300 static struct ctl_table *
sd_alloc_ctl_domain_table(struct sched_domain * sd)301 sd_alloc_ctl_domain_table(struct sched_domain *sd)
302 {
303 struct ctl_table *table;
304 unsigned int nr_entries = 14;
305
306 int i = 0;
307 struct sched_group *sg = sd->groups;
308
309 if (sg->sge) {
310 int nr_sgs = 0;
311
312 do {} while (nr_sgs++, sg = sg->next, sg != sd->groups);
313
314 nr_entries += nr_sgs;
315 }
316
317 table = sd_alloc_ctl_entry(nr_entries);
318
319 if (table == NULL)
320 return NULL;
321
322 set_table_entry(&table[0], "min_interval", &sd->min_interval,
323 sizeof(long), 0644, proc_doulongvec_minmax, false);
324 set_table_entry(&table[1], "max_interval", &sd->max_interval,
325 sizeof(long), 0644, proc_doulongvec_minmax, false);
326 set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
327 sizeof(int), 0644, proc_dointvec_minmax, true);
328 set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
329 sizeof(int), 0644, proc_dointvec_minmax, true);
330 set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
331 sizeof(int), 0644, proc_dointvec_minmax, true);
332 set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
333 sizeof(int), 0644, proc_dointvec_minmax, true);
334 set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
335 sizeof(int), 0644, proc_dointvec_minmax, true);
336 set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
337 sizeof(int), 0644, proc_dointvec_minmax, false);
338 set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
339 sizeof(int), 0644, proc_dointvec_minmax, false);
340 set_table_entry(&table[9], "cache_nice_tries",
341 &sd->cache_nice_tries,
342 sizeof(int), 0644, proc_dointvec_minmax, false);
343 set_table_entry(&table[10], "flags", &sd->flags,
344 sizeof(int), 0644, proc_dointvec_minmax, false);
345 set_table_entry(&table[11], "max_newidle_lb_cost",
346 &sd->max_newidle_lb_cost,
347 sizeof(long), 0644, proc_doulongvec_minmax, false);
348 set_table_entry(&table[12], "name", sd->name,
349 CORENAME_MAX_SIZE, 0444, proc_dostring, false);
350 sg = sd->groups;
351 if (sg->sge) {
352 char buf[32];
353 struct ctl_table *entry = &table[13];
354
355 do {
356 snprintf(buf, 32, "group%d", i);
357 entry->procname = kstrdup(buf, GFP_KERNEL);
358 entry->mode = 0555;
359 entry->child = sd_alloc_ctl_group_table(sg);
360 } while (entry++, i++, sg = sg->next, sg != sd->groups);
361 }
362 /* &table[nr_entries-1] is terminator */
363
364 return table;
365 }
366
sd_alloc_ctl_cpu_table(int cpu)367 static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu)
368 {
369 struct ctl_table *entry, *table;
370 struct sched_domain *sd;
371 int domain_num = 0, i;
372 char buf[32];
373
374 for_each_domain(cpu, sd)
375 domain_num++;
376 entry = table = sd_alloc_ctl_entry(domain_num + 1);
377 if (table == NULL)
378 return NULL;
379
380 i = 0;
381 for_each_domain(cpu, sd) {
382 snprintf(buf, 32, "domain%d", i);
383 entry->procname = kstrdup(buf, GFP_KERNEL);
384 entry->mode = 0555;
385 entry->child = sd_alloc_ctl_domain_table(sd);
386 entry++;
387 i++;
388 }
389 return table;
390 }
391
392 static struct ctl_table_header *sd_sysctl_header;
register_sched_domain_sysctl(void)393 void register_sched_domain_sysctl(void)
394 {
395 int i, cpu_num = num_possible_cpus();
396 struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
397 char buf[32];
398
399 WARN_ON(sd_ctl_dir[0].child);
400 sd_ctl_dir[0].child = entry;
401
402 if (entry == NULL)
403 return;
404
405 for_each_possible_cpu(i) {
406 snprintf(buf, 32, "cpu%d", i);
407 entry->procname = kstrdup(buf, GFP_KERNEL);
408 entry->mode = 0555;
409 entry->child = sd_alloc_ctl_cpu_table(i);
410 entry++;
411 }
412
413 WARN_ON(sd_sysctl_header);
414 sd_sysctl_header = register_sysctl_table(sd_ctl_root);
415 }
416
417 /* may be called multiple times per register */
unregister_sched_domain_sysctl(void)418 void unregister_sched_domain_sysctl(void)
419 {
420 unregister_sysctl_table(sd_sysctl_header);
421 sd_sysctl_header = NULL;
422 if (sd_ctl_dir[0].child)
423 sd_free_ctl_entry(&sd_ctl_dir[0].child);
424 }
425 #endif /* CONFIG_SYSCTL */
426 #endif /* CONFIG_SMP */
427
428 #ifdef CONFIG_FAIR_GROUP_SCHED
print_cfs_group_stats(struct seq_file * m,int cpu,struct task_group * tg)429 static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group *tg)
430 {
431 struct sched_entity *se = tg->se[cpu];
432
433 #define P(F) \
434 SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F)
435 #define P_SCHEDSTAT(F) \
436 SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)schedstat_val(F))
437 #define PN(F) \
438 SEQ_printf(m, " .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)F))
439 #define PN_SCHEDSTAT(F) \
440 SEQ_printf(m, " .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)schedstat_val(F)))
441
442 if (!se)
443 return;
444
445 PN(se->exec_start);
446 PN(se->vruntime);
447 PN(se->sum_exec_runtime);
448 if (schedstat_enabled()) {
449 PN_SCHEDSTAT(se->statistics.wait_start);
450 PN_SCHEDSTAT(se->statistics.sleep_start);
451 PN_SCHEDSTAT(se->statistics.block_start);
452 PN_SCHEDSTAT(se->statistics.sleep_max);
453 PN_SCHEDSTAT(se->statistics.block_max);
454 PN_SCHEDSTAT(se->statistics.exec_max);
455 PN_SCHEDSTAT(se->statistics.slice_max);
456 PN_SCHEDSTAT(se->statistics.wait_max);
457 PN_SCHEDSTAT(se->statistics.wait_sum);
458 P_SCHEDSTAT(se->statistics.wait_count);
459 }
460 P(se->load.weight);
461 #ifdef CONFIG_SMP
462 P(se->avg.load_avg);
463 P(se->avg.util_avg);
464 #endif
465
466 #undef PN_SCHEDSTAT
467 #undef PN
468 #undef P_SCHEDSTAT
469 #undef P
470 }
471 #endif
472
473 #ifdef CONFIG_CGROUP_SCHED
474 static char group_path[PATH_MAX];
475
task_group_path(struct task_group * tg)476 static char *task_group_path(struct task_group *tg)
477 {
478 if (autogroup_path(tg, group_path, PATH_MAX))
479 return group_path;
480
481 cgroup_path(tg->css.cgroup, group_path, PATH_MAX);
482 return group_path;
483 }
484 #endif
485
486 static void
print_task(struct seq_file * m,struct rq * rq,struct task_struct * p)487 print_task(struct seq_file *m, struct rq *rq, struct task_struct *p)
488 {
489 if (rq->curr == p)
490 SEQ_printf(m, "R");
491 else
492 SEQ_printf(m, " ");
493
494 SEQ_printf(m, "%15s %5d %9Ld.%06ld %9Ld %5d ",
495 p->comm, task_pid_nr(p),
496 SPLIT_NS(p->se.vruntime),
497 (long long)(p->nvcsw + p->nivcsw),
498 p->prio);
499
500 SEQ_printf(m, "%9Ld.%06ld %9Ld.%06ld %9Ld.%06ld",
501 SPLIT_NS(schedstat_val_or_zero(p->se.statistics.wait_sum)),
502 SPLIT_NS(p->se.sum_exec_runtime),
503 SPLIT_NS(schedstat_val_or_zero(p->se.statistics.sum_sleep_runtime)));
504
505 #ifdef CONFIG_NUMA_BALANCING
506 SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));
507 #endif
508 #ifdef CONFIG_CGROUP_SCHED
509 SEQ_printf(m, " %s", task_group_path(task_group(p)));
510 #endif
511
512 SEQ_printf(m, "\n");
513 }
514
print_rq(struct seq_file * m,struct rq * rq,int rq_cpu)515 static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu)
516 {
517 struct task_struct *g, *p;
518
519 SEQ_printf(m,
520 "\nrunnable tasks:\n"
521 " task PID tree-key switches prio"
522 " wait-time sum-exec sum-sleep\n"
523 "------------------------------------------------------"
524 "----------------------------------------------------\n");
525
526 rcu_read_lock();
527 for_each_process_thread(g, p) {
528 if (task_cpu(p) != rq_cpu)
529 continue;
530
531 print_task(m, rq, p);
532 }
533 rcu_read_unlock();
534 }
535
print_cfs_rq(struct seq_file * m,int cpu,struct cfs_rq * cfs_rq)536 void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
537 {
538 s64 MIN_vruntime = -1, min_vruntime, max_vruntime = -1,
539 spread, rq0_min_vruntime, spread0;
540 struct rq *rq = cpu_rq(cpu);
541 struct sched_entity *last;
542 unsigned long flags;
543
544 #ifdef CONFIG_FAIR_GROUP_SCHED
545 SEQ_printf(m, "\ncfs_rq[%d]:%s\n", cpu, task_group_path(cfs_rq->tg));
546 #else
547 SEQ_printf(m, "\ncfs_rq[%d]:\n", cpu);
548 #endif
549 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock",
550 SPLIT_NS(cfs_rq->exec_clock));
551
552 raw_spin_lock_irqsave(&rq->lock, flags);
553 if (cfs_rq->rb_leftmost)
554 MIN_vruntime = (__pick_first_entity(cfs_rq))->vruntime;
555 last = __pick_last_entity(cfs_rq);
556 if (last)
557 max_vruntime = last->vruntime;
558 min_vruntime = cfs_rq->min_vruntime;
559 rq0_min_vruntime = cpu_rq(0)->cfs.min_vruntime;
560 raw_spin_unlock_irqrestore(&rq->lock, flags);
561 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "MIN_vruntime",
562 SPLIT_NS(MIN_vruntime));
563 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "min_vruntime",
564 SPLIT_NS(min_vruntime));
565 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "max_vruntime",
566 SPLIT_NS(max_vruntime));
567 spread = max_vruntime - MIN_vruntime;
568 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread",
569 SPLIT_NS(spread));
570 spread0 = min_vruntime - rq0_min_vruntime;
571 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread0",
572 SPLIT_NS(spread0));
573 SEQ_printf(m, " .%-30s: %d\n", "nr_spread_over",
574 cfs_rq->nr_spread_over);
575 SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running);
576 SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight);
577 #ifdef CONFIG_SMP
578 SEQ_printf(m, " .%-30s: %lu\n", "load_avg",
579 cfs_rq->avg.load_avg);
580 SEQ_printf(m, " .%-30s: %lu\n", "runnable_load_avg",
581 cfs_rq->runnable_load_avg);
582 SEQ_printf(m, " .%-30s: %lu\n", "util_avg",
583 cfs_rq->avg.util_avg);
584 SEQ_printf(m, " .%-30s: %ld\n", "removed_load_avg",
585 atomic_long_read(&cfs_rq->removed_load_avg));
586 SEQ_printf(m, " .%-30s: %ld\n", "removed_util_avg",
587 atomic_long_read(&cfs_rq->removed_util_avg));
588 #ifdef CONFIG_FAIR_GROUP_SCHED
589 SEQ_printf(m, " .%-30s: %lu\n", "tg_load_avg_contrib",
590 cfs_rq->tg_load_avg_contrib);
591 SEQ_printf(m, " .%-30s: %ld\n", "tg_load_avg",
592 atomic_long_read(&cfs_rq->tg->load_avg));
593 #endif
594 #endif
595 #ifdef CONFIG_CFS_BANDWIDTH
596 SEQ_printf(m, " .%-30s: %d\n", "throttled",
597 cfs_rq->throttled);
598 SEQ_printf(m, " .%-30s: %d\n", "throttle_count",
599 cfs_rq->throttle_count);
600 #endif
601
602 #ifdef CONFIG_FAIR_GROUP_SCHED
603 print_cfs_group_stats(m, cpu, cfs_rq->tg);
604 #endif
605 }
606
print_rt_rq(struct seq_file * m,int cpu,struct rt_rq * rt_rq)607 void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq)
608 {
609 #ifdef CONFIG_RT_GROUP_SCHED
610 SEQ_printf(m, "\nrt_rq[%d]:%s\n", cpu, task_group_path(rt_rq->tg));
611 #else
612 SEQ_printf(m, "\nrt_rq[%d]:\n", cpu);
613 #endif
614
615 #define P(x) \
616 SEQ_printf(m, " .%-30s: %Ld\n", #x, (long long)(rt_rq->x))
617 #define PN(x) \
618 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rt_rq->x))
619
620 P(rt_nr_running);
621 P(rt_throttled);
622 PN(rt_time);
623 PN(rt_runtime);
624
625 #undef PN
626 #undef P
627 }
628
print_dl_rq(struct seq_file * m,int cpu,struct dl_rq * dl_rq)629 void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq)
630 {
631 struct dl_bw *dl_bw;
632
633 SEQ_printf(m, "\ndl_rq[%d]:\n", cpu);
634 SEQ_printf(m, " .%-30s: %ld\n", "dl_nr_running", dl_rq->dl_nr_running);
635 #ifdef CONFIG_SMP
636 dl_bw = &cpu_rq(cpu)->rd->dl_bw;
637 #else
638 dl_bw = &dl_rq->dl_bw;
639 #endif
640 SEQ_printf(m, " .%-30s: %lld\n", "dl_bw->bw", dl_bw->bw);
641 SEQ_printf(m, " .%-30s: %lld\n", "dl_bw->total_bw", dl_bw->total_bw);
642 }
643
644 extern __read_mostly int sched_clock_running;
645
print_cpu(struct seq_file * m,int cpu)646 static void print_cpu(struct seq_file *m, int cpu)
647 {
648 struct rq *rq = cpu_rq(cpu);
649 unsigned long flags;
650
651 #ifdef CONFIG_X86
652 {
653 unsigned int freq = cpu_khz ? : 1;
654
655 SEQ_printf(m, "cpu#%d, %u.%03u MHz\n",
656 cpu, freq / 1000, (freq % 1000));
657 }
658 #else
659 SEQ_printf(m, "cpu#%d\n", cpu);
660 #endif
661
662 #define P(x) \
663 do { \
664 if (sizeof(rq->x) == 4) \
665 SEQ_printf(m, " .%-30s: %ld\n", #x, (long)(rq->x)); \
666 else \
667 SEQ_printf(m, " .%-30s: %Ld\n", #x, (long long)(rq->x));\
668 } while (0)
669
670 #define PN(x) \
671 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rq->x))
672
673 P(nr_running);
674 SEQ_printf(m, " .%-30s: %lu\n", "load",
675 rq->load.weight);
676 P(nr_switches);
677 P(nr_load_updates);
678 P(nr_uninterruptible);
679 PN(next_balance);
680 SEQ_printf(m, " .%-30s: %ld\n", "curr->pid", (long)(task_pid_nr(rq->curr)));
681 PN(clock);
682 PN(clock_task);
683 P(cpu_load[0]);
684 P(cpu_load[1]);
685 P(cpu_load[2]);
686 P(cpu_load[3]);
687 P(cpu_load[4]);
688 #undef P
689 #undef PN
690
691 #ifdef CONFIG_SMP
692 #define P64(n) SEQ_printf(m, " .%-30s: %Ld\n", #n, rq->n);
693 P64(avg_idle);
694 P64(max_idle_balance_cost);
695 #undef P64
696 #endif
697
698 #define P(n) SEQ_printf(m, " .%-30s: %d\n", #n, schedstat_val(rq->n));
699 if (schedstat_enabled()) {
700 P(yld_count);
701 P(sched_count);
702 P(sched_goidle);
703 P(ttwu_count);
704 P(ttwu_local);
705 }
706 #undef P
707
708 spin_lock_irqsave(&sched_debug_lock, flags);
709 print_cfs_stats(m, cpu);
710 print_rt_stats(m, cpu);
711 print_dl_stats(m, cpu);
712
713 print_rq(m, rq, cpu);
714 spin_unlock_irqrestore(&sched_debug_lock, flags);
715 SEQ_printf(m, "\n");
716 }
717
718 static const char *sched_tunable_scaling_names[] = {
719 "none",
720 "logaritmic",
721 "linear"
722 };
723
sched_debug_header(struct seq_file * m)724 static void sched_debug_header(struct seq_file *m)
725 {
726 u64 ktime, sched_clk, cpu_clk;
727 unsigned long flags;
728
729 local_irq_save(flags);
730 ktime = ktime_to_ns(ktime_get());
731 sched_clk = sched_clock();
732 cpu_clk = local_clock();
733 local_irq_restore(flags);
734
735 SEQ_printf(m, "Sched Debug Version: v0.11, %s %.*s\n",
736 init_utsname()->release,
737 (int)strcspn(init_utsname()->version, " "),
738 init_utsname()->version);
739
740 #define P(x) \
741 SEQ_printf(m, "%-40s: %Ld\n", #x, (long long)(x))
742 #define PN(x) \
743 SEQ_printf(m, "%-40s: %Ld.%06ld\n", #x, SPLIT_NS(x))
744 PN(ktime);
745 PN(sched_clk);
746 PN(cpu_clk);
747 P(jiffies);
748 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
749 P(sched_clock_stable());
750 #endif
751 #undef PN
752 #undef P
753
754 SEQ_printf(m, "\n");
755 SEQ_printf(m, "sysctl_sched\n");
756
757 #define P(x) \
758 SEQ_printf(m, " .%-40s: %Ld\n", #x, (long long)(x))
759 #define PN(x) \
760 SEQ_printf(m, " .%-40s: %Ld.%06ld\n", #x, SPLIT_NS(x))
761 PN(sysctl_sched_latency);
762 PN(sysctl_sched_min_granularity);
763 PN(sysctl_sched_wakeup_granularity);
764 P(sysctl_sched_child_runs_first);
765 P(sysctl_sched_features);
766 #undef PN
767 #undef P
768
769 SEQ_printf(m, " .%-40s: %d (%s)\n",
770 "sysctl_sched_tunable_scaling",
771 sysctl_sched_tunable_scaling,
772 sched_tunable_scaling_names[sysctl_sched_tunable_scaling]);
773 SEQ_printf(m, "\n");
774 }
775
sched_debug_show(struct seq_file * m,void * v)776 static int sched_debug_show(struct seq_file *m, void *v)
777 {
778 int cpu = (unsigned long)(v - 2);
779
780 if (cpu != -1)
781 print_cpu(m, cpu);
782 else
783 sched_debug_header(m);
784
785 return 0;
786 }
787
sysrq_sched_debug_show(void)788 void sysrq_sched_debug_show(void)
789 {
790 int cpu;
791
792 sched_debug_header(NULL);
793 for_each_online_cpu(cpu)
794 print_cpu(NULL, cpu);
795
796 }
797
798 /*
799 * This itererator needs some explanation.
800 * It returns 1 for the header position.
801 * This means 2 is cpu 0.
802 * In a hotplugged system some cpus, including cpu 0, may be missing so we have
803 * to use cpumask_* to iterate over the cpus.
804 */
sched_debug_start(struct seq_file * file,loff_t * offset)805 static void *sched_debug_start(struct seq_file *file, loff_t *offset)
806 {
807 unsigned long n = *offset;
808
809 if (n == 0)
810 return (void *) 1;
811
812 n--;
813
814 if (n > 0)
815 n = cpumask_next(n - 1, cpu_online_mask);
816 else
817 n = cpumask_first(cpu_online_mask);
818
819 *offset = n + 1;
820
821 if (n < nr_cpu_ids)
822 return (void *)(unsigned long)(n + 2);
823 return NULL;
824 }
825
sched_debug_next(struct seq_file * file,void * data,loff_t * offset)826 static void *sched_debug_next(struct seq_file *file, void *data, loff_t *offset)
827 {
828 (*offset)++;
829 return sched_debug_start(file, offset);
830 }
831
sched_debug_stop(struct seq_file * file,void * data)832 static void sched_debug_stop(struct seq_file *file, void *data)
833 {
834 }
835
836 static const struct seq_operations sched_debug_sops = {
837 .start = sched_debug_start,
838 .next = sched_debug_next,
839 .stop = sched_debug_stop,
840 .show = sched_debug_show,
841 };
842
sched_debug_release(struct inode * inode,struct file * file)843 static int sched_debug_release(struct inode *inode, struct file *file)
844 {
845 seq_release(inode, file);
846
847 return 0;
848 }
849
sched_debug_open(struct inode * inode,struct file * filp)850 static int sched_debug_open(struct inode *inode, struct file *filp)
851 {
852 int ret = 0;
853
854 ret = seq_open(filp, &sched_debug_sops);
855
856 return ret;
857 }
858
859 static const struct file_operations sched_debug_fops = {
860 .open = sched_debug_open,
861 .read = seq_read,
862 .llseek = seq_lseek,
863 .release = sched_debug_release,
864 };
865
init_sched_debug_procfs(void)866 static int __init init_sched_debug_procfs(void)
867 {
868 struct proc_dir_entry *pe;
869
870 pe = proc_create("sched_debug", 0444, NULL, &sched_debug_fops);
871 if (!pe)
872 return -ENOMEM;
873 return 0;
874 }
875
876 __initcall(init_sched_debug_procfs);
877
878 #define __P(F) \
879 SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)F)
880 #define P(F) \
881 SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)p->F)
882 #define __PN(F) \
883 SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F))
884 #define PN(F) \
885 SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F))
886
887
888 #ifdef CONFIG_NUMA_BALANCING
print_numa_stats(struct seq_file * m,int node,unsigned long tsf,unsigned long tpf,unsigned long gsf,unsigned long gpf)889 void print_numa_stats(struct seq_file *m, int node, unsigned long tsf,
890 unsigned long tpf, unsigned long gsf, unsigned long gpf)
891 {
892 SEQ_printf(m, "numa_faults node=%d ", node);
893 SEQ_printf(m, "task_private=%lu task_shared=%lu ", tsf, tpf);
894 SEQ_printf(m, "group_private=%lu group_shared=%lu\n", gsf, gpf);
895 }
896 #endif
897
898
sched_show_numa(struct task_struct * p,struct seq_file * m)899 static void sched_show_numa(struct task_struct *p, struct seq_file *m)
900 {
901 #ifdef CONFIG_NUMA_BALANCING
902 struct mempolicy *pol;
903
904 if (p->mm)
905 P(mm->numa_scan_seq);
906
907 task_lock(p);
908 pol = p->mempolicy;
909 if (pol && !(pol->flags & MPOL_F_MORON))
910 pol = NULL;
911 mpol_get(pol);
912 task_unlock(p);
913
914 P(numa_pages_migrated);
915 P(numa_preferred_nid);
916 P(total_numa_faults);
917 SEQ_printf(m, "current_node=%d, numa_group_id=%d\n",
918 task_node(p), task_numa_group_id(p));
919 show_numa_stats(p, m);
920 mpol_put(pol);
921 #endif
922 }
923
proc_sched_show_task(struct task_struct * p,struct seq_file * m)924 void proc_sched_show_task(struct task_struct *p, struct seq_file *m)
925 {
926 unsigned long nr_switches;
927
928 SEQ_printf(m, "%s (%d, #threads: %d)\n", p->comm, task_pid_nr(p),
929 get_nr_threads(p));
930 SEQ_printf(m,
931 "---------------------------------------------------------"
932 "----------\n");
933 #define __P(F) \
934 SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)F)
935 #define P(F) \
936 SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)p->F)
937 #define P_SCHEDSTAT(F) \
938 SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)schedstat_val(p->F))
939 #define __PN(F) \
940 SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F))
941 #define PN(F) \
942 SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F))
943 #define PN_SCHEDSTAT(F) \
944 SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)schedstat_val(p->F)))
945
946 PN(se.exec_start);
947 PN(se.vruntime);
948 PN(se.sum_exec_runtime);
949
950 nr_switches = p->nvcsw + p->nivcsw;
951
952 P(se.nr_migrations);
953
954 if (schedstat_enabled()) {
955 u64 avg_atom, avg_per_cpu;
956
957 PN_SCHEDSTAT(se.statistics.sum_sleep_runtime);
958 PN_SCHEDSTAT(se.statistics.wait_start);
959 PN_SCHEDSTAT(se.statistics.sleep_start);
960 PN_SCHEDSTAT(se.statistics.block_start);
961 PN_SCHEDSTAT(se.statistics.sleep_max);
962 PN_SCHEDSTAT(se.statistics.block_max);
963 PN_SCHEDSTAT(se.statistics.exec_max);
964 PN_SCHEDSTAT(se.statistics.slice_max);
965 PN_SCHEDSTAT(se.statistics.wait_max);
966 PN_SCHEDSTAT(se.statistics.wait_sum);
967 P_SCHEDSTAT(se.statistics.wait_count);
968 PN_SCHEDSTAT(se.statistics.iowait_sum);
969 P_SCHEDSTAT(se.statistics.iowait_count);
970 P_SCHEDSTAT(se.statistics.nr_migrations_cold);
971 P_SCHEDSTAT(se.statistics.nr_failed_migrations_affine);
972 P_SCHEDSTAT(se.statistics.nr_failed_migrations_running);
973 P_SCHEDSTAT(se.statistics.nr_failed_migrations_hot);
974 P_SCHEDSTAT(se.statistics.nr_forced_migrations);
975 P_SCHEDSTAT(se.statistics.nr_wakeups);
976 P_SCHEDSTAT(se.statistics.nr_wakeups_sync);
977 P_SCHEDSTAT(se.statistics.nr_wakeups_migrate);
978 P_SCHEDSTAT(se.statistics.nr_wakeups_local);
979 P_SCHEDSTAT(se.statistics.nr_wakeups_remote);
980 P_SCHEDSTAT(se.statistics.nr_wakeups_affine);
981 P_SCHEDSTAT(se.statistics.nr_wakeups_affine_attempts);
982 P_SCHEDSTAT(se.statistics.nr_wakeups_passive);
983 P_SCHEDSTAT(se.statistics.nr_wakeups_idle);
984 /* eas */
985 /* select_idle_sibling() */
986 P_SCHEDSTAT(se.statistics.nr_wakeups_sis_attempts);
987 P_SCHEDSTAT(se.statistics.nr_wakeups_sis_idle);
988 P_SCHEDSTAT(se.statistics.nr_wakeups_sis_cache_affine);
989 P_SCHEDSTAT(se.statistics.nr_wakeups_sis_suff_cap);
990 P_SCHEDSTAT(se.statistics.nr_wakeups_sis_idle_cpu);
991 P_SCHEDSTAT(se.statistics.nr_wakeups_sis_count);
992 /* select_energy_cpu_brute() */
993 P_SCHEDSTAT(se.statistics.nr_wakeups_secb_attempts);
994 P_SCHEDSTAT(se.statistics.nr_wakeups_secb_sync);
995 P_SCHEDSTAT(se.statistics.nr_wakeups_secb_idle_bt);
996 P_SCHEDSTAT(se.statistics.nr_wakeups_secb_insuff_cap);
997 P_SCHEDSTAT(se.statistics.nr_wakeups_secb_no_nrg_sav);
998 P_SCHEDSTAT(se.statistics.nr_wakeups_secb_nrg_sav);
999 P_SCHEDSTAT(se.statistics.nr_wakeups_secb_count);
1000 /* find_best_target() */
1001 P_SCHEDSTAT(se.statistics.nr_wakeups_fbt_attempts);
1002 P_SCHEDSTAT(se.statistics.nr_wakeups_fbt_no_cpu);
1003 P_SCHEDSTAT(se.statistics.nr_wakeups_fbt_no_sd);
1004 P_SCHEDSTAT(se.statistics.nr_wakeups_fbt_pref_idle);
1005 P_SCHEDSTAT(se.statistics.nr_wakeups_fbt_count);
1006 /* cas */
1007 /* select_task_rq_fair() */
1008 P_SCHEDSTAT(se.statistics.nr_wakeups_cas_attempts);
1009 P_SCHEDSTAT(se.statistics.nr_wakeups_cas_count);
1010
1011 avg_atom = p->se.sum_exec_runtime;
1012 if (nr_switches)
1013 avg_atom = div64_ul(avg_atom, nr_switches);
1014 else
1015 avg_atom = -1LL;
1016
1017 avg_per_cpu = p->se.sum_exec_runtime;
1018 if (p->se.nr_migrations) {
1019 avg_per_cpu = div64_u64(avg_per_cpu,
1020 p->se.nr_migrations);
1021 } else {
1022 avg_per_cpu = -1LL;
1023 }
1024
1025 __PN(avg_atom);
1026 __PN(avg_per_cpu);
1027 }
1028
1029 __P(nr_switches);
1030 SEQ_printf(m, "%-45s:%21Ld\n",
1031 "nr_voluntary_switches", (long long)p->nvcsw);
1032 SEQ_printf(m, "%-45s:%21Ld\n",
1033 "nr_involuntary_switches", (long long)p->nivcsw);
1034
1035 P(se.load.weight);
1036 #ifdef CONFIG_SMP
1037 P(se.avg.load_sum);
1038 P(se.avg.util_sum);
1039 P(se.avg.load_avg);
1040 P(se.avg.util_avg);
1041 P(se.avg.last_update_time);
1042 #endif
1043 P(policy);
1044 P(prio);
1045 #undef PN_SCHEDSTAT
1046 #undef PN
1047 #undef __PN
1048 #undef P_SCHEDSTAT
1049 #undef P
1050 #undef __P
1051
1052 {
1053 unsigned int this_cpu = raw_smp_processor_id();
1054 u64 t0, t1;
1055
1056 t0 = cpu_clock(this_cpu);
1057 t1 = cpu_clock(this_cpu);
1058 SEQ_printf(m, "%-45s:%21Ld\n",
1059 "clock-delta", (long long)(t1-t0));
1060 }
1061
1062 sched_show_numa(p, m);
1063 }
1064
proc_sched_set_task(struct task_struct * p)1065 void proc_sched_set_task(struct task_struct *p)
1066 {
1067 #ifdef CONFIG_SCHEDSTATS
1068 memset(&p->se.statistics, 0, sizeof(p->se.statistics));
1069 #endif
1070 }
1071