1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* delayacct.c - per-task delay accounting
3 *
4 * Copyright (C) Shailabh Nagar, IBM Corp. 2006
5 */
6
7 #include <linux/sched.h>
8 #include <linux/sched/task.h>
9 #include <linux/sched/cputime.h>
10 #include <linux/sched/clock.h>
11 #include <linux/slab.h>
12 #include <linux/taskstats.h>
13 #include <linux/sysctl.h>
14 #include <linux/delayacct.h>
15 #include <linux/module.h>
16
17 #ifdef CONFIG_TASK_DELAY_ACCT
18
19 DEFINE_STATIC_KEY_FALSE(delayacct_key);
20 int delayacct_on __read_mostly; /* Delay accounting turned on/off */
21 struct kmem_cache *delayacct_cache;
22
set_delayacct(bool enabled)23 static void set_delayacct(bool enabled)
24 {
25 if (enabled) {
26 static_branch_enable(&delayacct_key);
27 delayacct_on = 1;
28 } else {
29 delayacct_on = 0;
30 static_branch_disable(&delayacct_key);
31 }
32 }
33
delayacct_setup_enable(char * str)34 static int __init delayacct_setup_enable(char *str)
35 {
36 delayacct_on = 1;
37 return 1;
38 }
39 __setup("delayacct", delayacct_setup_enable);
40
delayacct_init(void)41 void delayacct_init(void)
42 {
43 delayacct_cache = KMEM_CACHE(task_delay_info, SLAB_PANIC|SLAB_ACCOUNT);
44 delayacct_tsk_init(&init_task);
45 set_delayacct(delayacct_on);
46 }
47
48 #ifdef CONFIG_PROC_SYSCTL
sysctl_delayacct(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)49 static int sysctl_delayacct(struct ctl_table *table, int write, void *buffer,
50 size_t *lenp, loff_t *ppos)
51 {
52 int state = delayacct_on;
53 struct ctl_table t;
54 int err;
55
56 if (write && !capable(CAP_SYS_ADMIN))
57 return -EPERM;
58
59 t = *table;
60 t.data = &state;
61 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
62 if (err < 0)
63 return err;
64 if (write)
65 set_delayacct(state);
66 return err;
67 }
68
69 static struct ctl_table kern_delayacct_table[] = {
70 {
71 .procname = "task_delayacct",
72 .data = NULL,
73 .maxlen = sizeof(unsigned int),
74 .mode = 0644,
75 .proc_handler = sysctl_delayacct,
76 .extra1 = SYSCTL_ZERO,
77 .extra2 = SYSCTL_ONE,
78 },
79 { }
80 };
81
kernel_delayacct_sysctls_init(void)82 static __init int kernel_delayacct_sysctls_init(void)
83 {
84 register_sysctl_init("kernel", kern_delayacct_table);
85 return 0;
86 }
87 late_initcall(kernel_delayacct_sysctls_init);
88 #endif
89
__delayacct_tsk_init(struct task_struct * tsk)90 void __delayacct_tsk_init(struct task_struct *tsk)
91 {
92 tsk->delays = kmem_cache_zalloc(delayacct_cache, GFP_KERNEL);
93 if (tsk->delays)
94 raw_spin_lock_init(&tsk->delays->lock);
95 }
96
97 /*
98 * Finish delay accounting for a statistic using its timestamps (@start),
99 * accumalator (@total) and @count
100 */
delayacct_end(raw_spinlock_t * lock,u64 * start,u64 * total,u32 * count)101 static void delayacct_end(raw_spinlock_t *lock, u64 *start, u64 *total, u32 *count)
102 {
103 s64 ns = local_clock() - *start;
104 unsigned long flags;
105
106 if (ns > 0) {
107 raw_spin_lock_irqsave(lock, flags);
108 *total += ns;
109 (*count)++;
110 raw_spin_unlock_irqrestore(lock, flags);
111 }
112 }
113
__delayacct_blkio_start(void)114 void __delayacct_blkio_start(void)
115 {
116 current->delays->blkio_start = local_clock();
117 }
118
119 /*
120 * We cannot rely on the `current` macro, as we haven't yet switched back to
121 * the process being woken.
122 */
__delayacct_blkio_end(struct task_struct * p)123 void __delayacct_blkio_end(struct task_struct *p)
124 {
125 delayacct_end(&p->delays->lock,
126 &p->delays->blkio_start,
127 &p->delays->blkio_delay,
128 &p->delays->blkio_count);
129 }
130
delayacct_add_tsk(struct taskstats * d,struct task_struct * tsk)131 int delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk)
132 {
133 u64 utime, stime, stimescaled, utimescaled;
134 unsigned long long t2, t3;
135 unsigned long flags, t1;
136 s64 tmp;
137
138 task_cputime(tsk, &utime, &stime);
139 tmp = (s64)d->cpu_run_real_total;
140 tmp += utime + stime;
141 d->cpu_run_real_total = (tmp < (s64)d->cpu_run_real_total) ? 0 : tmp;
142
143 task_cputime_scaled(tsk, &utimescaled, &stimescaled);
144 tmp = (s64)d->cpu_scaled_run_real_total;
145 tmp += utimescaled + stimescaled;
146 d->cpu_scaled_run_real_total =
147 (tmp < (s64)d->cpu_scaled_run_real_total) ? 0 : tmp;
148
149 /*
150 * No locking available for sched_info (and too expensive to add one)
151 * Mitigate by taking snapshot of values
152 */
153 t1 = tsk->sched_info.pcount;
154 t2 = tsk->sched_info.run_delay;
155 t3 = tsk->se.sum_exec_runtime;
156
157 d->cpu_count += t1;
158
159 tmp = (s64)d->cpu_delay_total + t2;
160 d->cpu_delay_total = (tmp < (s64)d->cpu_delay_total) ? 0 : tmp;
161
162 tmp = (s64)d->cpu_run_virtual_total + t3;
163 d->cpu_run_virtual_total =
164 (tmp < (s64)d->cpu_run_virtual_total) ? 0 : tmp;
165
166 if (!tsk->delays)
167 return 0;
168
169 /* zero XXX_total, non-zero XXX_count implies XXX stat overflowed */
170
171 raw_spin_lock_irqsave(&tsk->delays->lock, flags);
172 tmp = d->blkio_delay_total + tsk->delays->blkio_delay;
173 d->blkio_delay_total = (tmp < d->blkio_delay_total) ? 0 : tmp;
174 tmp = d->swapin_delay_total + tsk->delays->swapin_delay;
175 d->swapin_delay_total = (tmp < d->swapin_delay_total) ? 0 : tmp;
176 tmp = d->freepages_delay_total + tsk->delays->freepages_delay;
177 d->freepages_delay_total = (tmp < d->freepages_delay_total) ? 0 : tmp;
178 tmp = d->thrashing_delay_total + tsk->delays->thrashing_delay;
179 d->thrashing_delay_total = (tmp < d->thrashing_delay_total) ? 0 : tmp;
180 tmp = d->compact_delay_total + tsk->delays->compact_delay;
181 d->compact_delay_total = (tmp < d->compact_delay_total) ? 0 : tmp;
182 tmp = d->wpcopy_delay_total + tsk->delays->wpcopy_delay;
183 d->wpcopy_delay_total = (tmp < d->wpcopy_delay_total) ? 0 : tmp;
184 d->blkio_count += tsk->delays->blkio_count;
185 d->swapin_count += tsk->delays->swapin_count;
186 d->freepages_count += tsk->delays->freepages_count;
187 d->thrashing_count += tsk->delays->thrashing_count;
188 d->compact_count += tsk->delays->compact_count;
189 d->wpcopy_count += tsk->delays->wpcopy_count;
190 raw_spin_unlock_irqrestore(&tsk->delays->lock, flags);
191
192 return 0;
193 }
194
__delayacct_blkio_ticks(struct task_struct * tsk)195 __u64 __delayacct_blkio_ticks(struct task_struct *tsk)
196 {
197 __u64 ret;
198 unsigned long flags;
199
200 raw_spin_lock_irqsave(&tsk->delays->lock, flags);
201 ret = nsec_to_clock_t(tsk->delays->blkio_delay);
202 raw_spin_unlock_irqrestore(&tsk->delays->lock, flags);
203 return ret;
204 }
205
__delayacct_freepages_start(void)206 void __delayacct_freepages_start(void)
207 {
208 current->delays->freepages_start = local_clock();
209 }
210
__delayacct_freepages_end(void)211 void __delayacct_freepages_end(void)
212 {
213 delayacct_end(¤t->delays->lock,
214 ¤t->delays->freepages_start,
215 ¤t->delays->freepages_delay,
216 ¤t->delays->freepages_count);
217 }
218
__delayacct_thrashing_start(bool * in_thrashing)219 void __delayacct_thrashing_start(bool *in_thrashing)
220 {
221 *in_thrashing = !!current->in_thrashing;
222 if (*in_thrashing)
223 return;
224
225 current->in_thrashing = 1;
226 current->delays->thrashing_start = local_clock();
227 }
228
__delayacct_thrashing_end(bool * in_thrashing)229 void __delayacct_thrashing_end(bool *in_thrashing)
230 {
231 if (*in_thrashing)
232 return;
233
234 current->in_thrashing = 0;
235 delayacct_end(¤t->delays->lock,
236 ¤t->delays->thrashing_start,
237 ¤t->delays->thrashing_delay,
238 ¤t->delays->thrashing_count);
239 }
240
__delayacct_swapin_start(void)241 void __delayacct_swapin_start(void)
242 {
243 current->delays->swapin_start = local_clock();
244 }
245
__delayacct_swapin_end(void)246 void __delayacct_swapin_end(void)
247 {
248 delayacct_end(¤t->delays->lock,
249 ¤t->delays->swapin_start,
250 ¤t->delays->swapin_delay,
251 ¤t->delays->swapin_count);
252 }
253
__delayacct_compact_start(void)254 void __delayacct_compact_start(void)
255 {
256 current->delays->compact_start = local_clock();
257 }
258
__delayacct_compact_end(void)259 void __delayacct_compact_end(void)
260 {
261 delayacct_end(¤t->delays->lock,
262 ¤t->delays->compact_start,
263 ¤t->delays->compact_delay,
264 ¤t->delays->compact_count);
265 }
266
__delayacct_wpcopy_start(void)267 void __delayacct_wpcopy_start(void)
268 {
269 current->delays->wpcopy_start = local_clock();
270 }
271
__delayacct_wpcopy_end(void)272 void __delayacct_wpcopy_end(void)
273 {
274 delayacct_end(¤t->delays->lock,
275 ¤t->delays->wpcopy_start,
276 ¤t->delays->wpcopy_delay,
277 ¤t->delays->wpcopy_count);
278 }
279 #else
280 #include <trace/hooks/delayacct.h>
281
282 int delayacct_enabled __read_mostly; /* Delay accounting turned on/off */
get_delayacct_enabled(void)283 bool get_delayacct_enabled(void)
284 {
285 return delayacct_enabled;
286 }
287
set_delayacct_enabled(bool enabled)288 void set_delayacct_enabled(bool enabled)
289 {
290 delayacct_enabled = enabled;
291 }
292 EXPORT_SYMBOL_GPL(set_delayacct_enabled);
293
_trace_android_rvh_delayacct_init(void)294 void _trace_android_rvh_delayacct_init(void)
295 {
296 trace_android_rvh_delayacct_init(NULL);
297 }
298
_trace_android_rvh_delayacct_tsk_init(struct task_struct * tsk)299 void _trace_android_rvh_delayacct_tsk_init(struct task_struct *tsk)
300 {
301 trace_android_rvh_delayacct_tsk_init(tsk);
302 }
303
_trace_android_rvh_delayacct_tsk_free(struct task_struct * tsk)304 void _trace_android_rvh_delayacct_tsk_free(struct task_struct *tsk)
305 {
306 trace_android_rvh_delayacct_tsk_free(tsk);
307 }
308
_trace_android_vh_delayacct_blkio_start(void)309 void _trace_android_vh_delayacct_blkio_start(void)
310 {
311 trace_android_vh_delayacct_blkio_start(NULL);
312 }
313
_trace_android_vh_delayacct_blkio_end(struct task_struct * p)314 void _trace_android_vh_delayacct_blkio_end(struct task_struct *p)
315 {
316 trace_android_vh_delayacct_blkio_end(p);
317 }
318
_trace_android_vh_delayacct_add_tsk(struct taskstats * d,struct task_struct * tsk,int * ret)319 void _trace_android_vh_delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk, int *ret)
320 {
321 trace_android_vh_delayacct_add_tsk(d, tsk, ret);
322 }
323
_trace_android_vh_delayacct_blkio_ticks(struct task_struct * tsk,__u64 * ret)324 void _trace_android_vh_delayacct_blkio_ticks(struct task_struct *tsk, __u64 *ret)
325 {
326 trace_android_vh_delayacct_blkio_ticks(tsk, ret);
327 }
328
_trace_android_vh_delayacct_freepages_start(void)329 void _trace_android_vh_delayacct_freepages_start(void)
330 {
331 trace_android_vh_delayacct_freepages_start(NULL);
332 }
333
_trace_android_vh_delayacct_freepages_end(void)334 void _trace_android_vh_delayacct_freepages_end(void)
335 {
336 trace_android_vh_delayacct_freepages_end(NULL);
337 }
338
_trace_android_vh_delayacct_thrashing_start(bool * in_thrashing)339 void _trace_android_vh_delayacct_thrashing_start(bool *in_thrashing)
340 {
341 trace_android_vh_delayacct_thrashing_start(in_thrashing);
342 }
343
_trace_android_vh_delayacct_thrashing_end(bool * in_thrashing)344 void _trace_android_vh_delayacct_thrashing_end(bool *in_thrashing)
345 {
346 trace_android_vh_delayacct_thrashing_end(in_thrashing);
347 }
348
_trace_android_vh_delayacct_swapin_start(void)349 void _trace_android_vh_delayacct_swapin_start(void)
350 {
351 trace_android_vh_delayacct_swapin_start(NULL);
352 }
353
_trace_android_vh_delayacct_swapin_end(void)354 void _trace_android_vh_delayacct_swapin_end(void)
355 {
356 trace_android_vh_delayacct_swapin_end(NULL);
357 }
358
_trace_android_vh_delayacct_compact_start(void)359 void _trace_android_vh_delayacct_compact_start(void)
360 {
361 trace_android_vh_delayacct_compact_start(NULL);
362 }
363
_trace_android_vh_delayacct_compact_end(void)364 void _trace_android_vh_delayacct_compact_end(void)
365 {
366 trace_android_vh_delayacct_compact_end(NULL);
367 }
368
_trace_android_vh_delayacct_wpcopy_start(void)369 void _trace_android_vh_delayacct_wpcopy_start(void)
370 {
371 trace_android_vh_delayacct_wpcopy_start(NULL);
372 }
373
_trace_android_vh_delayacct_wpcopy_end(void)374 void _trace_android_vh_delayacct_wpcopy_end(void)
375 {
376 trace_android_vh_delayacct_wpcopy_end(NULL);
377 }
378
379 #endif
380
381