1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2019 Facebook
3 #include "vmlinux.h"
4 #include <bpf/bpf_helpers.h>
5 #include "runqslower.h"
6
7 #define TASK_RUNNING 0
8 #define BPF_F_CURRENT_CPU 0xffffffffULL
9
10 const volatile __u64 min_us = 0;
11 const volatile pid_t targ_pid = 0;
12
13 struct {
14 __uint(type, BPF_MAP_TYPE_HASH);
15 __uint(max_entries, 10240);
16 __type(key, u32);
17 __type(value, u64);
18 } start SEC(".maps");
19
20 struct {
21 __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
22 __uint(key_size, sizeof(u32));
23 __uint(value_size, sizeof(u32));
24 } events SEC(".maps");
25
26 /* record enqueue timestamp */
27 __always_inline
trace_enqueue(u32 tgid,u32 pid)28 static int trace_enqueue(u32 tgid, u32 pid)
29 {
30 u64 ts;
31
32 if (!pid || (targ_pid && targ_pid != pid))
33 return 0;
34
35 ts = bpf_ktime_get_ns();
36 bpf_map_update_elem(&start, &pid, &ts, 0);
37 return 0;
38 }
39
40 SEC("tp_btf/sched_wakeup")
handle__sched_wakeup(u64 * ctx)41 int handle__sched_wakeup(u64 *ctx)
42 {
43 /* TP_PROTO(struct task_struct *p) */
44 struct task_struct *p = (void *)ctx[0];
45
46 return trace_enqueue(p->tgid, p->pid);
47 }
48
49 SEC("tp_btf/sched_wakeup_new")
handle__sched_wakeup_new(u64 * ctx)50 int handle__sched_wakeup_new(u64 *ctx)
51 {
52 /* TP_PROTO(struct task_struct *p) */
53 struct task_struct *p = (void *)ctx[0];
54
55 return trace_enqueue(p->tgid, p->pid);
56 }
57
58 SEC("tp_btf/sched_switch")
handle__sched_switch(u64 * ctx)59 int handle__sched_switch(u64 *ctx)
60 {
61 /* TP_PROTO(bool preempt, struct task_struct *prev,
62 * struct task_struct *next)
63 */
64 struct task_struct *prev = (struct task_struct *)ctx[1];
65 struct task_struct *next = (struct task_struct *)ctx[2];
66 struct event event = {};
67 u64 *tsp, delta_us;
68 long state;
69 u32 pid;
70
71 /* ivcsw: treat like an enqueue event and store timestamp */
72 if (prev->state == TASK_RUNNING)
73 trace_enqueue(prev->tgid, prev->pid);
74
75 pid = next->pid;
76
77 /* fetch timestamp and calculate delta */
78 tsp = bpf_map_lookup_elem(&start, &pid);
79 if (!tsp)
80 return 0; /* missed enqueue */
81
82 delta_us = (bpf_ktime_get_ns() - *tsp) / 1000;
83 if (min_us && delta_us <= min_us)
84 return 0;
85
86 event.pid = pid;
87 event.delta_us = delta_us;
88 bpf_get_current_comm(&event.task, sizeof(event.task));
89
90 /* output */
91 bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU,
92 &event, sizeof(event));
93
94 bpf_map_delete_elem(&start, &pid);
95 return 0;
96 }
97
98 char LICENSE[] SEC("license") = "GPL";
99