1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2019 Facebook
3 #include <vmlinux.h>
4 #include <bpf/bpf_helpers.h>
5 #include "runqslower.h"
6 #include "core_fixes.bpf.h"
7
8 #define TASK_RUNNING 0
9
10 const volatile __u64 min_us = 0;
11 const volatile pid_t targ_pid = 0;
12 const volatile pid_t targ_tgid = 0;
13
14 struct {
15 __uint(type, BPF_MAP_TYPE_HASH);
16 __uint(max_entries, 10240);
17 __type(key, u32);
18 __type(value, u64);
19 } start SEC(".maps");
20
21 struct {
22 __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
23 __uint(key_size, sizeof(u32));
24 __uint(value_size, sizeof(u32));
25 } events SEC(".maps");
26
27 /* record enqueue timestamp */
28 static __always_inline
trace_enqueue(u32 tgid,u32 pid)29 int trace_enqueue(u32 tgid, u32 pid)
30 {
31 u64 ts;
32
33 if (!pid)
34 return 0;
35 if (targ_tgid && targ_tgid != tgid)
36 return 0;
37 if (targ_pid && targ_pid != pid)
38 return 0;
39
40 ts = bpf_ktime_get_ns();
41 bpf_map_update_elem(&start, &pid, &ts, 0);
42 return 0;
43 }
44
45 SEC("tp_btf/sched_wakeup")
handle__sched_wakeup(u64 * ctx)46 int handle__sched_wakeup(u64 *ctx)
47 {
48 /* TP_PROTO(struct task_struct *p) */
49 struct task_struct *p = (void *)ctx[0];
50
51 return trace_enqueue(p->tgid, p->pid);
52 }
53
54 SEC("tp_btf/sched_wakeup_new")
handle__sched_wakeup_new(u64 * ctx)55 int handle__sched_wakeup_new(u64 *ctx)
56 {
57 /* TP_PROTO(struct task_struct *p) */
58 struct task_struct *p = (void *)ctx[0];
59
60 return trace_enqueue(p->tgid, p->pid);
61 }
62
63 SEC("tp_btf/sched_switch")
handle__sched_switch(u64 * ctx)64 int handle__sched_switch(u64 *ctx)
65 {
66 /* TP_PROTO(bool preempt, struct task_struct *prev,
67 * struct task_struct *next)
68 */
69 struct task_struct *prev = (struct task_struct *)ctx[1];
70 struct task_struct *next = (struct task_struct *)ctx[2];
71 struct event event = {};
72 u64 *tsp, delta_us;
73 u32 pid;
74
75 /* ivcsw: treat like an enqueue event and store timestamp */
76 if (get_task_state(prev) == TASK_RUNNING)
77 trace_enqueue(prev->tgid, prev->pid);
78
79 pid = next->pid;
80
81 /* fetch timestamp and calculate delta */
82 tsp = bpf_map_lookup_elem(&start, &pid);
83 if (!tsp)
84 return 0; /* missed enqueue */
85
86 delta_us = (bpf_ktime_get_ns() - *tsp) / 1000;
87 if (min_us && delta_us <= min_us)
88 return 0;
89
90 event.pid = pid;
91 event.prev_pid = prev->pid;
92 event.delta_us = delta_us;
93 bpf_probe_read_kernel_str(&event.task, sizeof(event.task), next->comm);
94 bpf_probe_read_kernel_str(&event.prev_task, sizeof(event.prev_task), prev->comm);
95
96 /* output */
97 bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU,
98 &event, sizeof(event));
99
100 bpf_map_delete_elem(&start, &pid);
101 return 0;
102 }
103
104 char LICENSE[] SEC("license") = "GPL";
105