1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 // Copyright (c) 2022 Google
3 #include "vmlinux.h"
4 #include <bpf/bpf_helpers.h>
5 #include <bpf/bpf_tracing.h>
6 #include <bpf/bpf_core_read.h>
7
8 /* maximum stack trace depth */
9 #define MAX_STACKS 8
10
11 /* default buffer size */
12 #define MAX_ENTRIES 10240
13
14 struct contention_key {
15 __s32 stack_id;
16 };
17
18 struct contention_data {
19 __u64 total_time;
20 __u64 min_time;
21 __u64 max_time;
22 __u32 count;
23 __u32 flags;
24 };
25
26 struct tstamp_data {
27 __u64 timestamp;
28 __u64 lock;
29 __u32 flags;
30 __s32 stack_id;
31 };
32
33 /* callstack storage */
34 struct {
35 __uint(type, BPF_MAP_TYPE_STACK_TRACE);
36 __uint(key_size, sizeof(__u32));
37 __uint(value_size, MAX_STACKS * sizeof(__u64));
38 __uint(max_entries, MAX_ENTRIES);
39 } stacks SEC(".maps");
40
41 /* maintain timestamp at the beginning of contention */
42 struct {
43 __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
44 __uint(map_flags, BPF_F_NO_PREALLOC);
45 __type(key, int);
46 __type(value, struct tstamp_data);
47 } tstamp SEC(".maps");
48
49 /* actual lock contention statistics */
50 struct {
51 __uint(type, BPF_MAP_TYPE_HASH);
52 __uint(key_size, sizeof(struct contention_key));
53 __uint(value_size, sizeof(struct contention_data));
54 __uint(max_entries, MAX_ENTRIES);
55 } lock_stat SEC(".maps");
56
57 struct {
58 __uint(type, BPF_MAP_TYPE_HASH);
59 __uint(key_size, sizeof(__u32));
60 __uint(value_size, sizeof(__u8));
61 __uint(max_entries, 1);
62 } cpu_filter SEC(".maps");
63
64 struct {
65 __uint(type, BPF_MAP_TYPE_HASH);
66 __uint(key_size, sizeof(__u32));
67 __uint(value_size, sizeof(__u8));
68 __uint(max_entries, 1);
69 } task_filter SEC(".maps");
70
71 /* control flags */
72 int enabled;
73 int has_cpu;
74 int has_task;
75 int stack_skip;
76
77 /* error stat */
78 int lost;
79
can_record(void)80 static inline int can_record(void)
81 {
82 if (has_cpu) {
83 __u32 cpu = bpf_get_smp_processor_id();
84 __u8 *ok;
85
86 ok = bpf_map_lookup_elem(&cpu_filter, &cpu);
87 if (!ok)
88 return 0;
89 }
90
91 if (has_task) {
92 __u8 *ok;
93 __u32 pid = bpf_get_current_pid_tgid();
94
95 ok = bpf_map_lookup_elem(&task_filter, &pid);
96 if (!ok)
97 return 0;
98 }
99
100 return 1;
101 }
102
103 SEC("tp_btf/contention_begin")
contention_begin(u64 * ctx)104 int contention_begin(u64 *ctx)
105 {
106 struct task_struct *curr;
107 struct tstamp_data *pelem;
108
109 if (!enabled || !can_record())
110 return 0;
111
112 curr = bpf_get_current_task_btf();
113 pelem = bpf_task_storage_get(&tstamp, curr, NULL,
114 BPF_LOCAL_STORAGE_GET_F_CREATE);
115 if (!pelem || pelem->lock)
116 return 0;
117
118 pelem->timestamp = bpf_ktime_get_ns();
119 pelem->lock = (__u64)ctx[0];
120 pelem->flags = (__u32)ctx[1];
121 pelem->stack_id = bpf_get_stackid(ctx, &stacks, BPF_F_FAST_STACK_CMP | stack_skip);
122
123 if (pelem->stack_id < 0)
124 lost++;
125 return 0;
126 }
127
128 SEC("tp_btf/contention_end")
contention_end(u64 * ctx)129 int contention_end(u64 *ctx)
130 {
131 struct task_struct *curr;
132 struct tstamp_data *pelem;
133 struct contention_key key;
134 struct contention_data *data;
135 __u64 duration;
136
137 if (!enabled)
138 return 0;
139
140 curr = bpf_get_current_task_btf();
141 pelem = bpf_task_storage_get(&tstamp, curr, NULL, 0);
142 if (!pelem || pelem->lock != ctx[0])
143 return 0;
144
145 duration = bpf_ktime_get_ns() - pelem->timestamp;
146
147 key.stack_id = pelem->stack_id;
148 data = bpf_map_lookup_elem(&lock_stat, &key);
149 if (!data) {
150 struct contention_data first = {
151 .total_time = duration,
152 .max_time = duration,
153 .min_time = duration,
154 .count = 1,
155 .flags = pelem->flags,
156 };
157
158 bpf_map_update_elem(&lock_stat, &key, &first, BPF_NOEXIST);
159 pelem->lock = 0;
160 return 0;
161 }
162
163 __sync_fetch_and_add(&data->total_time, duration);
164 __sync_fetch_and_add(&data->count, 1);
165
166 /* FIXME: need atomic operations */
167 if (data->max_time < duration)
168 data->max_time = duration;
169 if (data->min_time > duration)
170 data->min_time = duration;
171
172 pelem->lock = 0;
173 return 0;
174 }
175
176 char LICENSE[] SEC("license") = "Dual BSD/GPL";
177