1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 // Copyright (c) 2022, Huawei
3
4 #include "vmlinux.h"
5 #include <bpf/bpf_helpers.h>
6 #include <bpf/bpf_tracing.h>
7
8 #define KWORK_COUNT 100
9 #define MAX_KWORKNAME 128
10
11 /*
12 * This should be in sync with "util/kwork.h"
13 */
14 enum kwork_class_type {
15 KWORK_CLASS_IRQ,
16 KWORK_CLASS_SOFTIRQ,
17 KWORK_CLASS_WORKQUEUE,
18 KWORK_CLASS_MAX,
19 };
20
21 struct work_key {
22 __u32 type;
23 __u32 cpu;
24 __u64 id;
25 };
26
27 struct report_data {
28 __u64 nr;
29 __u64 total_time;
30 __u64 max_time;
31 __u64 max_time_start;
32 __u64 max_time_end;
33 };
34
35 struct {
36 __uint(type, BPF_MAP_TYPE_HASH);
37 __uint(key_size, sizeof(struct work_key));
38 __uint(value_size, MAX_KWORKNAME);
39 __uint(max_entries, KWORK_COUNT);
40 } perf_kwork_names SEC(".maps");
41
42 struct {
43 __uint(type, BPF_MAP_TYPE_HASH);
44 __uint(key_size, sizeof(struct work_key));
45 __uint(value_size, sizeof(__u64));
46 __uint(max_entries, KWORK_COUNT);
47 } perf_kwork_time SEC(".maps");
48
49 struct {
50 __uint(type, BPF_MAP_TYPE_HASH);
51 __uint(key_size, sizeof(struct work_key));
52 __uint(value_size, sizeof(struct report_data));
53 __uint(max_entries, KWORK_COUNT);
54 } perf_kwork_report SEC(".maps");
55
56 struct {
57 __uint(type, BPF_MAP_TYPE_HASH);
58 __uint(key_size, sizeof(__u32));
59 __uint(value_size, sizeof(__u8));
60 __uint(max_entries, 1);
61 } perf_kwork_cpu_filter SEC(".maps");
62
63 struct {
64 __uint(type, BPF_MAP_TYPE_ARRAY);
65 __uint(key_size, sizeof(__u32));
66 __uint(value_size, MAX_KWORKNAME);
67 __uint(max_entries, 1);
68 } perf_kwork_name_filter SEC(".maps");
69
70 int enabled = 0;
71 int has_cpu_filter = 0;
72 int has_name_filter = 0;
73
local_strncmp(const char * s1,unsigned int sz,const char * s2)74 static __always_inline int local_strncmp(const char *s1,
75 unsigned int sz, const char *s2)
76 {
77 int ret = 0;
78 unsigned int i;
79
80 for (i = 0; i < sz; i++) {
81 ret = (unsigned char)s1[i] - (unsigned char)s2[i];
82 if (ret || !s1[i] || !s2[i])
83 break;
84 }
85
86 return ret;
87 }
88
trace_event_match(struct work_key * key,char * name)89 static __always_inline int trace_event_match(struct work_key *key, char *name)
90 {
91 __u8 *cpu_val;
92 char *name_val;
93 __u32 zero = 0;
94 __u32 cpu = bpf_get_smp_processor_id();
95
96 if (!enabled)
97 return 0;
98
99 if (has_cpu_filter) {
100 cpu_val = bpf_map_lookup_elem(&perf_kwork_cpu_filter, &cpu);
101 if (!cpu_val)
102 return 0;
103 }
104
105 if (has_name_filter && (name != NULL)) {
106 name_val = bpf_map_lookup_elem(&perf_kwork_name_filter, &zero);
107 if (name_val &&
108 (local_strncmp(name_val, MAX_KWORKNAME, name) != 0)) {
109 return 0;
110 }
111 }
112
113 return 1;
114 }
115
do_update_time(void * map,struct work_key * key,__u64 time_start,__u64 time_end)116 static __always_inline void do_update_time(void *map, struct work_key *key,
117 __u64 time_start, __u64 time_end)
118 {
119 struct report_data zero, *data;
120 __s64 delta = time_end - time_start;
121
122 if (delta < 0)
123 return;
124
125 data = bpf_map_lookup_elem(map, key);
126 if (!data) {
127 __builtin_memset(&zero, 0, sizeof(zero));
128 bpf_map_update_elem(map, key, &zero, BPF_NOEXIST);
129 data = bpf_map_lookup_elem(map, key);
130 if (!data)
131 return;
132 }
133
134 if ((delta > data->max_time) ||
135 (data->max_time == 0)) {
136 data->max_time = delta;
137 data->max_time_start = time_start;
138 data->max_time_end = time_end;
139 }
140
141 data->total_time += delta;
142 data->nr++;
143 }
144
do_update_timestart(void * map,struct work_key * key)145 static __always_inline void do_update_timestart(void *map, struct work_key *key)
146 {
147 __u64 ts = bpf_ktime_get_ns();
148
149 bpf_map_update_elem(map, key, &ts, BPF_ANY);
150 }
151
do_update_timeend(void * report_map,void * time_map,struct work_key * key)152 static __always_inline void do_update_timeend(void *report_map, void *time_map,
153 struct work_key *key)
154 {
155 __u64 *time = bpf_map_lookup_elem(time_map, key);
156
157 if (time) {
158 bpf_map_delete_elem(time_map, key);
159 do_update_time(report_map, key, *time, bpf_ktime_get_ns());
160 }
161 }
162
do_update_name(void * map,struct work_key * key,char * name)163 static __always_inline void do_update_name(void *map,
164 struct work_key *key, char *name)
165 {
166 if (!bpf_map_lookup_elem(map, key))
167 bpf_map_update_elem(map, key, name, BPF_ANY);
168 }
169
update_timestart(void * map,struct work_key * key)170 static __always_inline int update_timestart(void *map, struct work_key *key)
171 {
172 if (!trace_event_match(key, NULL))
173 return 0;
174
175 do_update_timestart(map, key);
176 return 0;
177 }
178
update_timestart_and_name(void * time_map,void * names_map,struct work_key * key,char * name)179 static __always_inline int update_timestart_and_name(void *time_map,
180 void *names_map,
181 struct work_key *key,
182 char *name)
183 {
184 if (!trace_event_match(key, name))
185 return 0;
186
187 do_update_timestart(time_map, key);
188 do_update_name(names_map, key, name);
189
190 return 0;
191 }
192
update_timeend(void * report_map,void * time_map,struct work_key * key)193 static __always_inline int update_timeend(void *report_map,
194 void *time_map, struct work_key *key)
195 {
196 if (!trace_event_match(key, NULL))
197 return 0;
198
199 do_update_timeend(report_map, time_map, key);
200
201 return 0;
202 }
203
update_timeend_and_name(void * report_map,void * time_map,void * names_map,struct work_key * key,char * name)204 static __always_inline int update_timeend_and_name(void *report_map,
205 void *time_map,
206 void *names_map,
207 struct work_key *key,
208 char *name)
209 {
210 if (!trace_event_match(key, name))
211 return 0;
212
213 do_update_timeend(report_map, time_map, key);
214 do_update_name(names_map, key, name);
215
216 return 0;
217 }
218
219 SEC("tracepoint/irq/irq_handler_entry")
report_irq_handler_entry(struct trace_event_raw_irq_handler_entry * ctx)220 int report_irq_handler_entry(struct trace_event_raw_irq_handler_entry *ctx)
221 {
222 char name[MAX_KWORKNAME];
223 struct work_key key = {
224 .type = KWORK_CLASS_IRQ,
225 .cpu = bpf_get_smp_processor_id(),
226 .id = (__u64)ctx->irq,
227 };
228 void *name_addr = (void *)ctx + (ctx->__data_loc_name & 0xffff);
229
230 bpf_probe_read_kernel_str(name, sizeof(name), name_addr);
231
232 return update_timestart_and_name(&perf_kwork_time,
233 &perf_kwork_names, &key, name);
234 }
235
236 SEC("tracepoint/irq/irq_handler_exit")
report_irq_handler_exit(struct trace_event_raw_irq_handler_exit * ctx)237 int report_irq_handler_exit(struct trace_event_raw_irq_handler_exit *ctx)
238 {
239 struct work_key key = {
240 .type = KWORK_CLASS_IRQ,
241 .cpu = bpf_get_smp_processor_id(),
242 .id = (__u64)ctx->irq,
243 };
244
245 return update_timeend(&perf_kwork_report, &perf_kwork_time, &key);
246 }
247
248 static char softirq_name_list[NR_SOFTIRQS][MAX_KWORKNAME] = {
249 { "HI" },
250 { "TIMER" },
251 { "NET_TX" },
252 { "NET_RX" },
253 { "BLOCK" },
254 { "IRQ_POLL" },
255 { "TASKLET" },
256 { "SCHED" },
257 { "HRTIMER" },
258 { "RCU" },
259 };
260
261 SEC("tracepoint/irq/softirq_entry")
report_softirq_entry(struct trace_event_raw_softirq * ctx)262 int report_softirq_entry(struct trace_event_raw_softirq *ctx)
263 {
264 unsigned int vec = ctx->vec;
265 struct work_key key = {
266 .type = KWORK_CLASS_SOFTIRQ,
267 .cpu = bpf_get_smp_processor_id(),
268 .id = (__u64)vec,
269 };
270
271 if (vec < NR_SOFTIRQS) {
272 return update_timestart_and_name(&perf_kwork_time,
273 &perf_kwork_names, &key,
274 softirq_name_list[vec]);
275 }
276
277 return 0;
278 }
279
280 SEC("tracepoint/irq/softirq_exit")
report_softirq_exit(struct trace_event_raw_softirq * ctx)281 int report_softirq_exit(struct trace_event_raw_softirq *ctx)
282 {
283 struct work_key key = {
284 .type = KWORK_CLASS_SOFTIRQ,
285 .cpu = bpf_get_smp_processor_id(),
286 .id = (__u64)ctx->vec,
287 };
288
289 return update_timeend(&perf_kwork_report, &perf_kwork_time, &key);
290 }
291
292 SEC("tracepoint/irq/softirq_raise")
latency_softirq_raise(struct trace_event_raw_softirq * ctx)293 int latency_softirq_raise(struct trace_event_raw_softirq *ctx)
294 {
295 unsigned int vec = ctx->vec;
296 struct work_key key = {
297 .type = KWORK_CLASS_SOFTIRQ,
298 .cpu = bpf_get_smp_processor_id(),
299 .id = (__u64)vec,
300 };
301
302 if (vec < NR_SOFTIRQS) {
303 return update_timestart_and_name(&perf_kwork_time,
304 &perf_kwork_names, &key,
305 softirq_name_list[vec]);
306 }
307
308 return 0;
309 }
310
311 SEC("tracepoint/irq/softirq_entry")
latency_softirq_entry(struct trace_event_raw_softirq * ctx)312 int latency_softirq_entry(struct trace_event_raw_softirq *ctx)
313 {
314 struct work_key key = {
315 .type = KWORK_CLASS_SOFTIRQ,
316 .cpu = bpf_get_smp_processor_id(),
317 .id = (__u64)ctx->vec,
318 };
319
320 return update_timeend(&perf_kwork_report, &perf_kwork_time, &key);
321 }
322
323 SEC("tracepoint/workqueue/workqueue_execute_start")
report_workqueue_execute_start(struct trace_event_raw_workqueue_execute_start * ctx)324 int report_workqueue_execute_start(struct trace_event_raw_workqueue_execute_start *ctx)
325 {
326 struct work_key key = {
327 .type = KWORK_CLASS_WORKQUEUE,
328 .cpu = bpf_get_smp_processor_id(),
329 .id = (__u64)ctx->work,
330 };
331
332 return update_timestart(&perf_kwork_time, &key);
333 }
334
335 SEC("tracepoint/workqueue/workqueue_execute_end")
report_workqueue_execute_end(struct trace_event_raw_workqueue_execute_end * ctx)336 int report_workqueue_execute_end(struct trace_event_raw_workqueue_execute_end *ctx)
337 {
338 char name[MAX_KWORKNAME];
339 struct work_key key = {
340 .type = KWORK_CLASS_WORKQUEUE,
341 .cpu = bpf_get_smp_processor_id(),
342 .id = (__u64)ctx->work,
343 };
344 unsigned long long func_addr = (unsigned long long)ctx->function;
345
346 __builtin_memset(name, 0, sizeof(name));
347 bpf_snprintf(name, sizeof(name), "%ps", &func_addr, sizeof(func_addr));
348
349 return update_timeend_and_name(&perf_kwork_report, &perf_kwork_time,
350 &perf_kwork_names, &key, name);
351 }
352
353 SEC("tracepoint/workqueue/workqueue_activate_work")
latency_workqueue_activate_work(struct trace_event_raw_workqueue_activate_work * ctx)354 int latency_workqueue_activate_work(struct trace_event_raw_workqueue_activate_work *ctx)
355 {
356 struct work_key key = {
357 .type = KWORK_CLASS_WORKQUEUE,
358 .cpu = bpf_get_smp_processor_id(),
359 .id = (__u64)ctx->work,
360 };
361
362 return update_timestart(&perf_kwork_time, &key);
363 }
364
365 SEC("tracepoint/workqueue/workqueue_execute_start")
latency_workqueue_execute_start(struct trace_event_raw_workqueue_execute_start * ctx)366 int latency_workqueue_execute_start(struct trace_event_raw_workqueue_execute_start *ctx)
367 {
368 char name[MAX_KWORKNAME];
369 struct work_key key = {
370 .type = KWORK_CLASS_WORKQUEUE,
371 .cpu = bpf_get_smp_processor_id(),
372 .id = (__u64)ctx->work,
373 };
374 unsigned long long func_addr = (unsigned long long)ctx->function;
375
376 __builtin_memset(name, 0, sizeof(name));
377 bpf_snprintf(name, sizeof(name), "%ps", &func_addr, sizeof(func_addr));
378
379 return update_timeend_and_name(&perf_kwork_report, &perf_kwork_time,
380 &perf_kwork_names, &key, name);
381 }
382
383 char LICENSE[] SEC("license") = "Dual BSD/GPL";
384