1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * bpf_kwork.c
4 *
5 * Copyright (c) 2022 Huawei Inc, Yang Jihong <yangjihong1@huawei.com>
6 */
7
8 #include <time.h>
9 #include <fcntl.h>
10 #include <stdio.h>
11 #include <unistd.h>
12
13 #include <linux/time64.h>
14
15 #include "util/debug.h"
16 #include "util/kwork.h"
17
18 #include <bpf/bpf.h>
19
20 #include "util/bpf_skel/kwork_trace.skel.h"
21
22 /*
23 * This should be in sync with "util/kwork_trace.bpf.c"
24 */
25 #define MAX_KWORKNAME 128
26
27 struct work_key {
28 u32 type;
29 u32 cpu;
30 u64 id;
31 };
32
33 struct report_data {
34 u64 nr;
35 u64 total_time;
36 u64 max_time;
37 u64 max_time_start;
38 u64 max_time_end;
39 };
40
41 struct kwork_class_bpf {
42 struct kwork_class *class;
43
44 void (*load_prepare)(struct perf_kwork *kwork);
45 int (*get_work_name)(struct work_key *key, char **ret_name);
46 };
47
48 static struct kwork_trace_bpf *skel;
49
50 static struct timespec ts_start;
51 static struct timespec ts_end;
52
perf_kwork__trace_start(void)53 void perf_kwork__trace_start(void)
54 {
55 clock_gettime(CLOCK_MONOTONIC, &ts_start);
56 skel->bss->enabled = 1;
57 }
58
perf_kwork__trace_finish(void)59 void perf_kwork__trace_finish(void)
60 {
61 clock_gettime(CLOCK_MONOTONIC, &ts_end);
62 skel->bss->enabled = 0;
63 }
64
get_work_name_from_map(struct work_key * key,char ** ret_name)65 static int get_work_name_from_map(struct work_key *key, char **ret_name)
66 {
67 char name[MAX_KWORKNAME] = { 0 };
68 int fd = bpf_map__fd(skel->maps.perf_kwork_names);
69
70 *ret_name = NULL;
71
72 if (fd < 0) {
73 pr_debug("Invalid names map fd\n");
74 return 0;
75 }
76
77 if ((bpf_map_lookup_elem(fd, key, name) == 0) && (strlen(name) != 0)) {
78 *ret_name = strdup(name);
79 if (*ret_name == NULL) {
80 pr_err("Failed to copy work name\n");
81 return -1;
82 }
83 }
84
85 return 0;
86 }
87
irq_load_prepare(struct perf_kwork * kwork)88 static void irq_load_prepare(struct perf_kwork *kwork)
89 {
90 if (kwork->report == KWORK_REPORT_RUNTIME) {
91 bpf_program__set_autoload(skel->progs.report_irq_handler_entry, true);
92 bpf_program__set_autoload(skel->progs.report_irq_handler_exit, true);
93 }
94 }
95
96 static struct kwork_class_bpf kwork_irq_bpf = {
97 .load_prepare = irq_load_prepare,
98 .get_work_name = get_work_name_from_map,
99 };
100
softirq_load_prepare(struct perf_kwork * kwork)101 static void softirq_load_prepare(struct perf_kwork *kwork)
102 {
103 if (kwork->report == KWORK_REPORT_RUNTIME) {
104 bpf_program__set_autoload(skel->progs.report_softirq_entry, true);
105 bpf_program__set_autoload(skel->progs.report_softirq_exit, true);
106 } else if (kwork->report == KWORK_REPORT_LATENCY) {
107 bpf_program__set_autoload(skel->progs.latency_softirq_raise, true);
108 bpf_program__set_autoload(skel->progs.latency_softirq_entry, true);
109 }
110 }
111
112 static struct kwork_class_bpf kwork_softirq_bpf = {
113 .load_prepare = softirq_load_prepare,
114 .get_work_name = get_work_name_from_map,
115 };
116
workqueue_load_prepare(struct perf_kwork * kwork)117 static void workqueue_load_prepare(struct perf_kwork *kwork)
118 {
119 if (kwork->report == KWORK_REPORT_RUNTIME) {
120 bpf_program__set_autoload(skel->progs.report_workqueue_execute_start, true);
121 bpf_program__set_autoload(skel->progs.report_workqueue_execute_end, true);
122 } else if (kwork->report == KWORK_REPORT_LATENCY) {
123 bpf_program__set_autoload(skel->progs.latency_workqueue_activate_work, true);
124 bpf_program__set_autoload(skel->progs.latency_workqueue_execute_start, true);
125 }
126 }
127
128 static struct kwork_class_bpf kwork_workqueue_bpf = {
129 .load_prepare = workqueue_load_prepare,
130 .get_work_name = get_work_name_from_map,
131 };
132
133 static struct kwork_class_bpf *
134 kwork_class_bpf_supported_list[KWORK_CLASS_MAX] = {
135 [KWORK_CLASS_IRQ] = &kwork_irq_bpf,
136 [KWORK_CLASS_SOFTIRQ] = &kwork_softirq_bpf,
137 [KWORK_CLASS_WORKQUEUE] = &kwork_workqueue_bpf,
138 };
139
valid_kwork_class_type(enum kwork_class_type type)140 static bool valid_kwork_class_type(enum kwork_class_type type)
141 {
142 return type >= 0 && type < KWORK_CLASS_MAX ? true : false;
143 }
144
setup_filters(struct perf_kwork * kwork)145 static int setup_filters(struct perf_kwork *kwork)
146 {
147 u8 val = 1;
148 int i, nr_cpus, key, fd;
149 struct perf_cpu_map *map;
150
151 if (kwork->cpu_list != NULL) {
152 fd = bpf_map__fd(skel->maps.perf_kwork_cpu_filter);
153 if (fd < 0) {
154 pr_debug("Invalid cpu filter fd\n");
155 return -1;
156 }
157
158 map = perf_cpu_map__new(kwork->cpu_list);
159 if (map == NULL) {
160 pr_debug("Invalid cpu_list\n");
161 return -1;
162 }
163
164 nr_cpus = libbpf_num_possible_cpus();
165 for (i = 0; i < perf_cpu_map__nr(map); i++) {
166 struct perf_cpu cpu = perf_cpu_map__cpu(map, i);
167
168 if (cpu.cpu >= nr_cpus) {
169 perf_cpu_map__put(map);
170 pr_err("Requested cpu %d too large\n", cpu.cpu);
171 return -1;
172 }
173 bpf_map_update_elem(fd, &cpu.cpu, &val, BPF_ANY);
174 }
175 perf_cpu_map__put(map);
176
177 skel->bss->has_cpu_filter = 1;
178 }
179
180 if (kwork->profile_name != NULL) {
181 if (strlen(kwork->profile_name) >= MAX_KWORKNAME) {
182 pr_err("Requested name filter %s too large, limit to %d\n",
183 kwork->profile_name, MAX_KWORKNAME - 1);
184 return -1;
185 }
186
187 fd = bpf_map__fd(skel->maps.perf_kwork_name_filter);
188 if (fd < 0) {
189 pr_debug("Invalid name filter fd\n");
190 return -1;
191 }
192
193 key = 0;
194 bpf_map_update_elem(fd, &key, kwork->profile_name, BPF_ANY);
195
196 skel->bss->has_name_filter = 1;
197 }
198
199 return 0;
200 }
201
perf_kwork__trace_prepare_bpf(struct perf_kwork * kwork)202 int perf_kwork__trace_prepare_bpf(struct perf_kwork *kwork)
203 {
204 struct bpf_program *prog;
205 struct kwork_class *class;
206 struct kwork_class_bpf *class_bpf;
207 enum kwork_class_type type;
208
209 skel = kwork_trace_bpf__open();
210 if (!skel) {
211 pr_debug("Failed to open kwork trace skeleton\n");
212 return -1;
213 }
214
215 /*
216 * set all progs to non-autoload,
217 * then set corresponding progs according to config
218 */
219 bpf_object__for_each_program(prog, skel->obj)
220 bpf_program__set_autoload(prog, false);
221
222 list_for_each_entry(class, &kwork->class_list, list) {
223 type = class->type;
224 if (!valid_kwork_class_type(type) ||
225 (kwork_class_bpf_supported_list[type] == NULL)) {
226 pr_err("Unsupported bpf trace class %s\n", class->name);
227 goto out;
228 }
229
230 class_bpf = kwork_class_bpf_supported_list[type];
231 class_bpf->class = class;
232
233 if (class_bpf->load_prepare != NULL)
234 class_bpf->load_prepare(kwork);
235 }
236
237 if (kwork_trace_bpf__load(skel)) {
238 pr_debug("Failed to load kwork trace skeleton\n");
239 goto out;
240 }
241
242 if (setup_filters(kwork))
243 goto out;
244
245 if (kwork_trace_bpf__attach(skel)) {
246 pr_debug("Failed to attach kwork trace skeleton\n");
247 goto out;
248 }
249
250 return 0;
251
252 out:
253 kwork_trace_bpf__destroy(skel);
254 return -1;
255 }
256
add_work(struct perf_kwork * kwork,struct work_key * key,struct report_data * data)257 static int add_work(struct perf_kwork *kwork,
258 struct work_key *key,
259 struct report_data *data)
260 {
261 struct kwork_work *work;
262 struct kwork_class_bpf *bpf_trace;
263 struct kwork_work tmp = {
264 .id = key->id,
265 .name = NULL,
266 .cpu = key->cpu,
267 };
268 enum kwork_class_type type = key->type;
269
270 if (!valid_kwork_class_type(type)) {
271 pr_debug("Invalid class type %d to add work\n", type);
272 return -1;
273 }
274
275 bpf_trace = kwork_class_bpf_supported_list[type];
276 tmp.class = bpf_trace->class;
277
278 if ((bpf_trace->get_work_name != NULL) &&
279 (bpf_trace->get_work_name(key, &tmp.name)))
280 return -1;
281
282 work = perf_kwork_add_work(kwork, tmp.class, &tmp);
283 if (work == NULL)
284 return -1;
285
286 if (kwork->report == KWORK_REPORT_RUNTIME) {
287 work->nr_atoms = data->nr;
288 work->total_runtime = data->total_time;
289 work->max_runtime = data->max_time;
290 work->max_runtime_start = data->max_time_start;
291 work->max_runtime_end = data->max_time_end;
292 } else if (kwork->report == KWORK_REPORT_LATENCY) {
293 work->nr_atoms = data->nr;
294 work->total_latency = data->total_time;
295 work->max_latency = data->max_time;
296 work->max_latency_start = data->max_time_start;
297 work->max_latency_end = data->max_time_end;
298 } else {
299 pr_debug("Invalid bpf report type %d\n", kwork->report);
300 return -1;
301 }
302
303 kwork->timestart = (u64)ts_start.tv_sec * NSEC_PER_SEC + ts_start.tv_nsec;
304 kwork->timeend = (u64)ts_end.tv_sec * NSEC_PER_SEC + ts_end.tv_nsec;
305
306 return 0;
307 }
308
perf_kwork__report_read_bpf(struct perf_kwork * kwork)309 int perf_kwork__report_read_bpf(struct perf_kwork *kwork)
310 {
311 struct report_data data;
312 struct work_key key = {
313 .type = 0,
314 .cpu = 0,
315 .id = 0,
316 };
317 struct work_key prev = {
318 .type = 0,
319 .cpu = 0,
320 .id = 0,
321 };
322 int fd = bpf_map__fd(skel->maps.perf_kwork_report);
323
324 if (fd < 0) {
325 pr_debug("Invalid report fd\n");
326 return -1;
327 }
328
329 while (!bpf_map_get_next_key(fd, &prev, &key)) {
330 if ((bpf_map_lookup_elem(fd, &key, &data)) != 0) {
331 pr_debug("Failed to lookup report elem\n");
332 return -1;
333 }
334
335 if ((data.nr != 0) && (add_work(kwork, &key, &data) != 0))
336 return -1;
337
338 prev = key;
339 }
340 return 0;
341 }
342
perf_kwork__report_cleanup_bpf(void)343 void perf_kwork__report_cleanup_bpf(void)
344 {
345 kwork_trace_bpf__destroy(skel);
346 }
347