1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
3 #include <stdio.h>
4 #include <stdlib.h>
5 #include <sys/socket.h>
6 #include <sys/ioctl.h>
7 #include <sys/select.h>
8 #include <netinet/in.h>
9 #include <arpa/inet.h>
10 #include <unistd.h>
11 #include <string.h>
12 #include <errno.h>
13 #include <sys/ioctl.h>
14 #include <stdbool.h>
15 #include <signal.h>
16 #include <fcntl.h>
17 #include <sys/wait.h>
18 #include <time.h>
19 #include <sched.h>
20
21 #include <sys/time.h>
22 #include <sys/resource.h>
23 #include <sys/types.h>
24 #include <sys/sendfile.h>
25
26 #include <linux/netlink.h>
27 #include <linux/socket.h>
28 #include <linux/sock_diag.h>
29 #include <linux/bpf.h>
30 #include <linux/if_link.h>
31 #include <assert.h>
32 #include <libgen.h>
33
34 #include <getopt.h>
35
36 #include <bpf/bpf.h>
37 #include <bpf/libbpf.h>
38
39 #include "bpf_util.h"
40 #include "bpf_rlimit.h"
41 #include "cgroup_helpers.h"
42
43 int running;
44 static void running_handler(int a);
45
46 /* randomly selected ports for testing on lo */
47 #define S1_PORT 10000
48 #define S2_PORT 10001
49
50 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o"
51 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
52 #define CG_PATH "/sockmap"
53
54 /* global sockets */
55 int s1, s2, c1, c2, p1, p2;
56 int test_cnt;
57 int passed;
58 int failed;
59 int map_fd[8];
60 struct bpf_map *maps[8];
61 int prog_fd[11];
62
63 int txmsg_pass;
64 int txmsg_noisy;
65 int txmsg_redir;
66 int txmsg_redir_noisy;
67 int txmsg_drop;
68 int txmsg_apply;
69 int txmsg_cork;
70 int txmsg_start;
71 int txmsg_end;
72 int txmsg_ingress;
73 int txmsg_skb;
74
75 static const struct option long_options[] = {
76 {"help", no_argument, NULL, 'h' },
77 {"cgroup", required_argument, NULL, 'c' },
78 {"rate", required_argument, NULL, 'r' },
79 {"verbose", no_argument, NULL, 'v' },
80 {"iov_count", required_argument, NULL, 'i' },
81 {"length", required_argument, NULL, 'l' },
82 {"test", required_argument, NULL, 't' },
83 {"data_test", no_argument, NULL, 'd' },
84 {"txmsg", no_argument, &txmsg_pass, 1 },
85 {"txmsg_noisy", no_argument, &txmsg_noisy, 1 },
86 {"txmsg_redir", no_argument, &txmsg_redir, 1 },
87 {"txmsg_redir_noisy", no_argument, &txmsg_redir_noisy, 1},
88 {"txmsg_drop", no_argument, &txmsg_drop, 1 },
89 {"txmsg_apply", required_argument, NULL, 'a'},
90 {"txmsg_cork", required_argument, NULL, 'k'},
91 {"txmsg_start", required_argument, NULL, 's'},
92 {"txmsg_end", required_argument, NULL, 'e'},
93 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
94 {"txmsg_skb", no_argument, &txmsg_skb, 1 },
95 {0, 0, NULL, 0 }
96 };
97
usage(char * argv[])98 static void usage(char *argv[])
99 {
100 int i;
101
102 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
103 printf(" options:\n");
104 for (i = 0; long_options[i].name != 0; i++) {
105 printf(" --%-12s", long_options[i].name);
106 if (long_options[i].flag != NULL)
107 printf(" flag (internal value:%d)\n",
108 *long_options[i].flag);
109 else
110 printf(" -%c\n", long_options[i].val);
111 }
112 printf("\n");
113 }
114
sockmap_init_sockets(int verbose)115 static int sockmap_init_sockets(int verbose)
116 {
117 int i, err, one = 1;
118 struct sockaddr_in addr;
119 int *fds[4] = {&s1, &s2, &c1, &c2};
120
121 s1 = s2 = p1 = p2 = c1 = c2 = 0;
122
123 /* Init sockets */
124 for (i = 0; i < 4; i++) {
125 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
126 if (*fds[i] < 0) {
127 perror("socket s1 failed()");
128 return errno;
129 }
130 }
131
132 /* Allow reuse */
133 for (i = 0; i < 2; i++) {
134 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
135 (char *)&one, sizeof(one));
136 if (err) {
137 perror("setsockopt failed()");
138 return errno;
139 }
140 }
141
142 /* Non-blocking sockets */
143 for (i = 0; i < 2; i++) {
144 err = ioctl(*fds[i], FIONBIO, (char *)&one);
145 if (err < 0) {
146 perror("ioctl s1 failed()");
147 return errno;
148 }
149 }
150
151 /* Bind server sockets */
152 memset(&addr, 0, sizeof(struct sockaddr_in));
153 addr.sin_family = AF_INET;
154 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
155
156 addr.sin_port = htons(S1_PORT);
157 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
158 if (err < 0) {
159 perror("bind s1 failed()\n");
160 return errno;
161 }
162
163 addr.sin_port = htons(S2_PORT);
164 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
165 if (err < 0) {
166 perror("bind s2 failed()\n");
167 return errno;
168 }
169
170 /* Listen server sockets */
171 addr.sin_port = htons(S1_PORT);
172 err = listen(s1, 32);
173 if (err < 0) {
174 perror("listen s1 failed()\n");
175 return errno;
176 }
177
178 addr.sin_port = htons(S2_PORT);
179 err = listen(s2, 32);
180 if (err < 0) {
181 perror("listen s1 failed()\n");
182 return errno;
183 }
184
185 /* Initiate Connect */
186 addr.sin_port = htons(S1_PORT);
187 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
188 if (err < 0 && errno != EINPROGRESS) {
189 perror("connect c1 failed()\n");
190 return errno;
191 }
192
193 addr.sin_port = htons(S2_PORT);
194 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
195 if (err < 0 && errno != EINPROGRESS) {
196 perror("connect c2 failed()\n");
197 return errno;
198 } else if (err < 0) {
199 err = 0;
200 }
201
202 /* Accept Connecrtions */
203 p1 = accept(s1, NULL, NULL);
204 if (p1 < 0) {
205 perror("accept s1 failed()\n");
206 return errno;
207 }
208
209 p2 = accept(s2, NULL, NULL);
210 if (p2 < 0) {
211 perror("accept s1 failed()\n");
212 return errno;
213 }
214
215 if (verbose) {
216 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
217 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
218 c1, s1, c2, s2);
219 }
220 return 0;
221 }
222
223 struct msg_stats {
224 size_t bytes_sent;
225 size_t bytes_recvd;
226 struct timespec start;
227 struct timespec end;
228 };
229
230 struct sockmap_options {
231 int verbose;
232 bool base;
233 bool sendpage;
234 bool data_test;
235 bool drop_expected;
236 int iov_count;
237 int iov_length;
238 int rate;
239 };
240
msg_loop_sendpage(int fd,int iov_length,int cnt,struct msg_stats * s,struct sockmap_options * opt)241 static int msg_loop_sendpage(int fd, int iov_length, int cnt,
242 struct msg_stats *s,
243 struct sockmap_options *opt)
244 {
245 bool drop = opt->drop_expected;
246 unsigned char k = 0;
247 FILE *file;
248 int i, fp;
249
250 file = fopen(".sendpage_tst.tmp", "w+");
251 if (!file) {
252 perror("create file for sendpage");
253 return 1;
254 }
255 for (i = 0; i < iov_length * cnt; i++, k++)
256 fwrite(&k, sizeof(char), 1, file);
257 fflush(file);
258 fseek(file, 0, SEEK_SET);
259 fclose(file);
260
261 fp = open(".sendpage_tst.tmp", O_RDONLY);
262 if (fp < 0) {
263 perror("reopen file for sendpage");
264 return 1;
265 }
266
267 clock_gettime(CLOCK_MONOTONIC, &s->start);
268 for (i = 0; i < cnt; i++) {
269 int sent = sendfile(fd, fp, NULL, iov_length);
270
271 if (!drop && sent < 0) {
272 perror("send loop error:");
273 close(fp);
274 return sent;
275 } else if (drop && sent >= 0) {
276 printf("sendpage loop error expected: %i\n", sent);
277 close(fp);
278 return -EIO;
279 }
280
281 if (sent > 0)
282 s->bytes_sent += sent;
283 }
284 clock_gettime(CLOCK_MONOTONIC, &s->end);
285 close(fp);
286 return 0;
287 }
288
msg_loop(int fd,int iov_count,int iov_length,int cnt,struct msg_stats * s,bool tx,struct sockmap_options * opt)289 static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
290 struct msg_stats *s, bool tx,
291 struct sockmap_options *opt)
292 {
293 struct msghdr msg = {0};
294 int err, i, flags = MSG_NOSIGNAL;
295 struct iovec *iov;
296 unsigned char k;
297 bool data_test = opt->data_test;
298 bool drop = opt->drop_expected;
299
300 iov = calloc(iov_count, sizeof(struct iovec));
301 if (!iov)
302 return errno;
303
304 k = 0;
305 for (i = 0; i < iov_count; i++) {
306 unsigned char *d = calloc(iov_length, sizeof(char));
307
308 if (!d) {
309 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
310 goto out_errno;
311 }
312 iov[i].iov_base = d;
313 iov[i].iov_len = iov_length;
314
315 if (data_test && tx) {
316 int j;
317
318 for (j = 0; j < iov_length; j++)
319 d[j] = k++;
320 }
321 }
322
323 msg.msg_iov = iov;
324 msg.msg_iovlen = iov_count;
325 k = 0;
326
327 if (tx) {
328 clock_gettime(CLOCK_MONOTONIC, &s->start);
329 for (i = 0; i < cnt; i++) {
330 int sent = sendmsg(fd, &msg, flags);
331
332 if (!drop && sent < 0) {
333 perror("send loop error:");
334 goto out_errno;
335 } else if (drop && sent >= 0) {
336 printf("send loop error expected: %i\n", sent);
337 errno = -EIO;
338 goto out_errno;
339 }
340 if (sent > 0)
341 s->bytes_sent += sent;
342 }
343 clock_gettime(CLOCK_MONOTONIC, &s->end);
344 } else {
345 int slct, recv, max_fd = fd;
346 int fd_flags = O_NONBLOCK;
347 struct timeval timeout;
348 float total_bytes;
349 int bytes_cnt = 0;
350 int chunk_sz;
351 fd_set w;
352
353 if (opt->sendpage)
354 chunk_sz = iov_length * cnt;
355 else
356 chunk_sz = iov_length * iov_count;
357
358 fcntl(fd, fd_flags);
359 total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
360 err = clock_gettime(CLOCK_MONOTONIC, &s->start);
361 if (err < 0)
362 perror("recv start time: ");
363 while (s->bytes_recvd < total_bytes) {
364 if (txmsg_cork) {
365 timeout.tv_sec = 0;
366 timeout.tv_usec = 300000;
367 } else {
368 timeout.tv_sec = 1;
369 timeout.tv_usec = 0;
370 }
371
372 /* FD sets */
373 FD_ZERO(&w);
374 FD_SET(fd, &w);
375
376 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
377 if (slct == -1) {
378 perror("select()");
379 clock_gettime(CLOCK_MONOTONIC, &s->end);
380 goto out_errno;
381 } else if (!slct) {
382 if (opt->verbose)
383 fprintf(stderr, "unexpected timeout\n");
384 errno = -EIO;
385 clock_gettime(CLOCK_MONOTONIC, &s->end);
386 goto out_errno;
387 }
388
389 recv = recvmsg(fd, &msg, flags);
390 if (recv < 0) {
391 if (errno != EWOULDBLOCK) {
392 clock_gettime(CLOCK_MONOTONIC, &s->end);
393 perror("recv failed()\n");
394 goto out_errno;
395 }
396 }
397
398 s->bytes_recvd += recv;
399
400 if (data_test) {
401 int j;
402
403 for (i = 0; i < msg.msg_iovlen; i++) {
404 unsigned char *d = iov[i].iov_base;
405
406 for (j = 0;
407 j < iov[i].iov_len && recv; j++) {
408 if (d[j] != k++) {
409 errno = -EIO;
410 fprintf(stderr,
411 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
412 i, j, d[j], k - 1, d[j+1], k);
413 goto out_errno;
414 }
415 bytes_cnt++;
416 if (bytes_cnt == chunk_sz) {
417 k = 0;
418 bytes_cnt = 0;
419 }
420 recv--;
421 }
422 }
423 }
424 }
425 clock_gettime(CLOCK_MONOTONIC, &s->end);
426 }
427
428 for (i = 0; i < iov_count; i++)
429 free(iov[i].iov_base);
430 free(iov);
431 return 0;
432 out_errno:
433 for (i = 0; i < iov_count; i++)
434 free(iov[i].iov_base);
435 free(iov);
436 return errno;
437 }
438
439 static float giga = 1000000000;
440
sentBps(struct msg_stats s)441 static inline float sentBps(struct msg_stats s)
442 {
443 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
444 }
445
recvdBps(struct msg_stats s)446 static inline float recvdBps(struct msg_stats s)
447 {
448 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
449 }
450
sendmsg_test(struct sockmap_options * opt)451 static int sendmsg_test(struct sockmap_options *opt)
452 {
453 float sent_Bps = 0, recvd_Bps = 0;
454 int rx_fd, txpid, rxpid, err = 0;
455 struct msg_stats s = {0};
456 int iov_count = opt->iov_count;
457 int iov_buf = opt->iov_length;
458 int rx_status, tx_status;
459 int cnt = opt->rate;
460
461 errno = 0;
462
463 if (opt->base)
464 rx_fd = p1;
465 else
466 rx_fd = p2;
467
468 rxpid = fork();
469 if (rxpid == 0) {
470 if (opt->drop_expected)
471 exit(0);
472
473 if (opt->sendpage)
474 iov_count = 1;
475 err = msg_loop(rx_fd, iov_count, iov_buf,
476 cnt, &s, false, opt);
477 if (err && opt->verbose)
478 fprintf(stderr,
479 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
480 iov_count, iov_buf, cnt, err);
481 shutdown(p2, SHUT_RDWR);
482 shutdown(p1, SHUT_RDWR);
483 if (s.end.tv_sec - s.start.tv_sec) {
484 sent_Bps = sentBps(s);
485 recvd_Bps = recvdBps(s);
486 }
487 if (opt->verbose)
488 fprintf(stdout,
489 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s\n",
490 s.bytes_sent, sent_Bps, sent_Bps/giga,
491 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
492 if (err && txmsg_cork)
493 err = 0;
494 exit(err ? 1 : 0);
495 } else if (rxpid == -1) {
496 perror("msg_loop_rx: ");
497 return errno;
498 }
499
500 txpid = fork();
501 if (txpid == 0) {
502 if (opt->sendpage)
503 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
504 else
505 err = msg_loop(c1, iov_count, iov_buf,
506 cnt, &s, true, opt);
507
508 if (err)
509 fprintf(stderr,
510 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
511 iov_count, iov_buf, cnt, err);
512 shutdown(c1, SHUT_RDWR);
513 if (s.end.tv_sec - s.start.tv_sec) {
514 sent_Bps = sentBps(s);
515 recvd_Bps = recvdBps(s);
516 }
517 if (opt->verbose)
518 fprintf(stdout,
519 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
520 s.bytes_sent, sent_Bps, sent_Bps/giga,
521 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
522 exit(err ? 1 : 0);
523 } else if (txpid == -1) {
524 perror("msg_loop_tx: ");
525 return errno;
526 }
527
528 assert(waitpid(rxpid, &rx_status, 0) == rxpid);
529 assert(waitpid(txpid, &tx_status, 0) == txpid);
530 if (WIFEXITED(rx_status)) {
531 err = WEXITSTATUS(rx_status);
532 if (err) {
533 fprintf(stderr, "rx thread exited with err %d. ", err);
534 goto out;
535 }
536 }
537 if (WIFEXITED(tx_status)) {
538 err = WEXITSTATUS(tx_status);
539 if (err)
540 fprintf(stderr, "tx thread exited with err %d. ", err);
541 }
542 out:
543 return err;
544 }
545
forever_ping_pong(int rate,struct sockmap_options * opt)546 static int forever_ping_pong(int rate, struct sockmap_options *opt)
547 {
548 struct timeval timeout;
549 char buf[1024] = {0};
550 int sc;
551
552 timeout.tv_sec = 10;
553 timeout.tv_usec = 0;
554
555 /* Ping/Pong data from client to server */
556 sc = send(c1, buf, sizeof(buf), 0);
557 if (sc < 0) {
558 perror("send failed()\n");
559 return sc;
560 }
561
562 do {
563 int s, rc, i, max_fd = p2;
564 fd_set w;
565
566 /* FD sets */
567 FD_ZERO(&w);
568 FD_SET(c1, &w);
569 FD_SET(c2, &w);
570 FD_SET(p1, &w);
571 FD_SET(p2, &w);
572
573 s = select(max_fd + 1, &w, NULL, NULL, &timeout);
574 if (s == -1) {
575 perror("select()");
576 break;
577 } else if (!s) {
578 fprintf(stderr, "unexpected timeout\n");
579 break;
580 }
581
582 for (i = 0; i <= max_fd && s > 0; ++i) {
583 if (!FD_ISSET(i, &w))
584 continue;
585
586 s--;
587
588 rc = recv(i, buf, sizeof(buf), 0);
589 if (rc < 0) {
590 if (errno != EWOULDBLOCK) {
591 perror("recv failed()\n");
592 return rc;
593 }
594 }
595
596 if (rc == 0) {
597 close(i);
598 break;
599 }
600
601 sc = send(i, buf, rc, 0);
602 if (sc < 0) {
603 perror("send failed()\n");
604 return sc;
605 }
606 }
607
608 if (rate)
609 sleep(rate);
610
611 if (opt->verbose) {
612 printf(".");
613 fflush(stdout);
614
615 }
616 } while (running);
617
618 return 0;
619 }
620
621 enum {
622 PING_PONG,
623 SENDMSG,
624 BASE,
625 BASE_SENDPAGE,
626 SENDPAGE,
627 };
628
run_options(struct sockmap_options * options,int cg_fd,int test)629 static int run_options(struct sockmap_options *options, int cg_fd, int test)
630 {
631 int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
632
633 /* If base test skip BPF setup */
634 if (test == BASE || test == BASE_SENDPAGE)
635 goto run;
636
637 /* Attach programs to sockmap */
638 err = bpf_prog_attach(prog_fd[0], map_fd[0],
639 BPF_SK_SKB_STREAM_PARSER, 0);
640 if (err) {
641 fprintf(stderr,
642 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
643 prog_fd[0], map_fd[0], err, strerror(errno));
644 return err;
645 }
646
647 err = bpf_prog_attach(prog_fd[1], map_fd[0],
648 BPF_SK_SKB_STREAM_VERDICT, 0);
649 if (err) {
650 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
651 err, strerror(errno));
652 return err;
653 }
654
655 /* Attach to cgroups */
656 err = bpf_prog_attach(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
657 if (err) {
658 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
659 err, strerror(errno));
660 return err;
661 }
662
663 run:
664 err = sockmap_init_sockets(options->verbose);
665 if (err) {
666 fprintf(stderr, "ERROR: test socket failed: %d\n", err);
667 goto out;
668 }
669
670 /* Attach txmsg program to sockmap */
671 if (txmsg_pass)
672 tx_prog_fd = prog_fd[3];
673 else if (txmsg_noisy)
674 tx_prog_fd = prog_fd[4];
675 else if (txmsg_redir)
676 tx_prog_fd = prog_fd[5];
677 else if (txmsg_redir_noisy)
678 tx_prog_fd = prog_fd[6];
679 else if (txmsg_drop)
680 tx_prog_fd = prog_fd[9];
681 /* apply and cork must be last */
682 else if (txmsg_apply)
683 tx_prog_fd = prog_fd[7];
684 else if (txmsg_cork)
685 tx_prog_fd = prog_fd[8];
686 else
687 tx_prog_fd = 0;
688
689 if (tx_prog_fd) {
690 int redir_fd, i = 0;
691
692 err = bpf_prog_attach(tx_prog_fd,
693 map_fd[1], BPF_SK_MSG_VERDICT, 0);
694 if (err) {
695 fprintf(stderr,
696 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
697 err, strerror(errno));
698 goto out;
699 }
700
701 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
702 if (err) {
703 fprintf(stderr,
704 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
705 err, strerror(errno));
706 goto out;
707 }
708
709 if (txmsg_redir || txmsg_redir_noisy)
710 redir_fd = c2;
711 else
712 redir_fd = c1;
713
714 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
715 if (err) {
716 fprintf(stderr,
717 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
718 err, strerror(errno));
719 goto out;
720 }
721
722 if (txmsg_apply) {
723 err = bpf_map_update_elem(map_fd[3],
724 &i, &txmsg_apply, BPF_ANY);
725 if (err) {
726 fprintf(stderr,
727 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
728 err, strerror(errno));
729 goto out;
730 }
731 }
732
733 if (txmsg_cork) {
734 err = bpf_map_update_elem(map_fd[4],
735 &i, &txmsg_cork, BPF_ANY);
736 if (err) {
737 fprintf(stderr,
738 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
739 err, strerror(errno));
740 goto out;
741 }
742 }
743
744 if (txmsg_start) {
745 err = bpf_map_update_elem(map_fd[5],
746 &i, &txmsg_start, BPF_ANY);
747 if (err) {
748 fprintf(stderr,
749 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
750 err, strerror(errno));
751 goto out;
752 }
753 }
754
755 if (txmsg_end) {
756 i = 1;
757 err = bpf_map_update_elem(map_fd[5],
758 &i, &txmsg_end, BPF_ANY);
759 if (err) {
760 fprintf(stderr,
761 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
762 err, strerror(errno));
763 goto out;
764 }
765 }
766
767 if (txmsg_ingress) {
768 int in = BPF_F_INGRESS;
769
770 i = 0;
771 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
772 if (err) {
773 fprintf(stderr,
774 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
775 err, strerror(errno));
776 }
777 i = 1;
778 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
779 if (err) {
780 fprintf(stderr,
781 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
782 err, strerror(errno));
783 }
784 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
785 if (err) {
786 fprintf(stderr,
787 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
788 err, strerror(errno));
789 }
790
791 i = 2;
792 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
793 if (err) {
794 fprintf(stderr,
795 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
796 err, strerror(errno));
797 }
798 }
799
800 if (txmsg_skb) {
801 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
802 p2 : p1;
803 int ingress = BPF_F_INGRESS;
804
805 i = 0;
806 err = bpf_map_update_elem(map_fd[7],
807 &i, &ingress, BPF_ANY);
808 if (err) {
809 fprintf(stderr,
810 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
811 err, strerror(errno));
812 }
813
814 i = 3;
815 err = bpf_map_update_elem(map_fd[0],
816 &i, &skb_fd, BPF_ANY);
817 if (err) {
818 fprintf(stderr,
819 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
820 err, strerror(errno));
821 }
822 }
823 }
824
825 if (txmsg_drop)
826 options->drop_expected = true;
827
828 if (test == PING_PONG)
829 err = forever_ping_pong(options->rate, options);
830 else if (test == SENDMSG) {
831 options->base = false;
832 options->sendpage = false;
833 err = sendmsg_test(options);
834 } else if (test == SENDPAGE) {
835 options->base = false;
836 options->sendpage = true;
837 err = sendmsg_test(options);
838 } else if (test == BASE) {
839 options->base = true;
840 options->sendpage = false;
841 err = sendmsg_test(options);
842 } else if (test == BASE_SENDPAGE) {
843 options->base = true;
844 options->sendpage = true;
845 err = sendmsg_test(options);
846 } else
847 fprintf(stderr, "unknown test\n");
848 out:
849 /* Detatch and zero all the maps */
850 bpf_prog_detach2(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS);
851 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
852 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
853 if (tx_prog_fd >= 0)
854 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
855
856 for (i = 0; i < 8; i++) {
857 key = next_key = 0;
858 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
859 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
860 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
861 key = next_key;
862 }
863 }
864
865 close(s1);
866 close(s2);
867 close(p1);
868 close(p2);
869 close(c1);
870 close(c2);
871 return err;
872 }
873
test_to_str(int test)874 static char *test_to_str(int test)
875 {
876 switch (test) {
877 case SENDMSG:
878 return "sendmsg";
879 case SENDPAGE:
880 return "sendpage";
881 }
882 return "unknown";
883 }
884
885 #define OPTSTRING 60
test_options(char * options)886 static void test_options(char *options)
887 {
888 char tstr[OPTSTRING];
889
890 memset(options, 0, OPTSTRING);
891
892 if (txmsg_pass)
893 strncat(options, "pass,", OPTSTRING);
894 if (txmsg_noisy)
895 strncat(options, "pass_noisy,", OPTSTRING);
896 if (txmsg_redir)
897 strncat(options, "redir,", OPTSTRING);
898 if (txmsg_redir_noisy)
899 strncat(options, "redir_noisy,", OPTSTRING);
900 if (txmsg_drop)
901 strncat(options, "drop,", OPTSTRING);
902 if (txmsg_apply) {
903 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
904 strncat(options, tstr, OPTSTRING);
905 }
906 if (txmsg_cork) {
907 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
908 strncat(options, tstr, OPTSTRING);
909 }
910 if (txmsg_start) {
911 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
912 strncat(options, tstr, OPTSTRING);
913 }
914 if (txmsg_end) {
915 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
916 strncat(options, tstr, OPTSTRING);
917 }
918 if (txmsg_ingress)
919 strncat(options, "ingress,", OPTSTRING);
920 if (txmsg_skb)
921 strncat(options, "skb,", OPTSTRING);
922 }
923
__test_exec(int cgrp,int test,struct sockmap_options * opt)924 static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
925 {
926 char *options = calloc(OPTSTRING, sizeof(char));
927 int err;
928
929 if (test == SENDPAGE)
930 opt->sendpage = true;
931 else
932 opt->sendpage = false;
933
934 if (txmsg_drop)
935 opt->drop_expected = true;
936 else
937 opt->drop_expected = false;
938
939 test_options(options);
940
941 fprintf(stdout,
942 "[TEST %i]: (%i, %i, %i, %s, %s): ",
943 test_cnt, opt->rate, opt->iov_count, opt->iov_length,
944 test_to_str(test), options);
945 fflush(stdout);
946 err = run_options(opt, cgrp, test);
947 fprintf(stdout, "%s\n", !err ? "PASS" : "FAILED");
948 test_cnt++;
949 !err ? passed++ : failed++;
950 free(options);
951 return err;
952 }
953
test_exec(int cgrp,struct sockmap_options * opt)954 static int test_exec(int cgrp, struct sockmap_options *opt)
955 {
956 int err = __test_exec(cgrp, SENDMSG, opt);
957
958 if (err)
959 goto out;
960
961 err = __test_exec(cgrp, SENDPAGE, opt);
962 out:
963 return err;
964 }
965
test_loop(int cgrp)966 static int test_loop(int cgrp)
967 {
968 struct sockmap_options opt;
969
970 int err, i, l, r;
971
972 opt.verbose = 0;
973 opt.base = false;
974 opt.sendpage = false;
975 opt.data_test = false;
976 opt.drop_expected = false;
977 opt.iov_count = 0;
978 opt.iov_length = 0;
979 opt.rate = 0;
980
981 r = 1;
982 for (i = 1; i < 100; i += 33) {
983 for (l = 1; l < 100; l += 33) {
984 opt.rate = r;
985 opt.iov_count = i;
986 opt.iov_length = l;
987 err = test_exec(cgrp, &opt);
988 if (err)
989 goto out;
990 }
991 }
992 sched_yield();
993 out:
994 return err;
995 }
996
test_txmsg(int cgrp)997 static int test_txmsg(int cgrp)
998 {
999 int err;
1000
1001 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0;
1002 txmsg_apply = txmsg_cork = 0;
1003 txmsg_ingress = txmsg_skb = 0;
1004
1005 txmsg_pass = 1;
1006 err = test_loop(cgrp);
1007 txmsg_pass = 0;
1008 if (err)
1009 goto out;
1010
1011 txmsg_redir = 1;
1012 err = test_loop(cgrp);
1013 txmsg_redir = 0;
1014 if (err)
1015 goto out;
1016
1017 txmsg_drop = 1;
1018 err = test_loop(cgrp);
1019 txmsg_drop = 0;
1020 if (err)
1021 goto out;
1022
1023 txmsg_redir = 1;
1024 txmsg_ingress = 1;
1025 err = test_loop(cgrp);
1026 txmsg_redir = 0;
1027 txmsg_ingress = 0;
1028 if (err)
1029 goto out;
1030 out:
1031 txmsg_pass = 0;
1032 txmsg_redir = 0;
1033 txmsg_drop = 0;
1034 return err;
1035 }
1036
test_send(struct sockmap_options * opt,int cgrp)1037 static int test_send(struct sockmap_options *opt, int cgrp)
1038 {
1039 int err;
1040
1041 opt->iov_length = 1;
1042 opt->iov_count = 1;
1043 opt->rate = 1;
1044 err = test_exec(cgrp, opt);
1045 if (err)
1046 goto out;
1047
1048 opt->iov_length = 1;
1049 opt->iov_count = 1024;
1050 opt->rate = 1;
1051 err = test_exec(cgrp, opt);
1052 if (err)
1053 goto out;
1054
1055 opt->iov_length = 1024;
1056 opt->iov_count = 1;
1057 opt->rate = 1;
1058 err = test_exec(cgrp, opt);
1059 if (err)
1060 goto out;
1061
1062 opt->iov_length = 1;
1063 opt->iov_count = 1;
1064 opt->rate = 512;
1065 err = test_exec(cgrp, opt);
1066 if (err)
1067 goto out;
1068
1069 opt->iov_length = 256;
1070 opt->iov_count = 1024;
1071 opt->rate = 2;
1072 err = test_exec(cgrp, opt);
1073 if (err)
1074 goto out;
1075
1076 opt->rate = 100;
1077 opt->iov_count = 1;
1078 opt->iov_length = 5;
1079 err = test_exec(cgrp, opt);
1080 if (err)
1081 goto out;
1082 out:
1083 sched_yield();
1084 return err;
1085 }
1086
test_mixed(int cgrp)1087 static int test_mixed(int cgrp)
1088 {
1089 struct sockmap_options opt = {0};
1090 int err;
1091
1092 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0;
1093 txmsg_apply = txmsg_cork = 0;
1094 txmsg_start = txmsg_end = 0;
1095 /* Test small and large iov_count values with pass/redir/apply/cork */
1096 txmsg_pass = 1;
1097 txmsg_redir = 0;
1098 txmsg_apply = 1;
1099 txmsg_cork = 0;
1100 err = test_send(&opt, cgrp);
1101 if (err)
1102 goto out;
1103
1104 txmsg_pass = 1;
1105 txmsg_redir = 0;
1106 txmsg_apply = 0;
1107 txmsg_cork = 1;
1108 err = test_send(&opt, cgrp);
1109 if (err)
1110 goto out;
1111
1112 txmsg_pass = 1;
1113 txmsg_redir = 0;
1114 txmsg_apply = 1;
1115 txmsg_cork = 1;
1116 err = test_send(&opt, cgrp);
1117 if (err)
1118 goto out;
1119
1120 txmsg_pass = 1;
1121 txmsg_redir = 0;
1122 txmsg_apply = 1024;
1123 txmsg_cork = 0;
1124 err = test_send(&opt, cgrp);
1125 if (err)
1126 goto out;
1127
1128 txmsg_pass = 1;
1129 txmsg_redir = 0;
1130 txmsg_apply = 0;
1131 txmsg_cork = 1024;
1132 err = test_send(&opt, cgrp);
1133 if (err)
1134 goto out;
1135
1136 txmsg_pass = 1;
1137 txmsg_redir = 0;
1138 txmsg_apply = 1024;
1139 txmsg_cork = 1024;
1140 err = test_send(&opt, cgrp);
1141 if (err)
1142 goto out;
1143
1144 txmsg_pass = 1;
1145 txmsg_redir = 0;
1146 txmsg_cork = 4096;
1147 txmsg_apply = 4096;
1148 err = test_send(&opt, cgrp);
1149 if (err)
1150 goto out;
1151
1152 txmsg_pass = 0;
1153 txmsg_redir = 1;
1154 txmsg_apply = 1;
1155 txmsg_cork = 0;
1156 err = test_send(&opt, cgrp);
1157 if (err)
1158 goto out;
1159
1160 txmsg_pass = 0;
1161 txmsg_redir = 1;
1162 txmsg_apply = 0;
1163 txmsg_cork = 1;
1164 err = test_send(&opt, cgrp);
1165 if (err)
1166 goto out;
1167
1168 txmsg_pass = 0;
1169 txmsg_redir = 1;
1170 txmsg_apply = 1024;
1171 txmsg_cork = 0;
1172 err = test_send(&opt, cgrp);
1173 if (err)
1174 goto out;
1175
1176 txmsg_pass = 0;
1177 txmsg_redir = 1;
1178 txmsg_apply = 0;
1179 txmsg_cork = 1024;
1180 err = test_send(&opt, cgrp);
1181 if (err)
1182 goto out;
1183
1184 txmsg_pass = 0;
1185 txmsg_redir = 1;
1186 txmsg_apply = 1024;
1187 txmsg_cork = 1024;
1188 err = test_send(&opt, cgrp);
1189 if (err)
1190 goto out;
1191
1192 txmsg_pass = 0;
1193 txmsg_redir = 1;
1194 txmsg_cork = 4096;
1195 txmsg_apply = 4096;
1196 err = test_send(&opt, cgrp);
1197 if (err)
1198 goto out;
1199 out:
1200 return err;
1201 }
1202
test_start_end(int cgrp)1203 static int test_start_end(int cgrp)
1204 {
1205 struct sockmap_options opt = {0};
1206 int err, i;
1207
1208 /* Test basic start/end with lots of iov_count and iov_lengths */
1209 txmsg_start = 1;
1210 txmsg_end = 2;
1211 err = test_txmsg(cgrp);
1212 if (err)
1213 goto out;
1214
1215 /* Test start/end with cork */
1216 opt.rate = 16;
1217 opt.iov_count = 1;
1218 opt.iov_length = 100;
1219 txmsg_cork = 1600;
1220
1221 for (i = 99; i <= 1600; i += 500) {
1222 txmsg_start = 0;
1223 txmsg_end = i;
1224 err = test_exec(cgrp, &opt);
1225 if (err)
1226 goto out;
1227 }
1228
1229 /* Test start/end with cork but pull data in middle */
1230 for (i = 199; i <= 1600; i += 500) {
1231 txmsg_start = 100;
1232 txmsg_end = i;
1233 err = test_exec(cgrp, &opt);
1234 if (err)
1235 goto out;
1236 }
1237
1238 /* Test start/end with cork pulling last sg entry */
1239 txmsg_start = 1500;
1240 txmsg_end = 1600;
1241 err = test_exec(cgrp, &opt);
1242 if (err)
1243 goto out;
1244
1245 /* Test start/end pull of single byte in last page */
1246 txmsg_start = 1111;
1247 txmsg_end = 1112;
1248 err = test_exec(cgrp, &opt);
1249 if (err)
1250 goto out;
1251
1252 /* Test start/end with end < start */
1253 txmsg_start = 1111;
1254 txmsg_end = 0;
1255 err = test_exec(cgrp, &opt);
1256 if (err)
1257 goto out;
1258
1259 /* Test start/end with end > data */
1260 txmsg_start = 0;
1261 txmsg_end = 1601;
1262 err = test_exec(cgrp, &opt);
1263 if (err)
1264 goto out;
1265
1266 /* Test start/end with start > data */
1267 txmsg_start = 1601;
1268 txmsg_end = 1600;
1269 err = test_exec(cgrp, &opt);
1270
1271 out:
1272 txmsg_start = 0;
1273 txmsg_end = 0;
1274 sched_yield();
1275 return err;
1276 }
1277
1278 char *map_names[] = {
1279 "sock_map",
1280 "sock_map_txmsg",
1281 "sock_map_redir",
1282 "sock_apply_bytes",
1283 "sock_cork_bytes",
1284 "sock_pull_bytes",
1285 "sock_redir_flags",
1286 "sock_skb_opts",
1287 };
1288
1289 int prog_attach_type[] = {
1290 BPF_SK_SKB_STREAM_PARSER,
1291 BPF_SK_SKB_STREAM_VERDICT,
1292 BPF_CGROUP_SOCK_OPS,
1293 BPF_SK_MSG_VERDICT,
1294 BPF_SK_MSG_VERDICT,
1295 BPF_SK_MSG_VERDICT,
1296 BPF_SK_MSG_VERDICT,
1297 BPF_SK_MSG_VERDICT,
1298 BPF_SK_MSG_VERDICT,
1299 BPF_SK_MSG_VERDICT,
1300 };
1301
1302 int prog_type[] = {
1303 BPF_PROG_TYPE_SK_SKB,
1304 BPF_PROG_TYPE_SK_SKB,
1305 BPF_PROG_TYPE_SOCK_OPS,
1306 BPF_PROG_TYPE_SK_MSG,
1307 BPF_PROG_TYPE_SK_MSG,
1308 BPF_PROG_TYPE_SK_MSG,
1309 BPF_PROG_TYPE_SK_MSG,
1310 BPF_PROG_TYPE_SK_MSG,
1311 BPF_PROG_TYPE_SK_MSG,
1312 BPF_PROG_TYPE_SK_MSG,
1313 };
1314
populate_progs(char * bpf_file)1315 static int populate_progs(char *bpf_file)
1316 {
1317 struct bpf_program *prog;
1318 struct bpf_object *obj;
1319 int i = 0;
1320 long err;
1321
1322 obj = bpf_object__open(bpf_file);
1323 err = libbpf_get_error(obj);
1324 if (err) {
1325 char err_buf[256];
1326
1327 libbpf_strerror(err, err_buf, sizeof(err_buf));
1328 printf("Unable to load eBPF objects in file '%s' : %s\n",
1329 bpf_file, err_buf);
1330 return -1;
1331 }
1332
1333 bpf_object__for_each_program(prog, obj) {
1334 bpf_program__set_type(prog, prog_type[i]);
1335 bpf_program__set_expected_attach_type(prog,
1336 prog_attach_type[i]);
1337 i++;
1338 }
1339
1340 i = bpf_object__load(obj);
1341 i = 0;
1342 bpf_object__for_each_program(prog, obj) {
1343 prog_fd[i] = bpf_program__fd(prog);
1344 i++;
1345 }
1346
1347 for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) {
1348 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1349 map_fd[i] = bpf_map__fd(maps[i]);
1350 if (map_fd[i] < 0) {
1351 fprintf(stderr, "load_bpf_file: (%i) %s\n",
1352 map_fd[i], strerror(errno));
1353 return -1;
1354 }
1355 }
1356
1357 return 0;
1358 }
1359
__test_suite(char * bpf_file)1360 static int __test_suite(char *bpf_file)
1361 {
1362 int cg_fd, err;
1363
1364 err = populate_progs(bpf_file);
1365 if (err < 0) {
1366 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1367 return err;
1368 }
1369
1370 if (setup_cgroup_environment()) {
1371 fprintf(stderr, "ERROR: cgroup env failed\n");
1372 return -EINVAL;
1373 }
1374
1375 cg_fd = create_and_get_cgroup(CG_PATH);
1376 if (cg_fd < 0) {
1377 fprintf(stderr,
1378 "ERROR: (%i) open cg path failed: %s\n",
1379 cg_fd, optarg);
1380 return cg_fd;
1381 }
1382
1383 if (join_cgroup(CG_PATH)) {
1384 fprintf(stderr, "ERROR: failed to join cgroup\n");
1385 return -EINVAL;
1386 }
1387
1388 /* Tests basic commands and APIs with range of iov values */
1389 txmsg_start = txmsg_end = 0;
1390 err = test_txmsg(cg_fd);
1391 if (err)
1392 goto out;
1393
1394 /* Tests interesting combinations of APIs used together */
1395 err = test_mixed(cg_fd);
1396 if (err)
1397 goto out;
1398
1399 /* Tests pull_data API using start/end API */
1400 err = test_start_end(cg_fd);
1401 if (err)
1402 goto out;
1403
1404 out:
1405 printf("Summary: %i PASSED %i FAILED\n", passed, failed);
1406 cleanup_cgroup_environment();
1407 close(cg_fd);
1408 return err;
1409 }
1410
test_suite(void)1411 static int test_suite(void)
1412 {
1413 int err;
1414
1415 err = __test_suite(BPF_SOCKMAP_FILENAME);
1416 if (err)
1417 goto out;
1418 err = __test_suite(BPF_SOCKHASH_FILENAME);
1419 out:
1420 return err;
1421 }
1422
main(int argc,char ** argv)1423 int main(int argc, char **argv)
1424 {
1425 int iov_count = 1, length = 1024, rate = 1;
1426 struct sockmap_options options = {0};
1427 int opt, longindex, err, cg_fd = 0;
1428 char *bpf_file = BPF_SOCKMAP_FILENAME;
1429 int test = PING_PONG;
1430
1431 if (argc < 2)
1432 return test_suite();
1433
1434 while ((opt = getopt_long(argc, argv, ":dhvc:r:i:l:t:",
1435 long_options, &longindex)) != -1) {
1436 switch (opt) {
1437 case 's':
1438 txmsg_start = atoi(optarg);
1439 break;
1440 case 'e':
1441 txmsg_end = atoi(optarg);
1442 break;
1443 case 'a':
1444 txmsg_apply = atoi(optarg);
1445 break;
1446 case 'k':
1447 txmsg_cork = atoi(optarg);
1448 break;
1449 case 'c':
1450 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
1451 if (cg_fd < 0) {
1452 fprintf(stderr,
1453 "ERROR: (%i) open cg path failed: %s\n",
1454 cg_fd, optarg);
1455 return cg_fd;
1456 }
1457 break;
1458 case 'r':
1459 rate = atoi(optarg);
1460 break;
1461 case 'v':
1462 options.verbose = 1;
1463 break;
1464 case 'i':
1465 iov_count = atoi(optarg);
1466 break;
1467 case 'l':
1468 length = atoi(optarg);
1469 break;
1470 case 'd':
1471 options.data_test = true;
1472 break;
1473 case 't':
1474 if (strcmp(optarg, "ping") == 0) {
1475 test = PING_PONG;
1476 } else if (strcmp(optarg, "sendmsg") == 0) {
1477 test = SENDMSG;
1478 } else if (strcmp(optarg, "base") == 0) {
1479 test = BASE;
1480 } else if (strcmp(optarg, "base_sendpage") == 0) {
1481 test = BASE_SENDPAGE;
1482 } else if (strcmp(optarg, "sendpage") == 0) {
1483 test = SENDPAGE;
1484 } else {
1485 usage(argv);
1486 return -1;
1487 }
1488 break;
1489 case 0:
1490 break;
1491 case 'h':
1492 default:
1493 usage(argv);
1494 return -1;
1495 }
1496 }
1497
1498 if (!cg_fd) {
1499 fprintf(stderr, "%s requires cgroup option: --cgroup <path>\n",
1500 argv[0]);
1501 return -1;
1502 }
1503
1504 err = populate_progs(bpf_file);
1505 if (err) {
1506 fprintf(stderr, "populate program: (%s) %s\n",
1507 bpf_file, strerror(errno));
1508 return 1;
1509 }
1510 running = 1;
1511
1512 /* catch SIGINT */
1513 signal(SIGINT, running_handler);
1514
1515 options.iov_count = iov_count;
1516 options.iov_length = length;
1517 options.rate = rate;
1518
1519 err = run_options(&options, cg_fd, test);
1520 close(cg_fd);
1521 return err;
1522 }
1523
running_handler(int a)1524 void running_handler(int a)
1525 {
1526 running = 0;
1527 }
1528