1 /*
2 * Copyright 2018 Google Inc.
3 * Author: Eric Dumazet (edumazet@google.com)
4 *
5 * Reference program demonstrating tcp mmap() usage,
6 * and SO_RCVLOWAT hints for receiver.
7 *
8 * Note : NIC with header split is needed to use mmap() on TCP :
9 * Each incoming frame must be a multiple of PAGE_SIZE bytes of TCP payload.
10 *
11 * How to use on loopback interface :
12 *
13 * ifconfig lo mtu 61512 # 15*4096 + 40 (ipv6 header) + 32 (TCP with TS option header)
14 * tcp_mmap -s -z &
15 * tcp_mmap -H ::1 -z
16 *
17 * Or leave default lo mtu, but use -M option to set TCP_MAXSEG option to (4096 + 12)
18 * (4096 : page size on x86, 12: TCP TS option length)
19 * tcp_mmap -s -z -M $((4096+12)) &
20 * tcp_mmap -H ::1 -z -M $((4096+12))
21 *
22 * Note: -z option on sender uses MSG_ZEROCOPY, which forces a copy when packets go through loopback interface.
23 * We might use sendfile() instead, but really this test program is about mmap(), for receivers ;)
24 *
25 * $ ./tcp_mmap -s & # Without mmap()
26 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done
27 * received 32768 MB (0 % mmap'ed) in 14.1157 s, 19.4732 Gbit
28 * cpu usage user:0.057 sys:7.815, 240.234 usec per MB, 65531 c-switches
29 * received 32768 MB (0 % mmap'ed) in 14.6833 s, 18.7204 Gbit
30 * cpu usage user:0.043 sys:8.103, 248.596 usec per MB, 65524 c-switches
31 * received 32768 MB (0 % mmap'ed) in 11.143 s, 24.6682 Gbit
32 * cpu usage user:0.044 sys:6.576, 202.026 usec per MB, 65519 c-switches
33 * received 32768 MB (0 % mmap'ed) in 14.9056 s, 18.4413 Gbit
34 * cpu usage user:0.036 sys:8.193, 251.129 usec per MB, 65530 c-switches
35 * $ kill %1 # kill tcp_mmap server
36 *
37 * $ ./tcp_mmap -s -z & # With mmap()
38 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done
39 * received 32768 MB (99.9939 % mmap'ed) in 6.73792 s, 40.7956 Gbit
40 * cpu usage user:0.045 sys:2.827, 87.6465 usec per MB, 65532 c-switches
41 * received 32768 MB (99.9939 % mmap'ed) in 7.26732 s, 37.8238 Gbit
42 * cpu usage user:0.037 sys:3.087, 95.3369 usec per MB, 65532 c-switches
43 * received 32768 MB (99.9939 % mmap'ed) in 7.61661 s, 36.0893 Gbit
44 * cpu usage user:0.046 sys:3.559, 110.016 usec per MB, 65529 c-switches
45 * received 32768 MB (99.9939 % mmap'ed) in 7.43764 s, 36.9577 Gbit
46 * cpu usage user:0.035 sys:3.467, 106.873 usec per MB, 65530 c-switches
47 *
48 * License (GPLv2):
49 *
50 * This program is free software; you can redistribute it and/or modify it
51 * under the terms and conditions of the GNU General Public License,
52 * version 2, as published by the Free Software Foundation.
53 *
54 * This program is distributed in the hope it will be useful, but WITHOUT
55 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
56 * FITNESS FOR A PARTICULAR PURPOSE. * See the GNU General Public License for
57 * more details.
58 *
59 * You should have received a copy of the GNU General Public License along with
60 * this program; if not, write to the Free Software Foundation, Inc.,
61 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
62 */
63 #define _GNU_SOURCE
64 #include <pthread.h>
65 #include <sys/types.h>
66 #include <fcntl.h>
67 #include <error.h>
68 #include <sys/socket.h>
69 #include <sys/mman.h>
70 #include <sys/resource.h>
71 #include <unistd.h>
72 #include <string.h>
73 #include <stdlib.h>
74 #include <stdio.h>
75 #include <errno.h>
76 #include <time.h>
77 #include <sys/time.h>
78 #include <netinet/in.h>
79 #include <arpa/inet.h>
80 #include <poll.h>
81 #include <linux/tcp.h>
82 #include <assert.h>
83
84 #ifndef MSG_ZEROCOPY
85 #define MSG_ZEROCOPY 0x4000000
86 #endif
87
88 #define FILE_SZ (1UL << 35)
89 static int cfg_family = AF_INET6;
90 static socklen_t cfg_alen = sizeof(struct sockaddr_in6);
91 static int cfg_port = 8787;
92
93 static int rcvbuf; /* Default: autotuning. Can be set with -r <integer> option */
94 static int sndbuf; /* Default: autotuning. Can be set with -w <integer> option */
95 static int zflg; /* zero copy option. (MSG_ZEROCOPY for sender, mmap() for receiver */
96 static int xflg; /* hash received data (simple xor) (-h option) */
97 static int keepflag; /* -k option: receiver shall keep all received file in memory (no munmap() calls) */
98
99 static int chunk_size = 512*1024;
100
101 unsigned long htotal;
102
prefetch(const void * x)103 static inline void prefetch(const void *x)
104 {
105 #if defined(__x86_64__)
106 asm volatile("prefetcht0 %P0" : : "m" (*(const char *)x));
107 #endif
108 }
109
hash_zone(void * zone,unsigned int length)110 void hash_zone(void *zone, unsigned int length)
111 {
112 unsigned long temp = htotal;
113
114 while (length >= 8*sizeof(long)) {
115 prefetch(zone + 384);
116 temp ^= *(unsigned long *)zone;
117 temp ^= *(unsigned long *)(zone + sizeof(long));
118 temp ^= *(unsigned long *)(zone + 2*sizeof(long));
119 temp ^= *(unsigned long *)(zone + 3*sizeof(long));
120 temp ^= *(unsigned long *)(zone + 4*sizeof(long));
121 temp ^= *(unsigned long *)(zone + 5*sizeof(long));
122 temp ^= *(unsigned long *)(zone + 6*sizeof(long));
123 temp ^= *(unsigned long *)(zone + 7*sizeof(long));
124 zone += 8*sizeof(long);
125 length -= 8*sizeof(long);
126 }
127 while (length >= 1) {
128 temp ^= *(unsigned char *)zone;
129 zone += 1;
130 length--;
131 }
132 htotal = temp;
133 }
134
child_thread(void * arg)135 void *child_thread(void *arg)
136 {
137 unsigned long total_mmap = 0, total = 0;
138 struct tcp_zerocopy_receive zc;
139 unsigned long delta_usec;
140 int flags = MAP_SHARED;
141 struct timeval t0, t1;
142 char *buffer = NULL;
143 void *addr = NULL;
144 double throughput;
145 struct rusage ru;
146 int lu, fd;
147
148 fd = (int)(unsigned long)arg;
149
150 gettimeofday(&t0, NULL);
151
152 fcntl(fd, F_SETFL, O_NDELAY);
153 buffer = malloc(chunk_size);
154 if (!buffer) {
155 perror("malloc");
156 goto error;
157 }
158 if (zflg) {
159 addr = mmap(NULL, chunk_size, PROT_READ, flags, fd, 0);
160 if (addr == (void *)-1)
161 zflg = 0;
162 }
163 while (1) {
164 struct pollfd pfd = { .fd = fd, .events = POLLIN, };
165 int sub;
166
167 poll(&pfd, 1, 10000);
168 if (zflg) {
169 socklen_t zc_len = sizeof(zc);
170 int res;
171
172 zc.address = (__u64)addr;
173 zc.length = chunk_size;
174 zc.recv_skip_hint = 0;
175 res = getsockopt(fd, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE,
176 &zc, &zc_len);
177 if (res == -1)
178 break;
179
180 if (zc.length) {
181 assert(zc.length <= chunk_size);
182 total_mmap += zc.length;
183 if (xflg)
184 hash_zone(addr, zc.length);
185 total += zc.length;
186 }
187 if (zc.recv_skip_hint) {
188 assert(zc.recv_skip_hint <= chunk_size);
189 lu = read(fd, buffer, zc.recv_skip_hint);
190 if (lu > 0) {
191 if (xflg)
192 hash_zone(buffer, lu);
193 total += lu;
194 }
195 }
196 continue;
197 }
198 sub = 0;
199 while (sub < chunk_size) {
200 lu = read(fd, buffer + sub, chunk_size - sub);
201 if (lu == 0)
202 goto end;
203 if (lu < 0)
204 break;
205 if (xflg)
206 hash_zone(buffer + sub, lu);
207 total += lu;
208 sub += lu;
209 }
210 }
211 end:
212 gettimeofday(&t1, NULL);
213 delta_usec = (t1.tv_sec - t0.tv_sec) * 1000000 + t1.tv_usec - t0.tv_usec;
214
215 throughput = 0;
216 if (delta_usec)
217 throughput = total * 8.0 / (double)delta_usec / 1000.0;
218 getrusage(RUSAGE_THREAD, &ru);
219 if (total > 1024*1024) {
220 unsigned long total_usec;
221 unsigned long mb = total >> 20;
222 total_usec = 1000000*ru.ru_utime.tv_sec + ru.ru_utime.tv_usec +
223 1000000*ru.ru_stime.tv_sec + ru.ru_stime.tv_usec;
224 printf("received %lg MB (%lg %% mmap'ed) in %lg s, %lg Gbit\n"
225 " cpu usage user:%lg sys:%lg, %lg usec per MB, %lu c-switches\n",
226 total / (1024.0 * 1024.0),
227 100.0*total_mmap/total,
228 (double)delta_usec / 1000000.0,
229 throughput,
230 (double)ru.ru_utime.tv_sec + (double)ru.ru_utime.tv_usec / 1000000.0,
231 (double)ru.ru_stime.tv_sec + (double)ru.ru_stime.tv_usec / 1000000.0,
232 (double)total_usec/mb,
233 ru.ru_nvcsw);
234 }
235 error:
236 free(buffer);
237 close(fd);
238 if (zflg)
239 munmap(addr, chunk_size);
240 pthread_exit(0);
241 }
242
apply_rcvsnd_buf(int fd)243 static void apply_rcvsnd_buf(int fd)
244 {
245 if (rcvbuf && setsockopt(fd, SOL_SOCKET,
246 SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)) == -1) {
247 perror("setsockopt SO_RCVBUF");
248 }
249
250 if (sndbuf && setsockopt(fd, SOL_SOCKET,
251 SO_SNDBUF, &sndbuf, sizeof(sndbuf)) == -1) {
252 perror("setsockopt SO_SNDBUF");
253 }
254 }
255
256
setup_sockaddr(int domain,const char * str_addr,struct sockaddr_storage * sockaddr)257 static void setup_sockaddr(int domain, const char *str_addr,
258 struct sockaddr_storage *sockaddr)
259 {
260 struct sockaddr_in6 *addr6 = (void *) sockaddr;
261 struct sockaddr_in *addr4 = (void *) sockaddr;
262
263 switch (domain) {
264 case PF_INET:
265 memset(addr4, 0, sizeof(*addr4));
266 addr4->sin_family = AF_INET;
267 addr4->sin_port = htons(cfg_port);
268 if (str_addr &&
269 inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1)
270 error(1, 0, "ipv4 parse error: %s", str_addr);
271 break;
272 case PF_INET6:
273 memset(addr6, 0, sizeof(*addr6));
274 addr6->sin6_family = AF_INET6;
275 addr6->sin6_port = htons(cfg_port);
276 if (str_addr &&
277 inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1)
278 error(1, 0, "ipv6 parse error: %s", str_addr);
279 break;
280 default:
281 error(1, 0, "illegal domain");
282 }
283 }
284
do_accept(int fdlisten)285 static void do_accept(int fdlisten)
286 {
287 if (setsockopt(fdlisten, SOL_SOCKET, SO_RCVLOWAT,
288 &chunk_size, sizeof(chunk_size)) == -1) {
289 perror("setsockopt SO_RCVLOWAT");
290 }
291
292 apply_rcvsnd_buf(fdlisten);
293
294 while (1) {
295 struct sockaddr_in addr;
296 socklen_t addrlen = sizeof(addr);
297 pthread_t th;
298 int fd, res;
299
300 fd = accept(fdlisten, (struct sockaddr *)&addr, &addrlen);
301 if (fd == -1) {
302 perror("accept");
303 continue;
304 }
305 res = pthread_create(&th, NULL, child_thread,
306 (void *)(unsigned long)fd);
307 if (res) {
308 errno = res;
309 perror("pthread_create");
310 close(fd);
311 }
312 }
313 }
314
main(int argc,char * argv[])315 int main(int argc, char *argv[])
316 {
317 struct sockaddr_storage listenaddr, addr;
318 unsigned int max_pacing_rate = 0;
319 unsigned long total = 0;
320 char *host = NULL;
321 int fd, c, on = 1;
322 char *buffer;
323 int sflg = 0;
324 int mss = 0;
325
326 while ((c = getopt(argc, argv, "46p:svr:w:H:zxkP:M:")) != -1) {
327 switch (c) {
328 case '4':
329 cfg_family = PF_INET;
330 cfg_alen = sizeof(struct sockaddr_in);
331 break;
332 case '6':
333 cfg_family = PF_INET6;
334 cfg_alen = sizeof(struct sockaddr_in6);
335 break;
336 case 'p':
337 cfg_port = atoi(optarg);
338 break;
339 case 'H':
340 host = optarg;
341 break;
342 case 's': /* server : listen for incoming connections */
343 sflg++;
344 break;
345 case 'r':
346 rcvbuf = atoi(optarg);
347 break;
348 case 'w':
349 sndbuf = atoi(optarg);
350 break;
351 case 'z':
352 zflg = 1;
353 break;
354 case 'M':
355 mss = atoi(optarg);
356 break;
357 case 'x':
358 xflg = 1;
359 break;
360 case 'k':
361 keepflag = 1;
362 break;
363 case 'P':
364 max_pacing_rate = atoi(optarg) ;
365 break;
366 default:
367 exit(1);
368 }
369 }
370 if (sflg) {
371 int fdlisten = socket(cfg_family, SOCK_STREAM, 0);
372
373 if (fdlisten == -1) {
374 perror("socket");
375 exit(1);
376 }
377 apply_rcvsnd_buf(fdlisten);
378 setsockopt(fdlisten, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
379
380 setup_sockaddr(cfg_family, host, &listenaddr);
381
382 if (mss &&
383 setsockopt(fdlisten, IPPROTO_TCP, TCP_MAXSEG,
384 &mss, sizeof(mss)) == -1) {
385 perror("setsockopt TCP_MAXSEG");
386 exit(1);
387 }
388 if (bind(fdlisten, (const struct sockaddr *)&listenaddr, cfg_alen) == -1) {
389 perror("bind");
390 exit(1);
391 }
392 if (listen(fdlisten, 128) == -1) {
393 perror("listen");
394 exit(1);
395 }
396 do_accept(fdlisten);
397 }
398 buffer = mmap(NULL, chunk_size, PROT_READ | PROT_WRITE,
399 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
400 if (buffer == (char *)-1) {
401 perror("mmap");
402 exit(1);
403 }
404
405 fd = socket(cfg_family, SOCK_STREAM, 0);
406 if (fd == -1) {
407 perror("socket");
408 exit(1);
409 }
410 apply_rcvsnd_buf(fd);
411
412 setup_sockaddr(cfg_family, host, &addr);
413
414 if (mss &&
415 setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, &mss, sizeof(mss)) == -1) {
416 perror("setsockopt TCP_MAXSEG");
417 exit(1);
418 }
419 if (connect(fd, (const struct sockaddr *)&addr, cfg_alen) == -1) {
420 perror("connect");
421 exit(1);
422 }
423 if (max_pacing_rate &&
424 setsockopt(fd, SOL_SOCKET, SO_MAX_PACING_RATE,
425 &max_pacing_rate, sizeof(max_pacing_rate)) == -1)
426 perror("setsockopt SO_MAX_PACING_RATE");
427
428 if (zflg && setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY,
429 &on, sizeof(on)) == -1) {
430 perror("setsockopt SO_ZEROCOPY, (-z option disabled)");
431 zflg = 0;
432 }
433 while (total < FILE_SZ) {
434 long wr = FILE_SZ - total;
435
436 if (wr > chunk_size)
437 wr = chunk_size;
438 /* Note : we just want to fill the pipe with 0 bytes */
439 wr = send(fd, buffer, wr, zflg ? MSG_ZEROCOPY : 0);
440 if (wr <= 0)
441 break;
442 total += wr;
443 }
444 close(fd);
445 munmap(buffer, chunk_size);
446 return 0;
447 }
448