1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Authors:
4 * Copyright 2001, 2002 by Robert Olsson <robert.olsson@its.uu.se>
5 * Uppsala University and
6 * Swedish University of Agricultural Sciences
7 *
8 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
9 * Ben Greear <greearb@candelatech.com>
10 * Jens Låås <jens.laas@data.slu.se>
11 *
12 * A tool for loading the network with preconfigurated packets.
13 * The tool is implemented as a linux module. Parameters are output
14 * device, delay (to hard_xmit), number of packets, and whether
15 * to use multiple SKBs or just the same one.
16 * pktgen uses the installed interface's output routine.
17 *
18 * Additional hacking by:
19 *
20 * Jens.Laas@data.slu.se
21 * Improved by ANK. 010120.
22 * Improved by ANK even more. 010212.
23 * MAC address typo fixed. 010417 --ro
24 * Integrated. 020301 --DaveM
25 * Added multiskb option 020301 --DaveM
26 * Scaling of results. 020417--sigurdur@linpro.no
27 * Significant re-work of the module:
28 * * Convert to threaded model to more efficiently be able to transmit
29 * and receive on multiple interfaces at once.
30 * * Converted many counters to __u64 to allow longer runs.
31 * * Allow configuration of ranges, like min/max IP address, MACs,
32 * and UDP-ports, for both source and destination, and can
33 * set to use a random distribution or sequentially walk the range.
34 * * Can now change most values after starting.
35 * * Place 12-byte packet in UDP payload with magic number,
36 * sequence number, and timestamp.
37 * * Add receiver code that detects dropped pkts, re-ordered pkts, and
38 * latencies (with micro-second) precision.
39 * * Add IOCTL interface to easily get counters & configuration.
40 * --Ben Greear <greearb@candelatech.com>
41 *
42 * Renamed multiskb to clone_skb and cleaned up sending core for two distinct
43 * skb modes. A clone_skb=0 mode for Ben "ranges" work and a clone_skb != 0
44 * as a "fastpath" with a configurable number of clones after alloc's.
45 * clone_skb=0 means all packets are allocated this also means ranges time
46 * stamps etc can be used. clone_skb=100 means 1 malloc is followed by 100
47 * clones.
48 *
49 * Also moved to /proc/net/pktgen/
50 * --ro
51 *
52 * Sept 10: Fixed threading/locking. Lots of bone-headed and more clever
53 * mistakes. Also merged in DaveM's patch in the -pre6 patch.
54 * --Ben Greear <greearb@candelatech.com>
55 *
56 * Integrated to 2.5.x 021029 --Lucio Maciel (luciomaciel@zipmail.com.br)
57 *
58 * 021124 Finished major redesign and rewrite for new functionality.
59 * See Documentation/networking/pktgen.rst for how to use this.
60 *
61 * The new operation:
62 * For each CPU one thread/process is created at start. This process checks
63 * for running devices in the if_list and sends packets until count is 0 it
64 * also the thread checks the thread->control which is used for inter-process
65 * communication. controlling process "posts" operations to the threads this
66 * way.
67 * The if_list is RCU protected, and the if_lock remains to protect updating
68 * of if_list, from "add_device" as it invoked from userspace (via proc write).
69 *
70 * By design there should only be *one* "controlling" process. In practice
71 * multiple write accesses gives unpredictable result. Understood by "write"
72 * to /proc gives result code that should be read be the "writer".
73 * For practical use this should be no problem.
74 *
75 * Note when adding devices to a specific CPU there good idea to also assign
76 * /proc/irq/XX/smp_affinity so TX-interrupts gets bound to the same CPU.
77 * --ro
78 *
79 * Fix refcount off by one if first packet fails, potential null deref,
80 * memleak 030710- KJP
81 *
82 * First "ranges" functionality for ipv6 030726 --ro
83 *
84 * Included flow support. 030802 ANK.
85 *
86 * Fixed unaligned access on IA-64 Grant Grundler <grundler@parisc-linux.org>
87 *
88 * Remove if fix from added Harald Welte <laforge@netfilter.org> 040419
89 * ia64 compilation fix from Aron Griffis <aron@hp.com> 040604
90 *
91 * New xmit() return, do_div and misc clean up by Stephen Hemminger
92 * <shemminger@osdl.org> 040923
93 *
94 * Randy Dunlap fixed u64 printk compiler warning
95 *
96 * Remove FCS from BW calculation. Lennert Buytenhek <buytenh@wantstofly.org>
97 * New time handling. Lennert Buytenhek <buytenh@wantstofly.org> 041213
98 *
99 * Corrections from Nikolai Malykh (nmalykh@bilim.com)
100 * Removed unused flags F_SET_SRCMAC & F_SET_SRCIP 041230
101 *
102 * interruptible_sleep_on_timeout() replaced Nishanth Aravamudan <nacc@us.ibm.com>
103 * 050103
104 *
105 * MPLS support by Steven Whitehouse <steve@chygwyn.com>
106 *
107 * 802.1Q/Q-in-Q support by Francesco Fondelli (FF) <francesco.fondelli@gmail.com>
108 *
109 * Fixed src_mac command to set source mac of packet to value specified in
110 * command by Adit Ranadive <adit.262@gmail.com>
111 */
112
113 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
114
115 #include <linux/sys.h>
116 #include <linux/types.h>
117 #include <linux/module.h>
118 #include <linux/moduleparam.h>
119 #include <linux/kernel.h>
120 #include <linux/mutex.h>
121 #include <linux/sched.h>
122 #include <linux/slab.h>
123 #include <linux/vmalloc.h>
124 #include <linux/unistd.h>
125 #include <linux/string.h>
126 #include <linux/ptrace.h>
127 #include <linux/errno.h>
128 #include <linux/ioport.h>
129 #include <linux/interrupt.h>
130 #include <linux/capability.h>
131 #include <linux/hrtimer.h>
132 #include <linux/freezer.h>
133 #include <linux/delay.h>
134 #include <linux/timer.h>
135 #include <linux/list.h>
136 #include <linux/init.h>
137 #include <linux/skbuff.h>
138 #include <linux/netdevice.h>
139 #include <linux/inet.h>
140 #include <linux/inetdevice.h>
141 #include <linux/rtnetlink.h>
142 #include <linux/if_arp.h>
143 #include <linux/if_vlan.h>
144 #include <linux/in.h>
145 #include <linux/ip.h>
146 #include <linux/ipv6.h>
147 #include <linux/udp.h>
148 #include <linux/proc_fs.h>
149 #include <linux/seq_file.h>
150 #include <linux/wait.h>
151 #include <linux/etherdevice.h>
152 #include <linux/kthread.h>
153 #include <linux/prefetch.h>
154 #include <linux/mmzone.h>
155 #include <net/net_namespace.h>
156 #include <net/checksum.h>
157 #include <net/ipv6.h>
158 #include <net/udp.h>
159 #include <net/ip6_checksum.h>
160 #include <net/addrconf.h>
161 #ifdef CONFIG_XFRM
162 #include <net/xfrm.h>
163 #endif
164 #include <net/netns/generic.h>
165 #include <asm/byteorder.h>
166 #include <linux/rcupdate.h>
167 #include <linux/bitops.h>
168 #include <linux/io.h>
169 #include <linux/timex.h>
170 #include <linux/uaccess.h>
171 #include <asm/dma.h>
172 #include <asm/div64.h> /* do_div */
173
174 #define VERSION "2.75"
175 #define IP_NAME_SZ 32
176 #define MAX_MPLS_LABELS 16 /* This is the max label stack depth */
177 #define MPLS_STACK_BOTTOM htonl(0x00000100)
178 /* Max number of internet mix entries that can be specified in imix_weights. */
179 #define MAX_IMIX_ENTRIES 20
180 #define IMIX_PRECISION 100 /* Precision of IMIX distribution */
181
182 #define func_enter() pr_debug("entering %s\n", __func__);
183
184 #define PKT_FLAGS \
185 pf(IPV6) /* Interface in IPV6 Mode */ \
186 pf(IPSRC_RND) /* IP-Src Random */ \
187 pf(IPDST_RND) /* IP-Dst Random */ \
188 pf(TXSIZE_RND) /* Transmit size is random */ \
189 pf(UDPSRC_RND) /* UDP-Src Random */ \
190 pf(UDPDST_RND) /* UDP-Dst Random */ \
191 pf(UDPCSUM) /* Include UDP checksum */ \
192 pf(NO_TIMESTAMP) /* Don't timestamp packets (default TS) */ \
193 pf(MPLS_RND) /* Random MPLS labels */ \
194 pf(QUEUE_MAP_RND) /* queue map Random */ \
195 pf(QUEUE_MAP_CPU) /* queue map mirrors smp_processor_id() */ \
196 pf(FLOW_SEQ) /* Sequential flows */ \
197 pf(IPSEC) /* ipsec on for flows */ \
198 pf(MACSRC_RND) /* MAC-Src Random */ \
199 pf(MACDST_RND) /* MAC-Dst Random */ \
200 pf(VID_RND) /* Random VLAN ID */ \
201 pf(SVID_RND) /* Random SVLAN ID */ \
202 pf(NODE) /* Node memory alloc*/ \
203 pf(SHARED) /* Shared SKB */ \
204
205 #define pf(flag) flag##_SHIFT,
206 enum pkt_flags {
207 PKT_FLAGS
208 };
209 #undef pf
210
211 /* Device flag bits */
212 #define pf(flag) static const __u32 F_##flag = (1<<flag##_SHIFT);
213 PKT_FLAGS
214 #undef pf
215
216 #define pf(flag) __stringify(flag),
217 static char *pkt_flag_names[] = {
218 PKT_FLAGS
219 };
220 #undef pf
221
222 #define NR_PKT_FLAGS ARRAY_SIZE(pkt_flag_names)
223
224 /* Thread control flag bits */
225 #define T_STOP (1<<0) /* Stop run */
226 #define T_RUN (1<<1) /* Start run */
227 #define T_REMDEVALL (1<<2) /* Remove all devs */
228 #define T_REMDEV (1<<3) /* Remove one dev */
229
230 /* Xmit modes */
231 #define M_START_XMIT 0 /* Default normal TX */
232 #define M_NETIF_RECEIVE 1 /* Inject packets into stack */
233 #define M_QUEUE_XMIT 2 /* Inject packet into qdisc */
234
235 /* If lock -- protects updating of if_list */
236 #define if_lock(t) mutex_lock(&(t->if_lock));
237 #define if_unlock(t) mutex_unlock(&(t->if_lock));
238
239 /* Used to help with determining the pkts on receive */
240 #define PKTGEN_MAGIC 0xbe9be955
241 #define PG_PROC_DIR "pktgen"
242 #define PGCTRL "pgctrl"
243
244 #define MAX_CFLOWS 65536
245
246 #define VLAN_TAG_SIZE(x) ((x)->vlan_id == 0xffff ? 0 : 4)
247 #define SVLAN_TAG_SIZE(x) ((x)->svlan_id == 0xffff ? 0 : 4)
248
249 struct imix_pkt {
250 u64 size;
251 u64 weight;
252 u64 count_so_far;
253 };
254
255 struct flow_state {
256 __be32 cur_daddr;
257 int count;
258 #ifdef CONFIG_XFRM
259 struct xfrm_state *x;
260 #endif
261 __u32 flags;
262 };
263
264 /* flow flag bits */
265 #define F_INIT (1<<0) /* flow has been initialized */
266
267 struct pktgen_dev {
268 /*
269 * Try to keep frequent/infrequent used vars. separated.
270 */
271 struct proc_dir_entry *entry; /* proc file */
272 struct pktgen_thread *pg_thread;/* the owner */
273 struct list_head list; /* chaining in the thread's run-queue */
274 struct rcu_head rcu; /* freed by RCU */
275
276 int running; /* if false, the test will stop */
277
278 /* If min != max, then we will either do a linear iteration, or
279 * we will do a random selection from within the range.
280 */
281 __u32 flags;
282 int xmit_mode;
283 int min_pkt_size;
284 int max_pkt_size;
285 int pkt_overhead; /* overhead for MPLS, VLANs, IPSEC etc */
286 int nfrags;
287 int removal_mark; /* non-zero => the device is marked for
288 * removal by worker thread */
289
290 struct page *page;
291 u64 delay; /* nano-seconds */
292
293 __u64 count; /* Default No packets to send */
294 __u64 sofar; /* How many pkts we've sent so far */
295 __u64 tx_bytes; /* How many bytes we've transmitted */
296 __u64 errors; /* Errors when trying to transmit, */
297
298 /* runtime counters relating to clone_skb */
299
300 __u32 clone_count;
301 int last_ok; /* Was last skb sent?
302 * Or a failed transmit of some sort?
303 * This will keep sequence numbers in order
304 */
305 ktime_t next_tx;
306 ktime_t started_at;
307 ktime_t stopped_at;
308 u64 idle_acc; /* nano-seconds */
309
310 __u32 seq_num;
311
312 int clone_skb; /*
313 * Use multiple SKBs during packet gen.
314 * If this number is greater than 1, then
315 * that many copies of the same packet will be
316 * sent before a new packet is allocated.
317 * If you want to send 1024 identical packets
318 * before creating a new packet,
319 * set clone_skb to 1024.
320 */
321
322 char dst_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */
323 char dst_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */
324 char src_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */
325 char src_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */
326
327 struct in6_addr in6_saddr;
328 struct in6_addr in6_daddr;
329 struct in6_addr cur_in6_daddr;
330 struct in6_addr cur_in6_saddr;
331 /* For ranges */
332 struct in6_addr min_in6_daddr;
333 struct in6_addr max_in6_daddr;
334 struct in6_addr min_in6_saddr;
335 struct in6_addr max_in6_saddr;
336
337 /* If we're doing ranges, random or incremental, then this
338 * defines the min/max for those ranges.
339 */
340 __be32 saddr_min; /* inclusive, source IP address */
341 __be32 saddr_max; /* exclusive, source IP address */
342 __be32 daddr_min; /* inclusive, dest IP address */
343 __be32 daddr_max; /* exclusive, dest IP address */
344
345 __u16 udp_src_min; /* inclusive, source UDP port */
346 __u16 udp_src_max; /* exclusive, source UDP port */
347 __u16 udp_dst_min; /* inclusive, dest UDP port */
348 __u16 udp_dst_max; /* exclusive, dest UDP port */
349
350 /* DSCP + ECN */
351 __u8 tos; /* six MSB of (former) IPv4 TOS
352 are for dscp codepoint */
353 __u8 traffic_class; /* ditto for the (former) Traffic Class in IPv6
354 (see RFC 3260, sec. 4) */
355
356 /* IMIX */
357 unsigned int n_imix_entries;
358 struct imix_pkt imix_entries[MAX_IMIX_ENTRIES];
359 /* Maps 0-IMIX_PRECISION range to imix_entry based on probability*/
360 __u8 imix_distribution[IMIX_PRECISION];
361
362 /* MPLS */
363 unsigned int nr_labels; /* Depth of stack, 0 = no MPLS */
364 __be32 labels[MAX_MPLS_LABELS];
365
366 /* VLAN/SVLAN (802.1Q/Q-in-Q) */
367 __u8 vlan_p;
368 __u8 vlan_cfi;
369 __u16 vlan_id; /* 0xffff means no vlan tag */
370
371 __u8 svlan_p;
372 __u8 svlan_cfi;
373 __u16 svlan_id; /* 0xffff means no svlan tag */
374
375 __u32 src_mac_count; /* How many MACs to iterate through */
376 __u32 dst_mac_count; /* How many MACs to iterate through */
377
378 unsigned char dst_mac[ETH_ALEN];
379 unsigned char src_mac[ETH_ALEN];
380
381 __u32 cur_dst_mac_offset;
382 __u32 cur_src_mac_offset;
383 __be32 cur_saddr;
384 __be32 cur_daddr;
385 __u16 ip_id;
386 __u16 cur_udp_dst;
387 __u16 cur_udp_src;
388 __u16 cur_queue_map;
389 __u32 cur_pkt_size;
390 __u32 last_pkt_size;
391
392 __u8 hh[14];
393 /* = {
394 0x00, 0x80, 0xC8, 0x79, 0xB3, 0xCB,
395
396 We fill in SRC address later
397 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
398 0x08, 0x00
399 };
400 */
401 __u16 pad; /* pad out the hh struct to an even 16 bytes */
402
403 struct sk_buff *skb; /* skb we are to transmit next, used for when we
404 * are transmitting the same one multiple times
405 */
406 struct net_device *odev; /* The out-going device.
407 * Note that the device should have it's
408 * pg_info pointer pointing back to this
409 * device.
410 * Set when the user specifies the out-going
411 * device name (not when the inject is
412 * started as it used to do.)
413 */
414 netdevice_tracker dev_tracker;
415 char odevname[32];
416 struct flow_state *flows;
417 unsigned int cflows; /* Concurrent flows (config) */
418 unsigned int lflow; /* Flow length (config) */
419 unsigned int nflows; /* accumulated flows (stats) */
420 unsigned int curfl; /* current sequenced flow (state)*/
421
422 u16 queue_map_min;
423 u16 queue_map_max;
424 __u32 skb_priority; /* skb priority field */
425 unsigned int burst; /* number of duplicated packets to burst */
426 int node; /* Memory node */
427
428 #ifdef CONFIG_XFRM
429 __u8 ipsmode; /* IPSEC mode (config) */
430 __u8 ipsproto; /* IPSEC type (config) */
431 __u32 spi;
432 struct xfrm_dst xdst;
433 struct dst_ops dstops;
434 #endif
435 char result[512];
436 };
437
438 struct pktgen_hdr {
439 __be32 pgh_magic;
440 __be32 seq_num;
441 __be32 tv_sec;
442 __be32 tv_usec;
443 };
444
445
446 static unsigned int pg_net_id __read_mostly;
447
448 struct pktgen_net {
449 struct net *net;
450 struct proc_dir_entry *proc_dir;
451 struct list_head pktgen_threads;
452 bool pktgen_exiting;
453 };
454
455 struct pktgen_thread {
456 struct mutex if_lock; /* for list of devices */
457 struct list_head if_list; /* All device here */
458 struct list_head th_list;
459 struct task_struct *tsk;
460 char result[512];
461
462 /* Field for thread to receive "posted" events terminate,
463 stop ifs etc. */
464
465 u32 control;
466 int cpu;
467
468 wait_queue_head_t queue;
469 struct completion start_done;
470 struct pktgen_net *net;
471 };
472
473 #define REMOVE 1
474 #define FIND 0
475
476 static const char version[] =
477 "Packet Generator for packet performance testing. "
478 "Version: " VERSION "\n";
479
480 static int pktgen_remove_device(struct pktgen_thread *t, struct pktgen_dev *i);
481 static int pktgen_add_device(struct pktgen_thread *t, const char *ifname);
482 static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t,
483 const char *ifname, bool exact);
484 static int pktgen_device_event(struct notifier_block *, unsigned long, void *);
485 static void pktgen_run_all_threads(struct pktgen_net *pn);
486 static void pktgen_reset_all_threads(struct pktgen_net *pn);
487 static void pktgen_stop_all_threads(struct pktgen_net *pn);
488
489 static void pktgen_stop(struct pktgen_thread *t);
490 static void pktgen_clear_counters(struct pktgen_dev *pkt_dev);
491 static void fill_imix_distribution(struct pktgen_dev *pkt_dev);
492
493 /* Module parameters, defaults. */
494 static int pg_count_d __read_mostly = 1000;
495 static int pg_delay_d __read_mostly;
496 static int pg_clone_skb_d __read_mostly;
497 static int debug __read_mostly;
498
499 static DEFINE_MUTEX(pktgen_thread_lock);
500
501 static struct notifier_block pktgen_notifier_block = {
502 .notifier_call = pktgen_device_event,
503 };
504
505 /*
506 * /proc handling functions
507 *
508 */
509
pgctrl_show(struct seq_file * seq,void * v)510 static int pgctrl_show(struct seq_file *seq, void *v)
511 {
512 seq_puts(seq, version);
513 return 0;
514 }
515
pgctrl_write(struct file * file,const char __user * buf,size_t count,loff_t * ppos)516 static ssize_t pgctrl_write(struct file *file, const char __user *buf,
517 size_t count, loff_t *ppos)
518 {
519 char data[128];
520 struct pktgen_net *pn = net_generic(current->nsproxy->net_ns, pg_net_id);
521
522 if (!capable(CAP_NET_ADMIN))
523 return -EPERM;
524
525 if (count == 0)
526 return -EINVAL;
527
528 if (count > sizeof(data))
529 count = sizeof(data);
530
531 if (copy_from_user(data, buf, count))
532 return -EFAULT;
533
534 data[count - 1] = 0; /* Strip trailing '\n' and terminate string */
535
536 if (!strcmp(data, "stop"))
537 pktgen_stop_all_threads(pn);
538 else if (!strcmp(data, "start"))
539 pktgen_run_all_threads(pn);
540 else if (!strcmp(data, "reset"))
541 pktgen_reset_all_threads(pn);
542 else
543 return -EINVAL;
544
545 return count;
546 }
547
pgctrl_open(struct inode * inode,struct file * file)548 static int pgctrl_open(struct inode *inode, struct file *file)
549 {
550 return single_open(file, pgctrl_show, pde_data(inode));
551 }
552
553 static const struct proc_ops pktgen_proc_ops = {
554 .proc_open = pgctrl_open,
555 .proc_read = seq_read,
556 .proc_lseek = seq_lseek,
557 .proc_write = pgctrl_write,
558 .proc_release = single_release,
559 };
560
pktgen_if_show(struct seq_file * seq,void * v)561 static int pktgen_if_show(struct seq_file *seq, void *v)
562 {
563 const struct pktgen_dev *pkt_dev = seq->private;
564 ktime_t stopped;
565 unsigned int i;
566 u64 idle;
567
568 seq_printf(seq,
569 "Params: count %llu min_pkt_size: %u max_pkt_size: %u\n",
570 (unsigned long long)pkt_dev->count, pkt_dev->min_pkt_size,
571 pkt_dev->max_pkt_size);
572
573 if (pkt_dev->n_imix_entries > 0) {
574 seq_puts(seq, " imix_weights: ");
575 for (i = 0; i < pkt_dev->n_imix_entries; i++) {
576 seq_printf(seq, "%llu,%llu ",
577 pkt_dev->imix_entries[i].size,
578 pkt_dev->imix_entries[i].weight);
579 }
580 seq_puts(seq, "\n");
581 }
582
583 seq_printf(seq,
584 " frags: %d delay: %llu clone_skb: %d ifname: %s\n",
585 pkt_dev->nfrags, (unsigned long long) pkt_dev->delay,
586 pkt_dev->clone_skb, pkt_dev->odevname);
587
588 seq_printf(seq, " flows: %u flowlen: %u\n", pkt_dev->cflows,
589 pkt_dev->lflow);
590
591 seq_printf(seq,
592 " queue_map_min: %u queue_map_max: %u\n",
593 pkt_dev->queue_map_min,
594 pkt_dev->queue_map_max);
595
596 if (pkt_dev->skb_priority)
597 seq_printf(seq, " skb_priority: %u\n",
598 pkt_dev->skb_priority);
599
600 if (pkt_dev->flags & F_IPV6) {
601 seq_printf(seq,
602 " saddr: %pI6c min_saddr: %pI6c max_saddr: %pI6c\n"
603 " daddr: %pI6c min_daddr: %pI6c max_daddr: %pI6c\n",
604 &pkt_dev->in6_saddr,
605 &pkt_dev->min_in6_saddr, &pkt_dev->max_in6_saddr,
606 &pkt_dev->in6_daddr,
607 &pkt_dev->min_in6_daddr, &pkt_dev->max_in6_daddr);
608 } else {
609 seq_printf(seq,
610 " dst_min: %s dst_max: %s\n",
611 pkt_dev->dst_min, pkt_dev->dst_max);
612 seq_printf(seq,
613 " src_min: %s src_max: %s\n",
614 pkt_dev->src_min, pkt_dev->src_max);
615 }
616
617 seq_puts(seq, " src_mac: ");
618
619 seq_printf(seq, "%pM ",
620 is_zero_ether_addr(pkt_dev->src_mac) ?
621 pkt_dev->odev->dev_addr : pkt_dev->src_mac);
622
623 seq_puts(seq, "dst_mac: ");
624 seq_printf(seq, "%pM\n", pkt_dev->dst_mac);
625
626 seq_printf(seq,
627 " udp_src_min: %d udp_src_max: %d"
628 " udp_dst_min: %d udp_dst_max: %d\n",
629 pkt_dev->udp_src_min, pkt_dev->udp_src_max,
630 pkt_dev->udp_dst_min, pkt_dev->udp_dst_max);
631
632 seq_printf(seq,
633 " src_mac_count: %d dst_mac_count: %d\n",
634 pkt_dev->src_mac_count, pkt_dev->dst_mac_count);
635
636 if (pkt_dev->nr_labels) {
637 seq_puts(seq, " mpls: ");
638 for (i = 0; i < pkt_dev->nr_labels; i++)
639 seq_printf(seq, "%08x%s", ntohl(pkt_dev->labels[i]),
640 i == pkt_dev->nr_labels-1 ? "\n" : ", ");
641 }
642
643 if (pkt_dev->vlan_id != 0xffff)
644 seq_printf(seq, " vlan_id: %u vlan_p: %u vlan_cfi: %u\n",
645 pkt_dev->vlan_id, pkt_dev->vlan_p,
646 pkt_dev->vlan_cfi);
647
648 if (pkt_dev->svlan_id != 0xffff)
649 seq_printf(seq, " svlan_id: %u vlan_p: %u vlan_cfi: %u\n",
650 pkt_dev->svlan_id, pkt_dev->svlan_p,
651 pkt_dev->svlan_cfi);
652
653 if (pkt_dev->tos)
654 seq_printf(seq, " tos: 0x%02x\n", pkt_dev->tos);
655
656 if (pkt_dev->traffic_class)
657 seq_printf(seq, " traffic_class: 0x%02x\n", pkt_dev->traffic_class);
658
659 if (pkt_dev->burst > 1)
660 seq_printf(seq, " burst: %d\n", pkt_dev->burst);
661
662 if (pkt_dev->node >= 0)
663 seq_printf(seq, " node: %d\n", pkt_dev->node);
664
665 if (pkt_dev->xmit_mode == M_NETIF_RECEIVE)
666 seq_puts(seq, " xmit_mode: netif_receive\n");
667 else if (pkt_dev->xmit_mode == M_QUEUE_XMIT)
668 seq_puts(seq, " xmit_mode: xmit_queue\n");
669
670 seq_puts(seq, " Flags: ");
671
672 for (i = 0; i < NR_PKT_FLAGS; i++) {
673 if (i == FLOW_SEQ_SHIFT)
674 if (!pkt_dev->cflows)
675 continue;
676
677 if (pkt_dev->flags & (1 << i)) {
678 seq_printf(seq, "%s ", pkt_flag_names[i]);
679 #ifdef CONFIG_XFRM
680 if (i == IPSEC_SHIFT && pkt_dev->spi)
681 seq_printf(seq, "spi:%u ", pkt_dev->spi);
682 #endif
683 } else if (i == FLOW_SEQ_SHIFT) {
684 seq_puts(seq, "FLOW_RND ");
685 }
686 }
687
688 seq_puts(seq, "\n");
689
690 /* not really stopped, more like last-running-at */
691 stopped = pkt_dev->running ? ktime_get() : pkt_dev->stopped_at;
692 idle = pkt_dev->idle_acc;
693 do_div(idle, NSEC_PER_USEC);
694
695 seq_printf(seq,
696 "Current:\n pkts-sofar: %llu errors: %llu\n",
697 (unsigned long long)pkt_dev->sofar,
698 (unsigned long long)pkt_dev->errors);
699
700 if (pkt_dev->n_imix_entries > 0) {
701 int i;
702
703 seq_puts(seq, " imix_size_counts: ");
704 for (i = 0; i < pkt_dev->n_imix_entries; i++) {
705 seq_printf(seq, "%llu,%llu ",
706 pkt_dev->imix_entries[i].size,
707 pkt_dev->imix_entries[i].count_so_far);
708 }
709 seq_puts(seq, "\n");
710 }
711
712 seq_printf(seq,
713 " started: %lluus stopped: %lluus idle: %lluus\n",
714 (unsigned long long) ktime_to_us(pkt_dev->started_at),
715 (unsigned long long) ktime_to_us(stopped),
716 (unsigned long long) idle);
717
718 seq_printf(seq,
719 " seq_num: %d cur_dst_mac_offset: %d cur_src_mac_offset: %d\n",
720 pkt_dev->seq_num, pkt_dev->cur_dst_mac_offset,
721 pkt_dev->cur_src_mac_offset);
722
723 if (pkt_dev->flags & F_IPV6) {
724 seq_printf(seq, " cur_saddr: %pI6c cur_daddr: %pI6c\n",
725 &pkt_dev->cur_in6_saddr,
726 &pkt_dev->cur_in6_daddr);
727 } else
728 seq_printf(seq, " cur_saddr: %pI4 cur_daddr: %pI4\n",
729 &pkt_dev->cur_saddr, &pkt_dev->cur_daddr);
730
731 seq_printf(seq, " cur_udp_dst: %d cur_udp_src: %d\n",
732 pkt_dev->cur_udp_dst, pkt_dev->cur_udp_src);
733
734 seq_printf(seq, " cur_queue_map: %u\n", pkt_dev->cur_queue_map);
735
736 seq_printf(seq, " flows: %u\n", pkt_dev->nflows);
737
738 if (pkt_dev->result[0])
739 seq_printf(seq, "Result: %s\n", pkt_dev->result);
740 else
741 seq_puts(seq, "Result: Idle\n");
742
743 return 0;
744 }
745
746
hex32_arg(const char __user * user_buffer,unsigned long maxlen,__u32 * num)747 static int hex32_arg(const char __user *user_buffer, unsigned long maxlen,
748 __u32 *num)
749 {
750 int i = 0;
751 *num = 0;
752
753 for (; i < maxlen; i++) {
754 int value;
755 char c;
756 *num <<= 4;
757 if (get_user(c, &user_buffer[i]))
758 return -EFAULT;
759 value = hex_to_bin(c);
760 if (value >= 0)
761 *num |= value;
762 else
763 break;
764 }
765 return i;
766 }
767
count_trail_chars(const char __user * user_buffer,unsigned int maxlen)768 static int count_trail_chars(const char __user * user_buffer,
769 unsigned int maxlen)
770 {
771 int i;
772
773 for (i = 0; i < maxlen; i++) {
774 char c;
775 if (get_user(c, &user_buffer[i]))
776 return -EFAULT;
777 switch (c) {
778 case '\"':
779 case '\n':
780 case '\r':
781 case '\t':
782 case ' ':
783 case '=':
784 break;
785 default:
786 goto done;
787 }
788 }
789 done:
790 return i;
791 }
792
num_arg(const char __user * user_buffer,unsigned long maxlen,unsigned long * num)793 static long num_arg(const char __user *user_buffer, unsigned long maxlen,
794 unsigned long *num)
795 {
796 int i;
797 *num = 0;
798
799 for (i = 0; i < maxlen; i++) {
800 char c;
801 if (get_user(c, &user_buffer[i]))
802 return -EFAULT;
803 if ((c >= '0') && (c <= '9')) {
804 *num *= 10;
805 *num += c - '0';
806 } else
807 break;
808 }
809 return i;
810 }
811
strn_len(const char __user * user_buffer,unsigned int maxlen)812 static int strn_len(const char __user * user_buffer, unsigned int maxlen)
813 {
814 int i;
815
816 for (i = 0; i < maxlen; i++) {
817 char c;
818 if (get_user(c, &user_buffer[i]))
819 return -EFAULT;
820 switch (c) {
821 case '\"':
822 case '\n':
823 case '\r':
824 case '\t':
825 case ' ':
826 goto done_str;
827 default:
828 break;
829 }
830 }
831 done_str:
832 return i;
833 }
834
835 /* Parses imix entries from user buffer.
836 * The user buffer should consist of imix entries separated by spaces
837 * where each entry consists of size and weight delimited by commas.
838 * "size1,weight_1 size2,weight_2 ... size_n,weight_n" for example.
839 */
get_imix_entries(const char __user * buffer,struct pktgen_dev * pkt_dev)840 static ssize_t get_imix_entries(const char __user *buffer,
841 struct pktgen_dev *pkt_dev)
842 {
843 const int max_digits = 10;
844 int i = 0;
845 long len;
846 char c;
847
848 pkt_dev->n_imix_entries = 0;
849
850 do {
851 unsigned long weight;
852 unsigned long size;
853
854 if (pkt_dev->n_imix_entries >= MAX_IMIX_ENTRIES)
855 return -E2BIG;
856
857 len = num_arg(&buffer[i], max_digits, &size);
858 if (len < 0)
859 return len;
860 i += len;
861 if (get_user(c, &buffer[i]))
862 return -EFAULT;
863 /* Check for comma between size_i and weight_i */
864 if (c != ',')
865 return -EINVAL;
866 i++;
867
868 if (size < 14 + 20 + 8)
869 size = 14 + 20 + 8;
870
871 len = num_arg(&buffer[i], max_digits, &weight);
872 if (len < 0)
873 return len;
874 if (weight <= 0)
875 return -EINVAL;
876
877 pkt_dev->imix_entries[pkt_dev->n_imix_entries].size = size;
878 pkt_dev->imix_entries[pkt_dev->n_imix_entries].weight = weight;
879
880 i += len;
881 if (get_user(c, &buffer[i]))
882 return -EFAULT;
883
884 i++;
885 pkt_dev->n_imix_entries++;
886 } while (c == ' ');
887
888 return i;
889 }
890
get_labels(const char __user * buffer,struct pktgen_dev * pkt_dev)891 static ssize_t get_labels(const char __user *buffer, struct pktgen_dev *pkt_dev)
892 {
893 unsigned int n = 0;
894 char c;
895 ssize_t i = 0;
896 int len;
897
898 pkt_dev->nr_labels = 0;
899 do {
900 __u32 tmp;
901
902 if (n >= MAX_MPLS_LABELS)
903 return -E2BIG;
904
905 len = hex32_arg(&buffer[i], 8, &tmp);
906 if (len <= 0)
907 return len;
908 pkt_dev->labels[n] = htonl(tmp);
909 if (pkt_dev->labels[n] & MPLS_STACK_BOTTOM)
910 pkt_dev->flags |= F_MPLS_RND;
911 i += len;
912 if (get_user(c, &buffer[i]))
913 return -EFAULT;
914 i++;
915 n++;
916 } while (c == ',');
917
918 pkt_dev->nr_labels = n;
919 return i;
920 }
921
pktgen_read_flag(const char * f,bool * disable)922 static __u32 pktgen_read_flag(const char *f, bool *disable)
923 {
924 __u32 i;
925
926 if (f[0] == '!') {
927 *disable = true;
928 f++;
929 }
930
931 for (i = 0; i < NR_PKT_FLAGS; i++) {
932 if (!IS_ENABLED(CONFIG_XFRM) && i == IPSEC_SHIFT)
933 continue;
934
935 /* allow only disabling ipv6 flag */
936 if (!*disable && i == IPV6_SHIFT)
937 continue;
938
939 if (strcmp(f, pkt_flag_names[i]) == 0)
940 return 1 << i;
941 }
942
943 if (strcmp(f, "FLOW_RND") == 0) {
944 *disable = !*disable;
945 return F_FLOW_SEQ;
946 }
947
948 return 0;
949 }
950
pktgen_if_write(struct file * file,const char __user * user_buffer,size_t count,loff_t * offset)951 static ssize_t pktgen_if_write(struct file *file,
952 const char __user * user_buffer, size_t count,
953 loff_t * offset)
954 {
955 struct seq_file *seq = file->private_data;
956 struct pktgen_dev *pkt_dev = seq->private;
957 int i, max, len;
958 char name[16], valstr[32];
959 unsigned long value = 0;
960 char *pg_result = NULL;
961 int tmp = 0;
962 char buf[128];
963
964 pg_result = &(pkt_dev->result[0]);
965
966 if (count < 1) {
967 pr_warn("wrong command format\n");
968 return -EINVAL;
969 }
970
971 max = count;
972 tmp = count_trail_chars(user_buffer, max);
973 if (tmp < 0) {
974 pr_warn("illegal format\n");
975 return tmp;
976 }
977 i = tmp;
978
979 /* Read variable name */
980
981 len = strn_len(&user_buffer[i], sizeof(name) - 1);
982 if (len < 0)
983 return len;
984
985 memset(name, 0, sizeof(name));
986 if (copy_from_user(name, &user_buffer[i], len))
987 return -EFAULT;
988 i += len;
989
990 max = count - i;
991 len = count_trail_chars(&user_buffer[i], max);
992 if (len < 0)
993 return len;
994
995 i += len;
996
997 if (debug) {
998 size_t copy = min_t(size_t, count + 1, 1024);
999 char *tp = strndup_user(user_buffer, copy);
1000
1001 if (IS_ERR(tp))
1002 return PTR_ERR(tp);
1003
1004 pr_debug("%s,%zu buffer -:%s:-\n", name, count, tp);
1005 kfree(tp);
1006 }
1007
1008 if (!strcmp(name, "min_pkt_size")) {
1009 len = num_arg(&user_buffer[i], 10, &value);
1010 if (len < 0)
1011 return len;
1012
1013 i += len;
1014 if (value < 14 + 20 + 8)
1015 value = 14 + 20 + 8;
1016 if (value != pkt_dev->min_pkt_size) {
1017 pkt_dev->min_pkt_size = value;
1018 pkt_dev->cur_pkt_size = value;
1019 }
1020 sprintf(pg_result, "OK: min_pkt_size=%d",
1021 pkt_dev->min_pkt_size);
1022 return count;
1023 }
1024
1025 if (!strcmp(name, "max_pkt_size")) {
1026 len = num_arg(&user_buffer[i], 10, &value);
1027 if (len < 0)
1028 return len;
1029
1030 i += len;
1031 if (value < 14 + 20 + 8)
1032 value = 14 + 20 + 8;
1033 if (value != pkt_dev->max_pkt_size) {
1034 pkt_dev->max_pkt_size = value;
1035 pkt_dev->cur_pkt_size = value;
1036 }
1037 sprintf(pg_result, "OK: max_pkt_size=%d",
1038 pkt_dev->max_pkt_size);
1039 return count;
1040 }
1041
1042 /* Shortcut for min = max */
1043
1044 if (!strcmp(name, "pkt_size")) {
1045 len = num_arg(&user_buffer[i], 10, &value);
1046 if (len < 0)
1047 return len;
1048
1049 i += len;
1050 if (value < 14 + 20 + 8)
1051 value = 14 + 20 + 8;
1052 if (value != pkt_dev->min_pkt_size) {
1053 pkt_dev->min_pkt_size = value;
1054 pkt_dev->max_pkt_size = value;
1055 pkt_dev->cur_pkt_size = value;
1056 }
1057 sprintf(pg_result, "OK: pkt_size=%d", pkt_dev->min_pkt_size);
1058 return count;
1059 }
1060
1061 if (!strcmp(name, "imix_weights")) {
1062 if (pkt_dev->clone_skb > 0)
1063 return -EINVAL;
1064
1065 len = get_imix_entries(&user_buffer[i], pkt_dev);
1066 if (len < 0)
1067 return len;
1068
1069 fill_imix_distribution(pkt_dev);
1070
1071 i += len;
1072 return count;
1073 }
1074
1075 if (!strcmp(name, "debug")) {
1076 len = num_arg(&user_buffer[i], 10, &value);
1077 if (len < 0)
1078 return len;
1079
1080 i += len;
1081 debug = value;
1082 sprintf(pg_result, "OK: debug=%u", debug);
1083 return count;
1084 }
1085
1086 if (!strcmp(name, "frags")) {
1087 len = num_arg(&user_buffer[i], 10, &value);
1088 if (len < 0)
1089 return len;
1090
1091 i += len;
1092 pkt_dev->nfrags = value;
1093 sprintf(pg_result, "OK: frags=%d", pkt_dev->nfrags);
1094 return count;
1095 }
1096 if (!strcmp(name, "delay")) {
1097 len = num_arg(&user_buffer[i], 10, &value);
1098 if (len < 0)
1099 return len;
1100
1101 i += len;
1102 if (value == 0x7FFFFFFF)
1103 pkt_dev->delay = ULLONG_MAX;
1104 else
1105 pkt_dev->delay = (u64)value;
1106
1107 sprintf(pg_result, "OK: delay=%llu",
1108 (unsigned long long) pkt_dev->delay);
1109 return count;
1110 }
1111 if (!strcmp(name, "rate")) {
1112 len = num_arg(&user_buffer[i], 10, &value);
1113 if (len < 0)
1114 return len;
1115
1116 i += len;
1117 if (!value)
1118 return len;
1119 pkt_dev->delay = pkt_dev->min_pkt_size*8*NSEC_PER_USEC/value;
1120 if (debug)
1121 pr_info("Delay set at: %llu ns\n", pkt_dev->delay);
1122
1123 sprintf(pg_result, "OK: rate=%lu", value);
1124 return count;
1125 }
1126 if (!strcmp(name, "ratep")) {
1127 len = num_arg(&user_buffer[i], 10, &value);
1128 if (len < 0)
1129 return len;
1130
1131 i += len;
1132 if (!value)
1133 return len;
1134 pkt_dev->delay = NSEC_PER_SEC/value;
1135 if (debug)
1136 pr_info("Delay set at: %llu ns\n", pkt_dev->delay);
1137
1138 sprintf(pg_result, "OK: rate=%lu", value);
1139 return count;
1140 }
1141 if (!strcmp(name, "udp_src_min")) {
1142 len = num_arg(&user_buffer[i], 10, &value);
1143 if (len < 0)
1144 return len;
1145
1146 i += len;
1147 if (value != pkt_dev->udp_src_min) {
1148 pkt_dev->udp_src_min = value;
1149 pkt_dev->cur_udp_src = value;
1150 }
1151 sprintf(pg_result, "OK: udp_src_min=%u", pkt_dev->udp_src_min);
1152 return count;
1153 }
1154 if (!strcmp(name, "udp_dst_min")) {
1155 len = num_arg(&user_buffer[i], 10, &value);
1156 if (len < 0)
1157 return len;
1158
1159 i += len;
1160 if (value != pkt_dev->udp_dst_min) {
1161 pkt_dev->udp_dst_min = value;
1162 pkt_dev->cur_udp_dst = value;
1163 }
1164 sprintf(pg_result, "OK: udp_dst_min=%u", pkt_dev->udp_dst_min);
1165 return count;
1166 }
1167 if (!strcmp(name, "udp_src_max")) {
1168 len = num_arg(&user_buffer[i], 10, &value);
1169 if (len < 0)
1170 return len;
1171
1172 i += len;
1173 if (value != pkt_dev->udp_src_max) {
1174 pkt_dev->udp_src_max = value;
1175 pkt_dev->cur_udp_src = value;
1176 }
1177 sprintf(pg_result, "OK: udp_src_max=%u", pkt_dev->udp_src_max);
1178 return count;
1179 }
1180 if (!strcmp(name, "udp_dst_max")) {
1181 len = num_arg(&user_buffer[i], 10, &value);
1182 if (len < 0)
1183 return len;
1184
1185 i += len;
1186 if (value != pkt_dev->udp_dst_max) {
1187 pkt_dev->udp_dst_max = value;
1188 pkt_dev->cur_udp_dst = value;
1189 }
1190 sprintf(pg_result, "OK: udp_dst_max=%u", pkt_dev->udp_dst_max);
1191 return count;
1192 }
1193 if (!strcmp(name, "clone_skb")) {
1194 len = num_arg(&user_buffer[i], 10, &value);
1195 if (len < 0)
1196 return len;
1197 /* clone_skb is not supported for netif_receive xmit_mode and
1198 * IMIX mode.
1199 */
1200 if ((value > 0) &&
1201 ((pkt_dev->xmit_mode == M_NETIF_RECEIVE) ||
1202 !(pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING)))
1203 return -ENOTSUPP;
1204 if (value > 0 && (pkt_dev->n_imix_entries > 0 ||
1205 !(pkt_dev->flags & F_SHARED)))
1206 return -EINVAL;
1207
1208 i += len;
1209 pkt_dev->clone_skb = value;
1210
1211 sprintf(pg_result, "OK: clone_skb=%d", pkt_dev->clone_skb);
1212 return count;
1213 }
1214 if (!strcmp(name, "count")) {
1215 len = num_arg(&user_buffer[i], 10, &value);
1216 if (len < 0)
1217 return len;
1218
1219 i += len;
1220 pkt_dev->count = value;
1221 sprintf(pg_result, "OK: count=%llu",
1222 (unsigned long long)pkt_dev->count);
1223 return count;
1224 }
1225 if (!strcmp(name, "src_mac_count")) {
1226 len = num_arg(&user_buffer[i], 10, &value);
1227 if (len < 0)
1228 return len;
1229
1230 i += len;
1231 if (pkt_dev->src_mac_count != value) {
1232 pkt_dev->src_mac_count = value;
1233 pkt_dev->cur_src_mac_offset = 0;
1234 }
1235 sprintf(pg_result, "OK: src_mac_count=%d",
1236 pkt_dev->src_mac_count);
1237 return count;
1238 }
1239 if (!strcmp(name, "dst_mac_count")) {
1240 len = num_arg(&user_buffer[i], 10, &value);
1241 if (len < 0)
1242 return len;
1243
1244 i += len;
1245 if (pkt_dev->dst_mac_count != value) {
1246 pkt_dev->dst_mac_count = value;
1247 pkt_dev->cur_dst_mac_offset = 0;
1248 }
1249 sprintf(pg_result, "OK: dst_mac_count=%d",
1250 pkt_dev->dst_mac_count);
1251 return count;
1252 }
1253 if (!strcmp(name, "burst")) {
1254 len = num_arg(&user_buffer[i], 10, &value);
1255 if (len < 0)
1256 return len;
1257
1258 i += len;
1259 if ((value > 1) &&
1260 ((pkt_dev->xmit_mode == M_QUEUE_XMIT) ||
1261 ((pkt_dev->xmit_mode == M_START_XMIT) &&
1262 (!(pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING)))))
1263 return -ENOTSUPP;
1264
1265 if (value > 1 && !(pkt_dev->flags & F_SHARED))
1266 return -EINVAL;
1267
1268 pkt_dev->burst = value < 1 ? 1 : value;
1269 sprintf(pg_result, "OK: burst=%u", pkt_dev->burst);
1270 return count;
1271 }
1272 if (!strcmp(name, "node")) {
1273 len = num_arg(&user_buffer[i], 10, &value);
1274 if (len < 0)
1275 return len;
1276
1277 i += len;
1278
1279 if (node_possible(value)) {
1280 pkt_dev->node = value;
1281 sprintf(pg_result, "OK: node=%d", pkt_dev->node);
1282 if (pkt_dev->page) {
1283 put_page(pkt_dev->page);
1284 pkt_dev->page = NULL;
1285 }
1286 }
1287 else
1288 sprintf(pg_result, "ERROR: node not possible");
1289 return count;
1290 }
1291 if (!strcmp(name, "xmit_mode")) {
1292 char f[32];
1293
1294 memset(f, 0, 32);
1295 len = strn_len(&user_buffer[i], sizeof(f) - 1);
1296 if (len < 0)
1297 return len;
1298
1299 if (copy_from_user(f, &user_buffer[i], len))
1300 return -EFAULT;
1301 i += len;
1302
1303 if (strcmp(f, "start_xmit") == 0) {
1304 pkt_dev->xmit_mode = M_START_XMIT;
1305 } else if (strcmp(f, "netif_receive") == 0) {
1306 /* clone_skb set earlier, not supported in this mode */
1307 if (pkt_dev->clone_skb > 0)
1308 return -ENOTSUPP;
1309
1310 pkt_dev->xmit_mode = M_NETIF_RECEIVE;
1311
1312 /* make sure new packet is allocated every time
1313 * pktgen_xmit() is called
1314 */
1315 pkt_dev->last_ok = 1;
1316 } else if (strcmp(f, "queue_xmit") == 0) {
1317 pkt_dev->xmit_mode = M_QUEUE_XMIT;
1318 pkt_dev->last_ok = 1;
1319 } else {
1320 sprintf(pg_result,
1321 "xmit_mode -:%s:- unknown\nAvailable modes: %s",
1322 f, "start_xmit, netif_receive\n");
1323 return count;
1324 }
1325 sprintf(pg_result, "OK: xmit_mode=%s", f);
1326 return count;
1327 }
1328 if (!strcmp(name, "flag")) {
1329 bool disable = false;
1330 __u32 flag;
1331 char f[32];
1332 char *end;
1333
1334 memset(f, 0, 32);
1335 len = strn_len(&user_buffer[i], sizeof(f) - 1);
1336 if (len < 0)
1337 return len;
1338
1339 if (copy_from_user(f, &user_buffer[i], len))
1340 return -EFAULT;
1341 i += len;
1342
1343 flag = pktgen_read_flag(f, &disable);
1344 if (flag) {
1345 if (disable) {
1346 /* If "clone_skb", or "burst" parameters are
1347 * configured, it means that the skb still
1348 * needs to be referenced by the pktgen, so
1349 * the skb must be shared.
1350 */
1351 if (flag == F_SHARED && (pkt_dev->clone_skb ||
1352 pkt_dev->burst > 1))
1353 return -EINVAL;
1354 pkt_dev->flags &= ~flag;
1355 } else {
1356 pkt_dev->flags |= flag;
1357 }
1358
1359 sprintf(pg_result, "OK: flags=0x%x", pkt_dev->flags);
1360 return count;
1361 }
1362
1363 /* Unknown flag */
1364 end = pkt_dev->result + sizeof(pkt_dev->result);
1365 pg_result += sprintf(pg_result,
1366 "Flag -:%s:- unknown\n"
1367 "Available flags, (prepend ! to un-set flag):\n", f);
1368
1369 for (int n = 0; n < NR_PKT_FLAGS && pg_result < end; n++) {
1370 if (!IS_ENABLED(CONFIG_XFRM) && n == IPSEC_SHIFT)
1371 continue;
1372 pg_result += snprintf(pg_result, end - pg_result,
1373 "%s, ", pkt_flag_names[n]);
1374 }
1375 if (!WARN_ON_ONCE(pg_result >= end)) {
1376 /* Remove the comma and whitespace at the end */
1377 *(pg_result - 2) = '\0';
1378 }
1379
1380 return count;
1381 }
1382 if (!strcmp(name, "dst_min") || !strcmp(name, "dst")) {
1383 len = strn_len(&user_buffer[i], sizeof(pkt_dev->dst_min) - 1);
1384 if (len < 0)
1385 return len;
1386
1387 if (copy_from_user(buf, &user_buffer[i], len))
1388 return -EFAULT;
1389 buf[len] = 0;
1390 if (strcmp(buf, pkt_dev->dst_min) != 0) {
1391 memset(pkt_dev->dst_min, 0, sizeof(pkt_dev->dst_min));
1392 strcpy(pkt_dev->dst_min, buf);
1393 pkt_dev->daddr_min = in_aton(pkt_dev->dst_min);
1394 pkt_dev->cur_daddr = pkt_dev->daddr_min;
1395 }
1396 if (debug)
1397 pr_debug("dst_min set to: %s\n", pkt_dev->dst_min);
1398 i += len;
1399 sprintf(pg_result, "OK: dst_min=%s", pkt_dev->dst_min);
1400 return count;
1401 }
1402 if (!strcmp(name, "dst_max")) {
1403 len = strn_len(&user_buffer[i], sizeof(pkt_dev->dst_max) - 1);
1404 if (len < 0)
1405 return len;
1406
1407 if (copy_from_user(buf, &user_buffer[i], len))
1408 return -EFAULT;
1409 buf[len] = 0;
1410 if (strcmp(buf, pkt_dev->dst_max) != 0) {
1411 memset(pkt_dev->dst_max, 0, sizeof(pkt_dev->dst_max));
1412 strcpy(pkt_dev->dst_max, buf);
1413 pkt_dev->daddr_max = in_aton(pkt_dev->dst_max);
1414 pkt_dev->cur_daddr = pkt_dev->daddr_max;
1415 }
1416 if (debug)
1417 pr_debug("dst_max set to: %s\n", pkt_dev->dst_max);
1418 i += len;
1419 sprintf(pg_result, "OK: dst_max=%s", pkt_dev->dst_max);
1420 return count;
1421 }
1422 if (!strcmp(name, "dst6")) {
1423 len = strn_len(&user_buffer[i], sizeof(buf) - 1);
1424 if (len < 0)
1425 return len;
1426
1427 pkt_dev->flags |= F_IPV6;
1428
1429 if (copy_from_user(buf, &user_buffer[i], len))
1430 return -EFAULT;
1431 buf[len] = 0;
1432
1433 in6_pton(buf, -1, pkt_dev->in6_daddr.s6_addr, -1, NULL);
1434 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_daddr);
1435
1436 pkt_dev->cur_in6_daddr = pkt_dev->in6_daddr;
1437
1438 if (debug)
1439 pr_debug("dst6 set to: %s\n", buf);
1440
1441 i += len;
1442 sprintf(pg_result, "OK: dst6=%s", buf);
1443 return count;
1444 }
1445 if (!strcmp(name, "dst6_min")) {
1446 len = strn_len(&user_buffer[i], sizeof(buf) - 1);
1447 if (len < 0)
1448 return len;
1449
1450 pkt_dev->flags |= F_IPV6;
1451
1452 if (copy_from_user(buf, &user_buffer[i], len))
1453 return -EFAULT;
1454 buf[len] = 0;
1455
1456 in6_pton(buf, -1, pkt_dev->min_in6_daddr.s6_addr, -1, NULL);
1457 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->min_in6_daddr);
1458
1459 pkt_dev->cur_in6_daddr = pkt_dev->min_in6_daddr;
1460 if (debug)
1461 pr_debug("dst6_min set to: %s\n", buf);
1462
1463 i += len;
1464 sprintf(pg_result, "OK: dst6_min=%s", buf);
1465 return count;
1466 }
1467 if (!strcmp(name, "dst6_max")) {
1468 len = strn_len(&user_buffer[i], sizeof(buf) - 1);
1469 if (len < 0)
1470 return len;
1471
1472 pkt_dev->flags |= F_IPV6;
1473
1474 if (copy_from_user(buf, &user_buffer[i], len))
1475 return -EFAULT;
1476 buf[len] = 0;
1477
1478 in6_pton(buf, -1, pkt_dev->max_in6_daddr.s6_addr, -1, NULL);
1479 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->max_in6_daddr);
1480
1481 if (debug)
1482 pr_debug("dst6_max set to: %s\n", buf);
1483
1484 i += len;
1485 sprintf(pg_result, "OK: dst6_max=%s", buf);
1486 return count;
1487 }
1488 if (!strcmp(name, "src6")) {
1489 len = strn_len(&user_buffer[i], sizeof(buf) - 1);
1490 if (len < 0)
1491 return len;
1492
1493 pkt_dev->flags |= F_IPV6;
1494
1495 if (copy_from_user(buf, &user_buffer[i], len))
1496 return -EFAULT;
1497 buf[len] = 0;
1498
1499 in6_pton(buf, -1, pkt_dev->in6_saddr.s6_addr, -1, NULL);
1500 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_saddr);
1501
1502 pkt_dev->cur_in6_saddr = pkt_dev->in6_saddr;
1503
1504 if (debug)
1505 pr_debug("src6 set to: %s\n", buf);
1506
1507 i += len;
1508 sprintf(pg_result, "OK: src6=%s", buf);
1509 return count;
1510 }
1511 if (!strcmp(name, "src_min")) {
1512 len = strn_len(&user_buffer[i], sizeof(pkt_dev->src_min) - 1);
1513 if (len < 0)
1514 return len;
1515
1516 if (copy_from_user(buf, &user_buffer[i], len))
1517 return -EFAULT;
1518 buf[len] = 0;
1519 if (strcmp(buf, pkt_dev->src_min) != 0) {
1520 memset(pkt_dev->src_min, 0, sizeof(pkt_dev->src_min));
1521 strcpy(pkt_dev->src_min, buf);
1522 pkt_dev->saddr_min = in_aton(pkt_dev->src_min);
1523 pkt_dev->cur_saddr = pkt_dev->saddr_min;
1524 }
1525 if (debug)
1526 pr_debug("src_min set to: %s\n", pkt_dev->src_min);
1527 i += len;
1528 sprintf(pg_result, "OK: src_min=%s", pkt_dev->src_min);
1529 return count;
1530 }
1531 if (!strcmp(name, "src_max")) {
1532 len = strn_len(&user_buffer[i], sizeof(pkt_dev->src_max) - 1);
1533 if (len < 0)
1534 return len;
1535
1536 if (copy_from_user(buf, &user_buffer[i], len))
1537 return -EFAULT;
1538 buf[len] = 0;
1539 if (strcmp(buf, pkt_dev->src_max) != 0) {
1540 memset(pkt_dev->src_max, 0, sizeof(pkt_dev->src_max));
1541 strcpy(pkt_dev->src_max, buf);
1542 pkt_dev->saddr_max = in_aton(pkt_dev->src_max);
1543 pkt_dev->cur_saddr = pkt_dev->saddr_max;
1544 }
1545 if (debug)
1546 pr_debug("src_max set to: %s\n", pkt_dev->src_max);
1547 i += len;
1548 sprintf(pg_result, "OK: src_max=%s", pkt_dev->src_max);
1549 return count;
1550 }
1551 if (!strcmp(name, "dst_mac")) {
1552 len = strn_len(&user_buffer[i], sizeof(valstr) - 1);
1553 if (len < 0)
1554 return len;
1555
1556 memset(valstr, 0, sizeof(valstr));
1557 if (copy_from_user(valstr, &user_buffer[i], len))
1558 return -EFAULT;
1559
1560 if (!mac_pton(valstr, pkt_dev->dst_mac))
1561 return -EINVAL;
1562 /* Set up Dest MAC */
1563 ether_addr_copy(&pkt_dev->hh[0], pkt_dev->dst_mac);
1564
1565 sprintf(pg_result, "OK: dstmac %pM", pkt_dev->dst_mac);
1566 return count;
1567 }
1568 if (!strcmp(name, "src_mac")) {
1569 len = strn_len(&user_buffer[i], sizeof(valstr) - 1);
1570 if (len < 0)
1571 return len;
1572
1573 memset(valstr, 0, sizeof(valstr));
1574 if (copy_from_user(valstr, &user_buffer[i], len))
1575 return -EFAULT;
1576
1577 if (!mac_pton(valstr, pkt_dev->src_mac))
1578 return -EINVAL;
1579 /* Set up Src MAC */
1580 ether_addr_copy(&pkt_dev->hh[6], pkt_dev->src_mac);
1581
1582 sprintf(pg_result, "OK: srcmac %pM", pkt_dev->src_mac);
1583 return count;
1584 }
1585
1586 if (!strcmp(name, "clear_counters")) {
1587 pktgen_clear_counters(pkt_dev);
1588 sprintf(pg_result, "OK: Clearing counters.\n");
1589 return count;
1590 }
1591
1592 if (!strcmp(name, "flows")) {
1593 len = num_arg(&user_buffer[i], 10, &value);
1594 if (len < 0)
1595 return len;
1596
1597 i += len;
1598 if (value > MAX_CFLOWS)
1599 value = MAX_CFLOWS;
1600
1601 pkt_dev->cflows = value;
1602 sprintf(pg_result, "OK: flows=%u", pkt_dev->cflows);
1603 return count;
1604 }
1605 #ifdef CONFIG_XFRM
1606 if (!strcmp(name, "spi")) {
1607 len = num_arg(&user_buffer[i], 10, &value);
1608 if (len < 0)
1609 return len;
1610
1611 i += len;
1612 pkt_dev->spi = value;
1613 sprintf(pg_result, "OK: spi=%u", pkt_dev->spi);
1614 return count;
1615 }
1616 #endif
1617 if (!strcmp(name, "flowlen")) {
1618 len = num_arg(&user_buffer[i], 10, &value);
1619 if (len < 0)
1620 return len;
1621
1622 i += len;
1623 pkt_dev->lflow = value;
1624 sprintf(pg_result, "OK: flowlen=%u", pkt_dev->lflow);
1625 return count;
1626 }
1627
1628 if (!strcmp(name, "queue_map_min")) {
1629 len = num_arg(&user_buffer[i], 5, &value);
1630 if (len < 0)
1631 return len;
1632
1633 i += len;
1634 pkt_dev->queue_map_min = value;
1635 sprintf(pg_result, "OK: queue_map_min=%u", pkt_dev->queue_map_min);
1636 return count;
1637 }
1638
1639 if (!strcmp(name, "queue_map_max")) {
1640 len = num_arg(&user_buffer[i], 5, &value);
1641 if (len < 0)
1642 return len;
1643
1644 i += len;
1645 pkt_dev->queue_map_max = value;
1646 sprintf(pg_result, "OK: queue_map_max=%u", pkt_dev->queue_map_max);
1647 return count;
1648 }
1649
1650 if (!strcmp(name, "mpls")) {
1651 unsigned int n, cnt;
1652
1653 len = get_labels(&user_buffer[i], pkt_dev);
1654 if (len < 0)
1655 return len;
1656 i += len;
1657 cnt = sprintf(pg_result, "OK: mpls=");
1658 for (n = 0; n < pkt_dev->nr_labels; n++)
1659 cnt += sprintf(pg_result + cnt,
1660 "%08x%s", ntohl(pkt_dev->labels[n]),
1661 n == pkt_dev->nr_labels-1 ? "" : ",");
1662
1663 if (pkt_dev->nr_labels && pkt_dev->vlan_id != 0xffff) {
1664 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */
1665 pkt_dev->svlan_id = 0xffff;
1666
1667 if (debug)
1668 pr_debug("VLAN/SVLAN auto turned off\n");
1669 }
1670 return count;
1671 }
1672
1673 if (!strcmp(name, "vlan_id")) {
1674 len = num_arg(&user_buffer[i], 4, &value);
1675 if (len < 0)
1676 return len;
1677
1678 i += len;
1679 if (value <= 4095) {
1680 pkt_dev->vlan_id = value; /* turn on VLAN */
1681
1682 if (debug)
1683 pr_debug("VLAN turned on\n");
1684
1685 if (debug && pkt_dev->nr_labels)
1686 pr_debug("MPLS auto turned off\n");
1687
1688 pkt_dev->nr_labels = 0; /* turn off MPLS */
1689 sprintf(pg_result, "OK: vlan_id=%u", pkt_dev->vlan_id);
1690 } else {
1691 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */
1692 pkt_dev->svlan_id = 0xffff;
1693
1694 if (debug)
1695 pr_debug("VLAN/SVLAN turned off\n");
1696 }
1697 return count;
1698 }
1699
1700 if (!strcmp(name, "vlan_p")) {
1701 len = num_arg(&user_buffer[i], 1, &value);
1702 if (len < 0)
1703 return len;
1704
1705 i += len;
1706 if ((value <= 7) && (pkt_dev->vlan_id != 0xffff)) {
1707 pkt_dev->vlan_p = value;
1708 sprintf(pg_result, "OK: vlan_p=%u", pkt_dev->vlan_p);
1709 } else {
1710 sprintf(pg_result, "ERROR: vlan_p must be 0-7");
1711 }
1712 return count;
1713 }
1714
1715 if (!strcmp(name, "vlan_cfi")) {
1716 len = num_arg(&user_buffer[i], 1, &value);
1717 if (len < 0)
1718 return len;
1719
1720 i += len;
1721 if ((value <= 1) && (pkt_dev->vlan_id != 0xffff)) {
1722 pkt_dev->vlan_cfi = value;
1723 sprintf(pg_result, "OK: vlan_cfi=%u", pkt_dev->vlan_cfi);
1724 } else {
1725 sprintf(pg_result, "ERROR: vlan_cfi must be 0-1");
1726 }
1727 return count;
1728 }
1729
1730 if (!strcmp(name, "svlan_id")) {
1731 len = num_arg(&user_buffer[i], 4, &value);
1732 if (len < 0)
1733 return len;
1734
1735 i += len;
1736 if ((value <= 4095) && ((pkt_dev->vlan_id != 0xffff))) {
1737 pkt_dev->svlan_id = value; /* turn on SVLAN */
1738
1739 if (debug)
1740 pr_debug("SVLAN turned on\n");
1741
1742 if (debug && pkt_dev->nr_labels)
1743 pr_debug("MPLS auto turned off\n");
1744
1745 pkt_dev->nr_labels = 0; /* turn off MPLS */
1746 sprintf(pg_result, "OK: svlan_id=%u", pkt_dev->svlan_id);
1747 } else {
1748 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */
1749 pkt_dev->svlan_id = 0xffff;
1750
1751 if (debug)
1752 pr_debug("VLAN/SVLAN turned off\n");
1753 }
1754 return count;
1755 }
1756
1757 if (!strcmp(name, "svlan_p")) {
1758 len = num_arg(&user_buffer[i], 1, &value);
1759 if (len < 0)
1760 return len;
1761
1762 i += len;
1763 if ((value <= 7) && (pkt_dev->svlan_id != 0xffff)) {
1764 pkt_dev->svlan_p = value;
1765 sprintf(pg_result, "OK: svlan_p=%u", pkt_dev->svlan_p);
1766 } else {
1767 sprintf(pg_result, "ERROR: svlan_p must be 0-7");
1768 }
1769 return count;
1770 }
1771
1772 if (!strcmp(name, "svlan_cfi")) {
1773 len = num_arg(&user_buffer[i], 1, &value);
1774 if (len < 0)
1775 return len;
1776
1777 i += len;
1778 if ((value <= 1) && (pkt_dev->svlan_id != 0xffff)) {
1779 pkt_dev->svlan_cfi = value;
1780 sprintf(pg_result, "OK: svlan_cfi=%u", pkt_dev->svlan_cfi);
1781 } else {
1782 sprintf(pg_result, "ERROR: svlan_cfi must be 0-1");
1783 }
1784 return count;
1785 }
1786
1787 if (!strcmp(name, "tos")) {
1788 __u32 tmp_value = 0;
1789 len = hex32_arg(&user_buffer[i], 2, &tmp_value);
1790 if (len < 0)
1791 return len;
1792
1793 i += len;
1794 if (len == 2) {
1795 pkt_dev->tos = tmp_value;
1796 sprintf(pg_result, "OK: tos=0x%02x", pkt_dev->tos);
1797 } else {
1798 sprintf(pg_result, "ERROR: tos must be 00-ff");
1799 }
1800 return count;
1801 }
1802
1803 if (!strcmp(name, "traffic_class")) {
1804 __u32 tmp_value = 0;
1805 len = hex32_arg(&user_buffer[i], 2, &tmp_value);
1806 if (len < 0)
1807 return len;
1808
1809 i += len;
1810 if (len == 2) {
1811 pkt_dev->traffic_class = tmp_value;
1812 sprintf(pg_result, "OK: traffic_class=0x%02x", pkt_dev->traffic_class);
1813 } else {
1814 sprintf(pg_result, "ERROR: traffic_class must be 00-ff");
1815 }
1816 return count;
1817 }
1818
1819 if (!strcmp(name, "skb_priority")) {
1820 len = num_arg(&user_buffer[i], 9, &value);
1821 if (len < 0)
1822 return len;
1823
1824 i += len;
1825 pkt_dev->skb_priority = value;
1826 sprintf(pg_result, "OK: skb_priority=%i",
1827 pkt_dev->skb_priority);
1828 return count;
1829 }
1830
1831 sprintf(pkt_dev->result, "No such parameter \"%s\"", name);
1832 return -EINVAL;
1833 }
1834
pktgen_if_open(struct inode * inode,struct file * file)1835 static int pktgen_if_open(struct inode *inode, struct file *file)
1836 {
1837 return single_open(file, pktgen_if_show, pde_data(inode));
1838 }
1839
1840 static const struct proc_ops pktgen_if_proc_ops = {
1841 .proc_open = pktgen_if_open,
1842 .proc_read = seq_read,
1843 .proc_lseek = seq_lseek,
1844 .proc_write = pktgen_if_write,
1845 .proc_release = single_release,
1846 };
1847
pktgen_thread_show(struct seq_file * seq,void * v)1848 static int pktgen_thread_show(struct seq_file *seq, void *v)
1849 {
1850 struct pktgen_thread *t = seq->private;
1851 const struct pktgen_dev *pkt_dev;
1852
1853 BUG_ON(!t);
1854
1855 seq_puts(seq, "Running: ");
1856
1857 rcu_read_lock();
1858 list_for_each_entry_rcu(pkt_dev, &t->if_list, list)
1859 if (pkt_dev->running)
1860 seq_printf(seq, "%s ", pkt_dev->odevname);
1861
1862 seq_puts(seq, "\nStopped: ");
1863
1864 list_for_each_entry_rcu(pkt_dev, &t->if_list, list)
1865 if (!pkt_dev->running)
1866 seq_printf(seq, "%s ", pkt_dev->odevname);
1867
1868 if (t->result[0])
1869 seq_printf(seq, "\nResult: %s\n", t->result);
1870 else
1871 seq_puts(seq, "\nResult: NA\n");
1872
1873 rcu_read_unlock();
1874
1875 return 0;
1876 }
1877
pktgen_thread_write(struct file * file,const char __user * user_buffer,size_t count,loff_t * offset)1878 static ssize_t pktgen_thread_write(struct file *file,
1879 const char __user * user_buffer,
1880 size_t count, loff_t * offset)
1881 {
1882 struct seq_file *seq = file->private_data;
1883 struct pktgen_thread *t = seq->private;
1884 int i, max, len, ret;
1885 char name[40];
1886 char *pg_result;
1887
1888 if (count < 1) {
1889 // sprintf(pg_result, "Wrong command format");
1890 return -EINVAL;
1891 }
1892
1893 max = count;
1894 len = count_trail_chars(user_buffer, max);
1895 if (len < 0)
1896 return len;
1897
1898 i = len;
1899
1900 /* Read variable name */
1901 max = min(sizeof(name) - 1, count - i);
1902 len = strn_len(&user_buffer[i], max);
1903 if (len < 0)
1904 return len;
1905
1906 memset(name, 0, sizeof(name));
1907 if (copy_from_user(name, &user_buffer[i], len))
1908 return -EFAULT;
1909 i += len;
1910
1911 max = count - i;
1912 len = count_trail_chars(&user_buffer[i], max);
1913 if (len < 0)
1914 return len;
1915
1916 i += len;
1917
1918 if (debug)
1919 pr_debug("t=%s, count=%lu\n", name, (unsigned long)count);
1920
1921 if (!t) {
1922 pr_err("ERROR: No thread\n");
1923 ret = -EINVAL;
1924 goto out;
1925 }
1926
1927 pg_result = &(t->result[0]);
1928
1929 if (!strcmp(name, "add_device")) {
1930 char f[32];
1931 memset(f, 0, 32);
1932 max = min(sizeof(f) - 1, count - i);
1933 len = strn_len(&user_buffer[i], max);
1934 if (len < 0) {
1935 ret = len;
1936 goto out;
1937 }
1938 if (copy_from_user(f, &user_buffer[i], len))
1939 return -EFAULT;
1940 i += len;
1941 mutex_lock(&pktgen_thread_lock);
1942 ret = pktgen_add_device(t, f);
1943 mutex_unlock(&pktgen_thread_lock);
1944 if (!ret) {
1945 ret = count;
1946 sprintf(pg_result, "OK: add_device=%s", f);
1947 } else
1948 sprintf(pg_result, "ERROR: can not add device %s", f);
1949 goto out;
1950 }
1951
1952 if (!strcmp(name, "rem_device_all")) {
1953 mutex_lock(&pktgen_thread_lock);
1954 t->control |= T_REMDEVALL;
1955 mutex_unlock(&pktgen_thread_lock);
1956 schedule_timeout_interruptible(msecs_to_jiffies(125)); /* Propagate thread->control */
1957 ret = count;
1958 sprintf(pg_result, "OK: rem_device_all");
1959 goto out;
1960 }
1961
1962 if (!strcmp(name, "max_before_softirq")) {
1963 sprintf(pg_result, "OK: Note! max_before_softirq is obsoleted -- Do not use");
1964 ret = count;
1965 goto out;
1966 }
1967
1968 ret = -EINVAL;
1969 out:
1970 return ret;
1971 }
1972
pktgen_thread_open(struct inode * inode,struct file * file)1973 static int pktgen_thread_open(struct inode *inode, struct file *file)
1974 {
1975 return single_open(file, pktgen_thread_show, pde_data(inode));
1976 }
1977
1978 static const struct proc_ops pktgen_thread_proc_ops = {
1979 .proc_open = pktgen_thread_open,
1980 .proc_read = seq_read,
1981 .proc_lseek = seq_lseek,
1982 .proc_write = pktgen_thread_write,
1983 .proc_release = single_release,
1984 };
1985
1986 /* Think find or remove for NN */
__pktgen_NN_threads(const struct pktgen_net * pn,const char * ifname,int remove)1987 static struct pktgen_dev *__pktgen_NN_threads(const struct pktgen_net *pn,
1988 const char *ifname, int remove)
1989 {
1990 struct pktgen_thread *t;
1991 struct pktgen_dev *pkt_dev = NULL;
1992 bool exact = (remove == FIND);
1993
1994 list_for_each_entry(t, &pn->pktgen_threads, th_list) {
1995 pkt_dev = pktgen_find_dev(t, ifname, exact);
1996 if (pkt_dev) {
1997 if (remove) {
1998 pkt_dev->removal_mark = 1;
1999 t->control |= T_REMDEV;
2000 }
2001 break;
2002 }
2003 }
2004 return pkt_dev;
2005 }
2006
2007 /*
2008 * mark a device for removal
2009 */
pktgen_mark_device(const struct pktgen_net * pn,const char * ifname)2010 static void pktgen_mark_device(const struct pktgen_net *pn, const char *ifname)
2011 {
2012 struct pktgen_dev *pkt_dev = NULL;
2013 const int max_tries = 10, msec_per_try = 125;
2014 int i = 0;
2015
2016 mutex_lock(&pktgen_thread_lock);
2017 pr_debug("%s: marking %s for removal\n", __func__, ifname);
2018
2019 while (1) {
2020
2021 pkt_dev = __pktgen_NN_threads(pn, ifname, REMOVE);
2022 if (pkt_dev == NULL)
2023 break; /* success */
2024
2025 mutex_unlock(&pktgen_thread_lock);
2026 pr_debug("%s: waiting for %s to disappear....\n",
2027 __func__, ifname);
2028 schedule_timeout_interruptible(msecs_to_jiffies(msec_per_try));
2029 mutex_lock(&pktgen_thread_lock);
2030
2031 if (++i >= max_tries) {
2032 pr_err("%s: timed out after waiting %d msec for device %s to be removed\n",
2033 __func__, msec_per_try * i, ifname);
2034 break;
2035 }
2036
2037 }
2038
2039 mutex_unlock(&pktgen_thread_lock);
2040 }
2041
pktgen_change_name(const struct pktgen_net * pn,struct net_device * dev)2042 static void pktgen_change_name(const struct pktgen_net *pn, struct net_device *dev)
2043 {
2044 struct pktgen_thread *t;
2045
2046 mutex_lock(&pktgen_thread_lock);
2047
2048 list_for_each_entry(t, &pn->pktgen_threads, th_list) {
2049 struct pktgen_dev *pkt_dev;
2050
2051 if_lock(t);
2052 list_for_each_entry(pkt_dev, &t->if_list, list) {
2053 if (pkt_dev->odev != dev)
2054 continue;
2055
2056 proc_remove(pkt_dev->entry);
2057
2058 pkt_dev->entry = proc_create_data(dev->name, 0600,
2059 pn->proc_dir,
2060 &pktgen_if_proc_ops,
2061 pkt_dev);
2062 if (!pkt_dev->entry)
2063 pr_err("can't move proc entry for '%s'\n",
2064 dev->name);
2065 break;
2066 }
2067 if_unlock(t);
2068 }
2069 mutex_unlock(&pktgen_thread_lock);
2070 }
2071
pktgen_device_event(struct notifier_block * unused,unsigned long event,void * ptr)2072 static int pktgen_device_event(struct notifier_block *unused,
2073 unsigned long event, void *ptr)
2074 {
2075 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
2076 struct pktgen_net *pn = net_generic(dev_net(dev), pg_net_id);
2077
2078 if (pn->pktgen_exiting)
2079 return NOTIFY_DONE;
2080
2081 /* It is OK that we do not hold the group lock right now,
2082 * as we run under the RTNL lock.
2083 */
2084
2085 switch (event) {
2086 case NETDEV_CHANGENAME:
2087 pktgen_change_name(pn, dev);
2088 break;
2089
2090 case NETDEV_UNREGISTER:
2091 pktgen_mark_device(pn, dev->name);
2092 break;
2093 }
2094
2095 return NOTIFY_DONE;
2096 }
2097
pktgen_dev_get_by_name(const struct pktgen_net * pn,struct pktgen_dev * pkt_dev,const char * ifname)2098 static struct net_device *pktgen_dev_get_by_name(const struct pktgen_net *pn,
2099 struct pktgen_dev *pkt_dev,
2100 const char *ifname)
2101 {
2102 char b[IFNAMSIZ+5];
2103 int i;
2104
2105 for (i = 0; ifname[i] != '@'; i++) {
2106 if (i == IFNAMSIZ)
2107 break;
2108
2109 b[i] = ifname[i];
2110 }
2111 b[i] = 0;
2112
2113 return dev_get_by_name(pn->net, b);
2114 }
2115
2116
2117 /* Associate pktgen_dev with a device. */
2118
pktgen_setup_dev(const struct pktgen_net * pn,struct pktgen_dev * pkt_dev,const char * ifname)2119 static int pktgen_setup_dev(const struct pktgen_net *pn,
2120 struct pktgen_dev *pkt_dev, const char *ifname)
2121 {
2122 struct net_device *odev;
2123 int err;
2124
2125 /* Clean old setups */
2126 if (pkt_dev->odev) {
2127 netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker);
2128 pkt_dev->odev = NULL;
2129 }
2130
2131 odev = pktgen_dev_get_by_name(pn, pkt_dev, ifname);
2132 if (!odev) {
2133 pr_err("no such netdevice: \"%s\"\n", ifname);
2134 return -ENODEV;
2135 }
2136
2137 if (odev->type != ARPHRD_ETHER && odev->type != ARPHRD_LOOPBACK) {
2138 pr_err("not an ethernet or loopback device: \"%s\"\n", ifname);
2139 err = -EINVAL;
2140 } else if (!netif_running(odev)) {
2141 pr_err("device is down: \"%s\"\n", ifname);
2142 err = -ENETDOWN;
2143 } else {
2144 pkt_dev->odev = odev;
2145 netdev_tracker_alloc(odev, &pkt_dev->dev_tracker, GFP_KERNEL);
2146 return 0;
2147 }
2148
2149 dev_put(odev);
2150 return err;
2151 }
2152
2153 /* Read pkt_dev from the interface and set up internal pktgen_dev
2154 * structure to have the right information to create/send packets
2155 */
pktgen_setup_inject(struct pktgen_dev * pkt_dev)2156 static void pktgen_setup_inject(struct pktgen_dev *pkt_dev)
2157 {
2158 int ntxq;
2159
2160 if (!pkt_dev->odev) {
2161 pr_err("ERROR: pkt_dev->odev == NULL in setup_inject\n");
2162 sprintf(pkt_dev->result,
2163 "ERROR: pkt_dev->odev == NULL in setup_inject.\n");
2164 return;
2165 }
2166
2167 /* make sure that we don't pick a non-existing transmit queue */
2168 ntxq = pkt_dev->odev->real_num_tx_queues;
2169
2170 if (ntxq <= pkt_dev->queue_map_min) {
2171 pr_warn("WARNING: Requested queue_map_min (zero-based) (%d) exceeds valid range [0 - %d] for (%d) queues on %s, resetting\n",
2172 pkt_dev->queue_map_min, (ntxq ?: 1) - 1, ntxq,
2173 pkt_dev->odevname);
2174 pkt_dev->queue_map_min = (ntxq ?: 1) - 1;
2175 }
2176 if (pkt_dev->queue_map_max >= ntxq) {
2177 pr_warn("WARNING: Requested queue_map_max (zero-based) (%d) exceeds valid range [0 - %d] for (%d) queues on %s, resetting\n",
2178 pkt_dev->queue_map_max, (ntxq ?: 1) - 1, ntxq,
2179 pkt_dev->odevname);
2180 pkt_dev->queue_map_max = (ntxq ?: 1) - 1;
2181 }
2182
2183 /* Default to the interface's mac if not explicitly set. */
2184
2185 if (is_zero_ether_addr(pkt_dev->src_mac))
2186 ether_addr_copy(&(pkt_dev->hh[6]), pkt_dev->odev->dev_addr);
2187
2188 /* Set up Dest MAC */
2189 ether_addr_copy(&(pkt_dev->hh[0]), pkt_dev->dst_mac);
2190
2191 if (pkt_dev->flags & F_IPV6) {
2192 int i, set = 0, err = 1;
2193 struct inet6_dev *idev;
2194
2195 if (pkt_dev->min_pkt_size == 0) {
2196 pkt_dev->min_pkt_size = 14 + sizeof(struct ipv6hdr)
2197 + sizeof(struct udphdr)
2198 + sizeof(struct pktgen_hdr)
2199 + pkt_dev->pkt_overhead;
2200 }
2201
2202 for (i = 0; i < sizeof(struct in6_addr); i++)
2203 if (pkt_dev->cur_in6_saddr.s6_addr[i]) {
2204 set = 1;
2205 break;
2206 }
2207
2208 if (!set) {
2209
2210 /*
2211 * Use linklevel address if unconfigured.
2212 *
2213 * use ipv6_get_lladdr if/when it's get exported
2214 */
2215
2216 rcu_read_lock();
2217 idev = __in6_dev_get(pkt_dev->odev);
2218 if (idev) {
2219 struct inet6_ifaddr *ifp;
2220
2221 read_lock_bh(&idev->lock);
2222 list_for_each_entry(ifp, &idev->addr_list, if_list) {
2223 if ((ifp->scope & IFA_LINK) &&
2224 !(ifp->flags & IFA_F_TENTATIVE)) {
2225 pkt_dev->cur_in6_saddr = ifp->addr;
2226 err = 0;
2227 break;
2228 }
2229 }
2230 read_unlock_bh(&idev->lock);
2231 }
2232 rcu_read_unlock();
2233 if (err)
2234 pr_err("ERROR: IPv6 link address not available\n");
2235 }
2236 } else {
2237 if (pkt_dev->min_pkt_size == 0) {
2238 pkt_dev->min_pkt_size = 14 + sizeof(struct iphdr)
2239 + sizeof(struct udphdr)
2240 + sizeof(struct pktgen_hdr)
2241 + pkt_dev->pkt_overhead;
2242 }
2243
2244 pkt_dev->saddr_min = 0;
2245 pkt_dev->saddr_max = 0;
2246 if (strlen(pkt_dev->src_min) == 0) {
2247
2248 struct in_device *in_dev;
2249
2250 rcu_read_lock();
2251 in_dev = __in_dev_get_rcu(pkt_dev->odev);
2252 if (in_dev) {
2253 const struct in_ifaddr *ifa;
2254
2255 ifa = rcu_dereference(in_dev->ifa_list);
2256 if (ifa) {
2257 pkt_dev->saddr_min = ifa->ifa_address;
2258 pkt_dev->saddr_max = pkt_dev->saddr_min;
2259 }
2260 }
2261 rcu_read_unlock();
2262 } else {
2263 pkt_dev->saddr_min = in_aton(pkt_dev->src_min);
2264 pkt_dev->saddr_max = in_aton(pkt_dev->src_max);
2265 }
2266
2267 pkt_dev->daddr_min = in_aton(pkt_dev->dst_min);
2268 pkt_dev->daddr_max = in_aton(pkt_dev->dst_max);
2269 }
2270 /* Initialize current values. */
2271 pkt_dev->cur_pkt_size = pkt_dev->min_pkt_size;
2272 if (pkt_dev->min_pkt_size > pkt_dev->max_pkt_size)
2273 pkt_dev->max_pkt_size = pkt_dev->min_pkt_size;
2274
2275 pkt_dev->cur_dst_mac_offset = 0;
2276 pkt_dev->cur_src_mac_offset = 0;
2277 pkt_dev->cur_saddr = pkt_dev->saddr_min;
2278 pkt_dev->cur_daddr = pkt_dev->daddr_min;
2279 pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min;
2280 pkt_dev->cur_udp_src = pkt_dev->udp_src_min;
2281 pkt_dev->nflows = 0;
2282 }
2283
2284
spin(struct pktgen_dev * pkt_dev,ktime_t spin_until)2285 static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until)
2286 {
2287 ktime_t start_time, end_time;
2288 s64 remaining;
2289 struct hrtimer_sleeper t;
2290
2291 hrtimer_init_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
2292 hrtimer_set_expires(&t.timer, spin_until);
2293
2294 remaining = ktime_to_ns(hrtimer_expires_remaining(&t.timer));
2295 if (remaining <= 0)
2296 goto out;
2297
2298 start_time = ktime_get();
2299 if (remaining < 100000) {
2300 /* for small delays (<100us), just loop until limit is reached */
2301 do {
2302 end_time = ktime_get();
2303 } while (ktime_compare(end_time, spin_until) < 0);
2304 } else {
2305 do {
2306 set_current_state(TASK_INTERRUPTIBLE);
2307 hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_ABS);
2308
2309 if (likely(t.task))
2310 schedule();
2311
2312 hrtimer_cancel(&t.timer);
2313 } while (t.task && pkt_dev->running && !signal_pending(current));
2314 __set_current_state(TASK_RUNNING);
2315 end_time = ktime_get();
2316 }
2317
2318 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(end_time, start_time));
2319 out:
2320 pkt_dev->next_tx = ktime_add_ns(spin_until, pkt_dev->delay);
2321 destroy_hrtimer_on_stack(&t.timer);
2322 }
2323
set_pkt_overhead(struct pktgen_dev * pkt_dev)2324 static inline void set_pkt_overhead(struct pktgen_dev *pkt_dev)
2325 {
2326 pkt_dev->pkt_overhead = 0;
2327 pkt_dev->pkt_overhead += pkt_dev->nr_labels*sizeof(u32);
2328 pkt_dev->pkt_overhead += VLAN_TAG_SIZE(pkt_dev);
2329 pkt_dev->pkt_overhead += SVLAN_TAG_SIZE(pkt_dev);
2330 }
2331
f_seen(const struct pktgen_dev * pkt_dev,int flow)2332 static inline int f_seen(const struct pktgen_dev *pkt_dev, int flow)
2333 {
2334 return !!(pkt_dev->flows[flow].flags & F_INIT);
2335 }
2336
f_pick(struct pktgen_dev * pkt_dev)2337 static inline int f_pick(struct pktgen_dev *pkt_dev)
2338 {
2339 int flow = pkt_dev->curfl;
2340
2341 if (pkt_dev->flags & F_FLOW_SEQ) {
2342 if (pkt_dev->flows[flow].count >= pkt_dev->lflow) {
2343 /* reset time */
2344 pkt_dev->flows[flow].count = 0;
2345 pkt_dev->flows[flow].flags = 0;
2346 pkt_dev->curfl += 1;
2347 if (pkt_dev->curfl >= pkt_dev->cflows)
2348 pkt_dev->curfl = 0; /*reset */
2349 }
2350 } else {
2351 flow = get_random_u32_below(pkt_dev->cflows);
2352 pkt_dev->curfl = flow;
2353
2354 if (pkt_dev->flows[flow].count > pkt_dev->lflow) {
2355 pkt_dev->flows[flow].count = 0;
2356 pkt_dev->flows[flow].flags = 0;
2357 }
2358 }
2359
2360 return pkt_dev->curfl;
2361 }
2362
2363
2364 #ifdef CONFIG_XFRM
2365 /* If there was already an IPSEC SA, we keep it as is, else
2366 * we go look for it ...
2367 */
2368 #define DUMMY_MARK 0
get_ipsec_sa(struct pktgen_dev * pkt_dev,int flow)2369 static void get_ipsec_sa(struct pktgen_dev *pkt_dev, int flow)
2370 {
2371 struct xfrm_state *x = pkt_dev->flows[flow].x;
2372 struct pktgen_net *pn = net_generic(dev_net(pkt_dev->odev), pg_net_id);
2373 if (!x) {
2374
2375 if (pkt_dev->spi) {
2376 /* We need as quick as possible to find the right SA
2377 * Searching with minimum criteria to achieve, this.
2378 */
2379 x = xfrm_state_lookup_byspi(pn->net, htonl(pkt_dev->spi), AF_INET);
2380 } else {
2381 /* slow path: we don't already have xfrm_state */
2382 x = xfrm_stateonly_find(pn->net, DUMMY_MARK, 0,
2383 (xfrm_address_t *)&pkt_dev->cur_daddr,
2384 (xfrm_address_t *)&pkt_dev->cur_saddr,
2385 AF_INET,
2386 pkt_dev->ipsmode,
2387 pkt_dev->ipsproto, 0);
2388 }
2389 if (x) {
2390 pkt_dev->flows[flow].x = x;
2391 set_pkt_overhead(pkt_dev);
2392 pkt_dev->pkt_overhead += x->props.header_len;
2393 }
2394
2395 }
2396 }
2397 #endif
set_cur_queue_map(struct pktgen_dev * pkt_dev)2398 static void set_cur_queue_map(struct pktgen_dev *pkt_dev)
2399 {
2400
2401 if (pkt_dev->flags & F_QUEUE_MAP_CPU)
2402 pkt_dev->cur_queue_map = smp_processor_id();
2403
2404 else if (pkt_dev->queue_map_min <= pkt_dev->queue_map_max) {
2405 __u16 t;
2406 if (pkt_dev->flags & F_QUEUE_MAP_RND) {
2407 t = get_random_u32_inclusive(pkt_dev->queue_map_min,
2408 pkt_dev->queue_map_max);
2409 } else {
2410 t = pkt_dev->cur_queue_map + 1;
2411 if (t > pkt_dev->queue_map_max)
2412 t = pkt_dev->queue_map_min;
2413 }
2414 pkt_dev->cur_queue_map = t;
2415 }
2416 pkt_dev->cur_queue_map = pkt_dev->cur_queue_map % pkt_dev->odev->real_num_tx_queues;
2417 }
2418
2419 /* Increment/randomize headers according to flags and current values
2420 * for IP src/dest, UDP src/dst port, MAC-Addr src/dst
2421 */
mod_cur_headers(struct pktgen_dev * pkt_dev)2422 static void mod_cur_headers(struct pktgen_dev *pkt_dev)
2423 {
2424 __u32 imn;
2425 __u32 imx;
2426 int flow = 0;
2427
2428 if (pkt_dev->cflows)
2429 flow = f_pick(pkt_dev);
2430
2431 /* Deal with source MAC */
2432 if (pkt_dev->src_mac_count > 1) {
2433 __u32 mc;
2434 __u32 tmp;
2435
2436 if (pkt_dev->flags & F_MACSRC_RND)
2437 mc = get_random_u32_below(pkt_dev->src_mac_count);
2438 else {
2439 mc = pkt_dev->cur_src_mac_offset++;
2440 if (pkt_dev->cur_src_mac_offset >=
2441 pkt_dev->src_mac_count)
2442 pkt_dev->cur_src_mac_offset = 0;
2443 }
2444
2445 tmp = pkt_dev->src_mac[5] + (mc & 0xFF);
2446 pkt_dev->hh[11] = tmp;
2447 tmp = (pkt_dev->src_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8));
2448 pkt_dev->hh[10] = tmp;
2449 tmp = (pkt_dev->src_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8));
2450 pkt_dev->hh[9] = tmp;
2451 tmp = (pkt_dev->src_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8));
2452 pkt_dev->hh[8] = tmp;
2453 tmp = (pkt_dev->src_mac[1] + (tmp >> 8));
2454 pkt_dev->hh[7] = tmp;
2455 }
2456
2457 /* Deal with Destination MAC */
2458 if (pkt_dev->dst_mac_count > 1) {
2459 __u32 mc;
2460 __u32 tmp;
2461
2462 if (pkt_dev->flags & F_MACDST_RND)
2463 mc = get_random_u32_below(pkt_dev->dst_mac_count);
2464
2465 else {
2466 mc = pkt_dev->cur_dst_mac_offset++;
2467 if (pkt_dev->cur_dst_mac_offset >=
2468 pkt_dev->dst_mac_count) {
2469 pkt_dev->cur_dst_mac_offset = 0;
2470 }
2471 }
2472
2473 tmp = pkt_dev->dst_mac[5] + (mc & 0xFF);
2474 pkt_dev->hh[5] = tmp;
2475 tmp = (pkt_dev->dst_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8));
2476 pkt_dev->hh[4] = tmp;
2477 tmp = (pkt_dev->dst_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8));
2478 pkt_dev->hh[3] = tmp;
2479 tmp = (pkt_dev->dst_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8));
2480 pkt_dev->hh[2] = tmp;
2481 tmp = (pkt_dev->dst_mac[1] + (tmp >> 8));
2482 pkt_dev->hh[1] = tmp;
2483 }
2484
2485 if (pkt_dev->flags & F_MPLS_RND) {
2486 unsigned int i;
2487 for (i = 0; i < pkt_dev->nr_labels; i++)
2488 if (pkt_dev->labels[i] & MPLS_STACK_BOTTOM)
2489 pkt_dev->labels[i] = MPLS_STACK_BOTTOM |
2490 ((__force __be32)get_random_u32() &
2491 htonl(0x000fffff));
2492 }
2493
2494 if ((pkt_dev->flags & F_VID_RND) && (pkt_dev->vlan_id != 0xffff)) {
2495 pkt_dev->vlan_id = get_random_u32_below(4096);
2496 }
2497
2498 if ((pkt_dev->flags & F_SVID_RND) && (pkt_dev->svlan_id != 0xffff)) {
2499 pkt_dev->svlan_id = get_random_u32_below(4096);
2500 }
2501
2502 if (pkt_dev->udp_src_min < pkt_dev->udp_src_max) {
2503 if (pkt_dev->flags & F_UDPSRC_RND)
2504 pkt_dev->cur_udp_src = get_random_u32_inclusive(pkt_dev->udp_src_min,
2505 pkt_dev->udp_src_max - 1);
2506
2507 else {
2508 pkt_dev->cur_udp_src++;
2509 if (pkt_dev->cur_udp_src >= pkt_dev->udp_src_max)
2510 pkt_dev->cur_udp_src = pkt_dev->udp_src_min;
2511 }
2512 }
2513
2514 if (pkt_dev->udp_dst_min < pkt_dev->udp_dst_max) {
2515 if (pkt_dev->flags & F_UDPDST_RND) {
2516 pkt_dev->cur_udp_dst = get_random_u32_inclusive(pkt_dev->udp_dst_min,
2517 pkt_dev->udp_dst_max - 1);
2518 } else {
2519 pkt_dev->cur_udp_dst++;
2520 if (pkt_dev->cur_udp_dst >= pkt_dev->udp_dst_max)
2521 pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min;
2522 }
2523 }
2524
2525 if (!(pkt_dev->flags & F_IPV6)) {
2526
2527 imn = ntohl(pkt_dev->saddr_min);
2528 imx = ntohl(pkt_dev->saddr_max);
2529 if (imn < imx) {
2530 __u32 t;
2531 if (pkt_dev->flags & F_IPSRC_RND)
2532 t = get_random_u32_inclusive(imn, imx - 1);
2533 else {
2534 t = ntohl(pkt_dev->cur_saddr);
2535 t++;
2536 if (t > imx)
2537 t = imn;
2538
2539 }
2540 pkt_dev->cur_saddr = htonl(t);
2541 }
2542
2543 if (pkt_dev->cflows && f_seen(pkt_dev, flow)) {
2544 pkt_dev->cur_daddr = pkt_dev->flows[flow].cur_daddr;
2545 } else {
2546 imn = ntohl(pkt_dev->daddr_min);
2547 imx = ntohl(pkt_dev->daddr_max);
2548 if (imn < imx) {
2549 __u32 t;
2550 __be32 s;
2551 if (pkt_dev->flags & F_IPDST_RND) {
2552
2553 do {
2554 t = get_random_u32_inclusive(imn, imx - 1);
2555 s = htonl(t);
2556 } while (ipv4_is_loopback(s) ||
2557 ipv4_is_multicast(s) ||
2558 ipv4_is_lbcast(s) ||
2559 ipv4_is_zeronet(s) ||
2560 ipv4_is_local_multicast(s));
2561 pkt_dev->cur_daddr = s;
2562 } else {
2563 t = ntohl(pkt_dev->cur_daddr);
2564 t++;
2565 if (t > imx) {
2566 t = imn;
2567 }
2568 pkt_dev->cur_daddr = htonl(t);
2569 }
2570 }
2571 if (pkt_dev->cflows) {
2572 pkt_dev->flows[flow].flags |= F_INIT;
2573 pkt_dev->flows[flow].cur_daddr =
2574 pkt_dev->cur_daddr;
2575 #ifdef CONFIG_XFRM
2576 if (pkt_dev->flags & F_IPSEC)
2577 get_ipsec_sa(pkt_dev, flow);
2578 #endif
2579 pkt_dev->nflows++;
2580 }
2581 }
2582 } else { /* IPV6 * */
2583
2584 if (!ipv6_addr_any(&pkt_dev->min_in6_daddr)) {
2585 int i;
2586
2587 /* Only random destinations yet */
2588
2589 for (i = 0; i < 4; i++) {
2590 pkt_dev->cur_in6_daddr.s6_addr32[i] =
2591 (((__force __be32)get_random_u32() |
2592 pkt_dev->min_in6_daddr.s6_addr32[i]) &
2593 pkt_dev->max_in6_daddr.s6_addr32[i]);
2594 }
2595 }
2596 }
2597
2598 if (pkt_dev->min_pkt_size < pkt_dev->max_pkt_size) {
2599 __u32 t;
2600 if (pkt_dev->flags & F_TXSIZE_RND) {
2601 t = get_random_u32_inclusive(pkt_dev->min_pkt_size,
2602 pkt_dev->max_pkt_size - 1);
2603 } else {
2604 t = pkt_dev->cur_pkt_size + 1;
2605 if (t > pkt_dev->max_pkt_size)
2606 t = pkt_dev->min_pkt_size;
2607 }
2608 pkt_dev->cur_pkt_size = t;
2609 } else if (pkt_dev->n_imix_entries > 0) {
2610 struct imix_pkt *entry;
2611 __u32 t = get_random_u32_below(IMIX_PRECISION);
2612 __u8 entry_index = pkt_dev->imix_distribution[t];
2613
2614 entry = &pkt_dev->imix_entries[entry_index];
2615 entry->count_so_far++;
2616 pkt_dev->cur_pkt_size = entry->size;
2617 }
2618
2619 set_cur_queue_map(pkt_dev);
2620
2621 pkt_dev->flows[flow].count++;
2622 }
2623
fill_imix_distribution(struct pktgen_dev * pkt_dev)2624 static void fill_imix_distribution(struct pktgen_dev *pkt_dev)
2625 {
2626 int cumulative_probabilites[MAX_IMIX_ENTRIES];
2627 int j = 0;
2628 __u64 cumulative_prob = 0;
2629 __u64 total_weight = 0;
2630 int i = 0;
2631
2632 for (i = 0; i < pkt_dev->n_imix_entries; i++)
2633 total_weight += pkt_dev->imix_entries[i].weight;
2634
2635 /* Fill cumulative_probabilites with sum of normalized probabilities */
2636 for (i = 0; i < pkt_dev->n_imix_entries - 1; i++) {
2637 cumulative_prob += div64_u64(pkt_dev->imix_entries[i].weight *
2638 IMIX_PRECISION,
2639 total_weight);
2640 cumulative_probabilites[i] = cumulative_prob;
2641 }
2642 cumulative_probabilites[pkt_dev->n_imix_entries - 1] = 100;
2643
2644 for (i = 0; i < IMIX_PRECISION; i++) {
2645 if (i == cumulative_probabilites[j])
2646 j++;
2647 pkt_dev->imix_distribution[i] = j;
2648 }
2649 }
2650
2651 #ifdef CONFIG_XFRM
2652 static u32 pktgen_dst_metrics[RTAX_MAX + 1] = {
2653
2654 [RTAX_HOPLIMIT] = 0x5, /* Set a static hoplimit */
2655 };
2656
pktgen_output_ipsec(struct sk_buff * skb,struct pktgen_dev * pkt_dev)2657 static int pktgen_output_ipsec(struct sk_buff *skb, struct pktgen_dev *pkt_dev)
2658 {
2659 struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x;
2660 int err = 0;
2661 struct net *net = dev_net(pkt_dev->odev);
2662
2663 if (!x)
2664 return 0;
2665 /* XXX: we dont support tunnel mode for now until
2666 * we resolve the dst issue */
2667 if ((x->props.mode != XFRM_MODE_TRANSPORT) && (pkt_dev->spi == 0))
2668 return 0;
2669
2670 /* But when user specify an valid SPI, transformation
2671 * supports both transport/tunnel mode + ESP/AH type.
2672 */
2673 if ((x->props.mode == XFRM_MODE_TUNNEL) && (pkt_dev->spi != 0))
2674 skb->_skb_refdst = (unsigned long)&pkt_dev->xdst.u.dst | SKB_DST_NOREF;
2675
2676 rcu_read_lock_bh();
2677 err = pktgen_xfrm_outer_mode_output(x, skb);
2678 rcu_read_unlock_bh();
2679 if (err) {
2680 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEMODEERROR);
2681 goto error;
2682 }
2683 err = x->type->output(x, skb);
2684 if (err) {
2685 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEPROTOERROR);
2686 goto error;
2687 }
2688 spin_lock_bh(&x->lock);
2689 x->curlft.bytes += skb->len;
2690 x->curlft.packets++;
2691 spin_unlock_bh(&x->lock);
2692 error:
2693 return err;
2694 }
2695
free_SAs(struct pktgen_dev * pkt_dev)2696 static void free_SAs(struct pktgen_dev *pkt_dev)
2697 {
2698 if (pkt_dev->cflows) {
2699 /* let go of the SAs if we have them */
2700 int i;
2701 for (i = 0; i < pkt_dev->cflows; i++) {
2702 struct xfrm_state *x = pkt_dev->flows[i].x;
2703 if (x) {
2704 xfrm_state_put(x);
2705 pkt_dev->flows[i].x = NULL;
2706 }
2707 }
2708 }
2709 }
2710
process_ipsec(struct pktgen_dev * pkt_dev,struct sk_buff * skb,__be16 protocol)2711 static int process_ipsec(struct pktgen_dev *pkt_dev,
2712 struct sk_buff *skb, __be16 protocol)
2713 {
2714 if (pkt_dev->flags & F_IPSEC) {
2715 struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x;
2716 int nhead = 0;
2717 if (x) {
2718 struct ethhdr *eth;
2719 struct iphdr *iph;
2720 int ret;
2721
2722 nhead = x->props.header_len - skb_headroom(skb);
2723 if (nhead > 0) {
2724 ret = pskb_expand_head(skb, nhead, 0, GFP_ATOMIC);
2725 if (ret < 0) {
2726 pr_err("Error expanding ipsec packet %d\n",
2727 ret);
2728 goto err;
2729 }
2730 }
2731
2732 /* ipsec is not expecting ll header */
2733 skb_pull(skb, ETH_HLEN);
2734 ret = pktgen_output_ipsec(skb, pkt_dev);
2735 if (ret) {
2736 pr_err("Error creating ipsec packet %d\n", ret);
2737 goto err;
2738 }
2739 /* restore ll */
2740 eth = skb_push(skb, ETH_HLEN);
2741 memcpy(eth, pkt_dev->hh, 2 * ETH_ALEN);
2742 eth->h_proto = protocol;
2743
2744 /* Update IPv4 header len as well as checksum value */
2745 iph = ip_hdr(skb);
2746 iph->tot_len = htons(skb->len - ETH_HLEN);
2747 ip_send_check(iph);
2748 }
2749 }
2750 return 1;
2751 err:
2752 kfree_skb(skb);
2753 return 0;
2754 }
2755 #endif
2756
mpls_push(__be32 * mpls,struct pktgen_dev * pkt_dev)2757 static void mpls_push(__be32 *mpls, struct pktgen_dev *pkt_dev)
2758 {
2759 unsigned int i;
2760 for (i = 0; i < pkt_dev->nr_labels; i++)
2761 *mpls++ = pkt_dev->labels[i] & ~MPLS_STACK_BOTTOM;
2762
2763 mpls--;
2764 *mpls |= MPLS_STACK_BOTTOM;
2765 }
2766
build_tci(unsigned int id,unsigned int cfi,unsigned int prio)2767 static inline __be16 build_tci(unsigned int id, unsigned int cfi,
2768 unsigned int prio)
2769 {
2770 return htons(id | (cfi << 12) | (prio << 13));
2771 }
2772
pktgen_finalize_skb(struct pktgen_dev * pkt_dev,struct sk_buff * skb,int datalen)2773 static void pktgen_finalize_skb(struct pktgen_dev *pkt_dev, struct sk_buff *skb,
2774 int datalen)
2775 {
2776 struct timespec64 timestamp;
2777 struct pktgen_hdr *pgh;
2778
2779 pgh = skb_put(skb, sizeof(*pgh));
2780 datalen -= sizeof(*pgh);
2781
2782 if (pkt_dev->nfrags <= 0) {
2783 skb_put_zero(skb, datalen);
2784 } else {
2785 int frags = pkt_dev->nfrags;
2786 int i, len;
2787 int frag_len;
2788
2789
2790 if (frags > MAX_SKB_FRAGS)
2791 frags = MAX_SKB_FRAGS;
2792 len = datalen - frags * PAGE_SIZE;
2793 if (len > 0) {
2794 skb_put_zero(skb, len);
2795 datalen = frags * PAGE_SIZE;
2796 }
2797
2798 i = 0;
2799 frag_len = (datalen/frags) < PAGE_SIZE ?
2800 (datalen/frags) : PAGE_SIZE;
2801 while (datalen > 0) {
2802 if (unlikely(!pkt_dev->page)) {
2803 int node = numa_node_id();
2804
2805 if (pkt_dev->node >= 0 && (pkt_dev->flags & F_NODE))
2806 node = pkt_dev->node;
2807 pkt_dev->page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
2808 if (!pkt_dev->page)
2809 break;
2810 }
2811 get_page(pkt_dev->page);
2812
2813 /*last fragment, fill rest of data*/
2814 if (i == (frags - 1))
2815 skb_frag_fill_page_desc(&skb_shinfo(skb)->frags[i],
2816 pkt_dev->page, 0,
2817 (datalen < PAGE_SIZE ?
2818 datalen : PAGE_SIZE));
2819 else
2820 skb_frag_fill_page_desc(&skb_shinfo(skb)->frags[i],
2821 pkt_dev->page, 0, frag_len);
2822
2823 datalen -= skb_frag_size(&skb_shinfo(skb)->frags[i]);
2824 skb->len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
2825 skb->data_len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
2826 i++;
2827 skb_shinfo(skb)->nr_frags = i;
2828 }
2829 }
2830
2831 /* Stamp the time, and sequence number,
2832 * convert them to network byte order
2833 */
2834 pgh->pgh_magic = htonl(PKTGEN_MAGIC);
2835 pgh->seq_num = htonl(pkt_dev->seq_num);
2836
2837 if (pkt_dev->flags & F_NO_TIMESTAMP) {
2838 pgh->tv_sec = 0;
2839 pgh->tv_usec = 0;
2840 } else {
2841 /*
2842 * pgh->tv_sec wraps in y2106 when interpreted as unsigned
2843 * as done by wireshark, or y2038 when interpreted as signed.
2844 * This is probably harmless, but if anyone wants to improve
2845 * it, we could introduce a variant that puts 64-bit nanoseconds
2846 * into the respective header bytes.
2847 * This would also be slightly faster to read.
2848 */
2849 ktime_get_real_ts64(×tamp);
2850 pgh->tv_sec = htonl(timestamp.tv_sec);
2851 pgh->tv_usec = htonl(timestamp.tv_nsec / NSEC_PER_USEC);
2852 }
2853 }
2854
pktgen_alloc_skb(struct net_device * dev,struct pktgen_dev * pkt_dev)2855 static struct sk_buff *pktgen_alloc_skb(struct net_device *dev,
2856 struct pktgen_dev *pkt_dev)
2857 {
2858 unsigned int extralen = LL_RESERVED_SPACE(dev);
2859 struct sk_buff *skb = NULL;
2860 unsigned int size;
2861
2862 size = pkt_dev->cur_pkt_size + 64 + extralen + pkt_dev->pkt_overhead;
2863 if (pkt_dev->flags & F_NODE) {
2864 int node = pkt_dev->node >= 0 ? pkt_dev->node : numa_node_id();
2865
2866 skb = __alloc_skb(NET_SKB_PAD + size, GFP_NOWAIT, 0, node);
2867 if (likely(skb)) {
2868 skb_reserve(skb, NET_SKB_PAD);
2869 skb->dev = dev;
2870 }
2871 } else {
2872 skb = __netdev_alloc_skb(dev, size, GFP_NOWAIT);
2873 }
2874
2875 /* the caller pre-fetches from skb->data and reserves for the mac hdr */
2876 if (likely(skb))
2877 skb_reserve(skb, extralen - 16);
2878
2879 return skb;
2880 }
2881
fill_packet_ipv4(struct net_device * odev,struct pktgen_dev * pkt_dev)2882 static struct sk_buff *fill_packet_ipv4(struct net_device *odev,
2883 struct pktgen_dev *pkt_dev)
2884 {
2885 struct sk_buff *skb = NULL;
2886 __u8 *eth;
2887 struct udphdr *udph;
2888 int datalen, iplen;
2889 struct iphdr *iph;
2890 __be16 protocol = htons(ETH_P_IP);
2891 __be32 *mpls;
2892 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */
2893 __be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */
2894 __be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */
2895 __be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */
2896 u16 queue_map;
2897
2898 if (pkt_dev->nr_labels)
2899 protocol = htons(ETH_P_MPLS_UC);
2900
2901 if (pkt_dev->vlan_id != 0xffff)
2902 protocol = htons(ETH_P_8021Q);
2903
2904 /* Update any of the values, used when we're incrementing various
2905 * fields.
2906 */
2907 mod_cur_headers(pkt_dev);
2908 queue_map = pkt_dev->cur_queue_map;
2909
2910 skb = pktgen_alloc_skb(odev, pkt_dev);
2911 if (!skb) {
2912 sprintf(pkt_dev->result, "No memory");
2913 return NULL;
2914 }
2915
2916 prefetchw(skb->data);
2917 skb_reserve(skb, 16);
2918
2919 /* Reserve for ethernet and IP header */
2920 eth = skb_push(skb, 14);
2921 mpls = skb_put(skb, pkt_dev->nr_labels * sizeof(__u32));
2922 if (pkt_dev->nr_labels)
2923 mpls_push(mpls, pkt_dev);
2924
2925 if (pkt_dev->vlan_id != 0xffff) {
2926 if (pkt_dev->svlan_id != 0xffff) {
2927 svlan_tci = skb_put(skb, sizeof(__be16));
2928 *svlan_tci = build_tci(pkt_dev->svlan_id,
2929 pkt_dev->svlan_cfi,
2930 pkt_dev->svlan_p);
2931 svlan_encapsulated_proto = skb_put(skb,
2932 sizeof(__be16));
2933 *svlan_encapsulated_proto = htons(ETH_P_8021Q);
2934 }
2935 vlan_tci = skb_put(skb, sizeof(__be16));
2936 *vlan_tci = build_tci(pkt_dev->vlan_id,
2937 pkt_dev->vlan_cfi,
2938 pkt_dev->vlan_p);
2939 vlan_encapsulated_proto = skb_put(skb, sizeof(__be16));
2940 *vlan_encapsulated_proto = htons(ETH_P_IP);
2941 }
2942
2943 skb_reset_mac_header(skb);
2944 skb_set_network_header(skb, skb->len);
2945 iph = skb_put(skb, sizeof(struct iphdr));
2946
2947 skb_set_transport_header(skb, skb->len);
2948 udph = skb_put(skb, sizeof(struct udphdr));
2949 skb_set_queue_mapping(skb, queue_map);
2950 skb->priority = pkt_dev->skb_priority;
2951
2952 memcpy(eth, pkt_dev->hh, 12);
2953 *(__be16 *) & eth[12] = protocol;
2954
2955 /* Eth + IPh + UDPh + mpls */
2956 datalen = pkt_dev->cur_pkt_size - 14 - 20 - 8 -
2957 pkt_dev->pkt_overhead;
2958 if (datalen < 0 || datalen < sizeof(struct pktgen_hdr))
2959 datalen = sizeof(struct pktgen_hdr);
2960
2961 udph->source = htons(pkt_dev->cur_udp_src);
2962 udph->dest = htons(pkt_dev->cur_udp_dst);
2963 udph->len = htons(datalen + 8); /* DATA + udphdr */
2964 udph->check = 0;
2965
2966 iph->ihl = 5;
2967 iph->version = 4;
2968 iph->ttl = 32;
2969 iph->tos = pkt_dev->tos;
2970 iph->protocol = IPPROTO_UDP; /* UDP */
2971 iph->saddr = pkt_dev->cur_saddr;
2972 iph->daddr = pkt_dev->cur_daddr;
2973 iph->id = htons(pkt_dev->ip_id);
2974 pkt_dev->ip_id++;
2975 iph->frag_off = 0;
2976 iplen = 20 + 8 + datalen;
2977 iph->tot_len = htons(iplen);
2978 ip_send_check(iph);
2979 skb->protocol = protocol;
2980 skb->dev = odev;
2981 skb->pkt_type = PACKET_HOST;
2982
2983 pktgen_finalize_skb(pkt_dev, skb, datalen);
2984
2985 if (!(pkt_dev->flags & F_UDPCSUM)) {
2986 skb->ip_summed = CHECKSUM_NONE;
2987 } else if (odev->features & (NETIF_F_HW_CSUM | NETIF_F_IP_CSUM)) {
2988 skb->ip_summed = CHECKSUM_PARTIAL;
2989 skb->csum = 0;
2990 udp4_hwcsum(skb, iph->saddr, iph->daddr);
2991 } else {
2992 __wsum csum = skb_checksum(skb, skb_transport_offset(skb), datalen + 8, 0);
2993
2994 /* add protocol-dependent pseudo-header */
2995 udph->check = csum_tcpudp_magic(iph->saddr, iph->daddr,
2996 datalen + 8, IPPROTO_UDP, csum);
2997
2998 if (udph->check == 0)
2999 udph->check = CSUM_MANGLED_0;
3000 }
3001
3002 #ifdef CONFIG_XFRM
3003 if (!process_ipsec(pkt_dev, skb, protocol))
3004 return NULL;
3005 #endif
3006
3007 return skb;
3008 }
3009
fill_packet_ipv6(struct net_device * odev,struct pktgen_dev * pkt_dev)3010 static struct sk_buff *fill_packet_ipv6(struct net_device *odev,
3011 struct pktgen_dev *pkt_dev)
3012 {
3013 struct sk_buff *skb = NULL;
3014 __u8 *eth;
3015 struct udphdr *udph;
3016 int datalen, udplen;
3017 struct ipv6hdr *iph;
3018 __be16 protocol = htons(ETH_P_IPV6);
3019 __be32 *mpls;
3020 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */
3021 __be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */
3022 __be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */
3023 __be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */
3024 u16 queue_map;
3025
3026 if (pkt_dev->nr_labels)
3027 protocol = htons(ETH_P_MPLS_UC);
3028
3029 if (pkt_dev->vlan_id != 0xffff)
3030 protocol = htons(ETH_P_8021Q);
3031
3032 /* Update any of the values, used when we're incrementing various
3033 * fields.
3034 */
3035 mod_cur_headers(pkt_dev);
3036 queue_map = pkt_dev->cur_queue_map;
3037
3038 skb = pktgen_alloc_skb(odev, pkt_dev);
3039 if (!skb) {
3040 sprintf(pkt_dev->result, "No memory");
3041 return NULL;
3042 }
3043
3044 prefetchw(skb->data);
3045 skb_reserve(skb, 16);
3046
3047 /* Reserve for ethernet and IP header */
3048 eth = skb_push(skb, 14);
3049 mpls = skb_put(skb, pkt_dev->nr_labels * sizeof(__u32));
3050 if (pkt_dev->nr_labels)
3051 mpls_push(mpls, pkt_dev);
3052
3053 if (pkt_dev->vlan_id != 0xffff) {
3054 if (pkt_dev->svlan_id != 0xffff) {
3055 svlan_tci = skb_put(skb, sizeof(__be16));
3056 *svlan_tci = build_tci(pkt_dev->svlan_id,
3057 pkt_dev->svlan_cfi,
3058 pkt_dev->svlan_p);
3059 svlan_encapsulated_proto = skb_put(skb,
3060 sizeof(__be16));
3061 *svlan_encapsulated_proto = htons(ETH_P_8021Q);
3062 }
3063 vlan_tci = skb_put(skb, sizeof(__be16));
3064 *vlan_tci = build_tci(pkt_dev->vlan_id,
3065 pkt_dev->vlan_cfi,
3066 pkt_dev->vlan_p);
3067 vlan_encapsulated_proto = skb_put(skb, sizeof(__be16));
3068 *vlan_encapsulated_proto = htons(ETH_P_IPV6);
3069 }
3070
3071 skb_reset_mac_header(skb);
3072 skb_set_network_header(skb, skb->len);
3073 iph = skb_put(skb, sizeof(struct ipv6hdr));
3074
3075 skb_set_transport_header(skb, skb->len);
3076 udph = skb_put(skb, sizeof(struct udphdr));
3077 skb_set_queue_mapping(skb, queue_map);
3078 skb->priority = pkt_dev->skb_priority;
3079
3080 memcpy(eth, pkt_dev->hh, 12);
3081 *(__be16 *) ð[12] = protocol;
3082
3083 /* Eth + IPh + UDPh + mpls */
3084 datalen = pkt_dev->cur_pkt_size - 14 -
3085 sizeof(struct ipv6hdr) - sizeof(struct udphdr) -
3086 pkt_dev->pkt_overhead;
3087
3088 if (datalen < 0 || datalen < sizeof(struct pktgen_hdr)) {
3089 datalen = sizeof(struct pktgen_hdr);
3090 net_info_ratelimited("increased datalen to %d\n", datalen);
3091 }
3092
3093 udplen = datalen + sizeof(struct udphdr);
3094 udph->source = htons(pkt_dev->cur_udp_src);
3095 udph->dest = htons(pkt_dev->cur_udp_dst);
3096 udph->len = htons(udplen);
3097 udph->check = 0;
3098
3099 *(__be32 *) iph = htonl(0x60000000); /* Version + flow */
3100
3101 if (pkt_dev->traffic_class) {
3102 /* Version + traffic class + flow (0) */
3103 *(__be32 *)iph |= htonl(0x60000000 | (pkt_dev->traffic_class << 20));
3104 }
3105
3106 iph->hop_limit = 32;
3107
3108 iph->payload_len = htons(udplen);
3109 iph->nexthdr = IPPROTO_UDP;
3110
3111 iph->daddr = pkt_dev->cur_in6_daddr;
3112 iph->saddr = pkt_dev->cur_in6_saddr;
3113
3114 skb->protocol = protocol;
3115 skb->dev = odev;
3116 skb->pkt_type = PACKET_HOST;
3117
3118 pktgen_finalize_skb(pkt_dev, skb, datalen);
3119
3120 if (!(pkt_dev->flags & F_UDPCSUM)) {
3121 skb->ip_summed = CHECKSUM_NONE;
3122 } else if (odev->features & (NETIF_F_HW_CSUM | NETIF_F_IPV6_CSUM)) {
3123 skb->ip_summed = CHECKSUM_PARTIAL;
3124 skb->csum_start = skb_transport_header(skb) - skb->head;
3125 skb->csum_offset = offsetof(struct udphdr, check);
3126 udph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, udplen, IPPROTO_UDP, 0);
3127 } else {
3128 __wsum csum = skb_checksum(skb, skb_transport_offset(skb), udplen, 0);
3129
3130 /* add protocol-dependent pseudo-header */
3131 udph->check = csum_ipv6_magic(&iph->saddr, &iph->daddr, udplen, IPPROTO_UDP, csum);
3132
3133 if (udph->check == 0)
3134 udph->check = CSUM_MANGLED_0;
3135 }
3136
3137 return skb;
3138 }
3139
fill_packet(struct net_device * odev,struct pktgen_dev * pkt_dev)3140 static struct sk_buff *fill_packet(struct net_device *odev,
3141 struct pktgen_dev *pkt_dev)
3142 {
3143 if (pkt_dev->flags & F_IPV6)
3144 return fill_packet_ipv6(odev, pkt_dev);
3145 else
3146 return fill_packet_ipv4(odev, pkt_dev);
3147 }
3148
pktgen_clear_counters(struct pktgen_dev * pkt_dev)3149 static void pktgen_clear_counters(struct pktgen_dev *pkt_dev)
3150 {
3151 pkt_dev->seq_num = 1;
3152 pkt_dev->idle_acc = 0;
3153 pkt_dev->sofar = 0;
3154 pkt_dev->tx_bytes = 0;
3155 pkt_dev->errors = 0;
3156 }
3157
3158 /* Set up structure for sending pkts, clear counters */
3159
pktgen_run(struct pktgen_thread * t)3160 static void pktgen_run(struct pktgen_thread *t)
3161 {
3162 struct pktgen_dev *pkt_dev;
3163 int started = 0;
3164
3165 func_enter();
3166
3167 rcu_read_lock();
3168 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) {
3169
3170 /*
3171 * setup odev and create initial packet.
3172 */
3173 pktgen_setup_inject(pkt_dev);
3174
3175 if (pkt_dev->odev) {
3176 pktgen_clear_counters(pkt_dev);
3177 pkt_dev->skb = NULL;
3178 pkt_dev->started_at = pkt_dev->next_tx = ktime_get();
3179
3180 set_pkt_overhead(pkt_dev);
3181
3182 strcpy(pkt_dev->result, "Starting");
3183 pkt_dev->running = 1; /* Cranke yeself! */
3184 started++;
3185 } else
3186 strcpy(pkt_dev->result, "Error starting");
3187 }
3188 rcu_read_unlock();
3189 if (started)
3190 t->control &= ~(T_STOP);
3191 }
3192
pktgen_handle_all_threads(struct pktgen_net * pn,u32 flags)3193 static void pktgen_handle_all_threads(struct pktgen_net *pn, u32 flags)
3194 {
3195 struct pktgen_thread *t;
3196
3197 mutex_lock(&pktgen_thread_lock);
3198
3199 list_for_each_entry(t, &pn->pktgen_threads, th_list)
3200 t->control |= (flags);
3201
3202 mutex_unlock(&pktgen_thread_lock);
3203 }
3204
pktgen_stop_all_threads(struct pktgen_net * pn)3205 static void pktgen_stop_all_threads(struct pktgen_net *pn)
3206 {
3207 func_enter();
3208
3209 pktgen_handle_all_threads(pn, T_STOP);
3210 }
3211
thread_is_running(const struct pktgen_thread * t)3212 static int thread_is_running(const struct pktgen_thread *t)
3213 {
3214 const struct pktgen_dev *pkt_dev;
3215
3216 rcu_read_lock();
3217 list_for_each_entry_rcu(pkt_dev, &t->if_list, list)
3218 if (pkt_dev->running) {
3219 rcu_read_unlock();
3220 return 1;
3221 }
3222 rcu_read_unlock();
3223 return 0;
3224 }
3225
pktgen_wait_thread_run(struct pktgen_thread * t)3226 static int pktgen_wait_thread_run(struct pktgen_thread *t)
3227 {
3228 while (thread_is_running(t)) {
3229
3230 /* note: 't' will still be around even after the unlock/lock
3231 * cycle because pktgen_thread threads are only cleared at
3232 * net exit
3233 */
3234 mutex_unlock(&pktgen_thread_lock);
3235 msleep_interruptible(100);
3236 mutex_lock(&pktgen_thread_lock);
3237
3238 if (signal_pending(current))
3239 goto signal;
3240 }
3241 return 1;
3242 signal:
3243 return 0;
3244 }
3245
pktgen_wait_all_threads_run(struct pktgen_net * pn)3246 static int pktgen_wait_all_threads_run(struct pktgen_net *pn)
3247 {
3248 struct pktgen_thread *t;
3249 int sig = 1;
3250
3251 /* prevent from racing with rmmod */
3252 if (!try_module_get(THIS_MODULE))
3253 return sig;
3254
3255 mutex_lock(&pktgen_thread_lock);
3256
3257 list_for_each_entry(t, &pn->pktgen_threads, th_list) {
3258 sig = pktgen_wait_thread_run(t);
3259 if (sig == 0)
3260 break;
3261 }
3262
3263 if (sig == 0)
3264 list_for_each_entry(t, &pn->pktgen_threads, th_list)
3265 t->control |= (T_STOP);
3266
3267 mutex_unlock(&pktgen_thread_lock);
3268 module_put(THIS_MODULE);
3269 return sig;
3270 }
3271
pktgen_run_all_threads(struct pktgen_net * pn)3272 static void pktgen_run_all_threads(struct pktgen_net *pn)
3273 {
3274 func_enter();
3275
3276 pktgen_handle_all_threads(pn, T_RUN);
3277
3278 /* Propagate thread->control */
3279 schedule_timeout_interruptible(msecs_to_jiffies(125));
3280
3281 pktgen_wait_all_threads_run(pn);
3282 }
3283
pktgen_reset_all_threads(struct pktgen_net * pn)3284 static void pktgen_reset_all_threads(struct pktgen_net *pn)
3285 {
3286 func_enter();
3287
3288 pktgen_handle_all_threads(pn, T_REMDEVALL);
3289
3290 /* Propagate thread->control */
3291 schedule_timeout_interruptible(msecs_to_jiffies(125));
3292
3293 pktgen_wait_all_threads_run(pn);
3294 }
3295
show_results(struct pktgen_dev * pkt_dev,int nr_frags)3296 static void show_results(struct pktgen_dev *pkt_dev, int nr_frags)
3297 {
3298 __u64 bps, mbps, pps;
3299 char *p = pkt_dev->result;
3300 ktime_t elapsed = ktime_sub(pkt_dev->stopped_at,
3301 pkt_dev->started_at);
3302 ktime_t idle = ns_to_ktime(pkt_dev->idle_acc);
3303
3304 p += sprintf(p, "OK: %llu(c%llu+d%llu) usec, %llu (%dbyte,%dfrags)\n",
3305 (unsigned long long)ktime_to_us(elapsed),
3306 (unsigned long long)ktime_to_us(ktime_sub(elapsed, idle)),
3307 (unsigned long long)ktime_to_us(idle),
3308 (unsigned long long)pkt_dev->sofar,
3309 pkt_dev->cur_pkt_size, nr_frags);
3310
3311 pps = div64_u64(pkt_dev->sofar * NSEC_PER_SEC,
3312 ktime_to_ns(elapsed));
3313
3314 if (pkt_dev->n_imix_entries > 0) {
3315 int i;
3316 struct imix_pkt *entry;
3317
3318 bps = 0;
3319 for (i = 0; i < pkt_dev->n_imix_entries; i++) {
3320 entry = &pkt_dev->imix_entries[i];
3321 bps += entry->size * entry->count_so_far;
3322 }
3323 bps = div64_u64(bps * 8 * NSEC_PER_SEC, ktime_to_ns(elapsed));
3324 } else {
3325 bps = pps * 8 * pkt_dev->cur_pkt_size;
3326 }
3327
3328 mbps = bps;
3329 do_div(mbps, 1000000);
3330 p += sprintf(p, " %llupps %lluMb/sec (%llubps) errors: %llu",
3331 (unsigned long long)pps,
3332 (unsigned long long)mbps,
3333 (unsigned long long)bps,
3334 (unsigned long long)pkt_dev->errors);
3335 }
3336
3337 /* Set stopped-at timer, remove from running list, do counters & statistics */
pktgen_stop_device(struct pktgen_dev * pkt_dev)3338 static int pktgen_stop_device(struct pktgen_dev *pkt_dev)
3339 {
3340 int nr_frags = pkt_dev->skb ? skb_shinfo(pkt_dev->skb)->nr_frags : -1;
3341
3342 if (!pkt_dev->running) {
3343 pr_warn("interface: %s is already stopped\n",
3344 pkt_dev->odevname);
3345 return -EINVAL;
3346 }
3347
3348 pkt_dev->running = 0;
3349 kfree_skb(pkt_dev->skb);
3350 pkt_dev->skb = NULL;
3351 pkt_dev->stopped_at = ktime_get();
3352
3353 show_results(pkt_dev, nr_frags);
3354
3355 return 0;
3356 }
3357
next_to_run(struct pktgen_thread * t)3358 static struct pktgen_dev *next_to_run(struct pktgen_thread *t)
3359 {
3360 struct pktgen_dev *pkt_dev, *best = NULL;
3361
3362 rcu_read_lock();
3363 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) {
3364 if (!pkt_dev->running)
3365 continue;
3366 if (best == NULL)
3367 best = pkt_dev;
3368 else if (ktime_compare(pkt_dev->next_tx, best->next_tx) < 0)
3369 best = pkt_dev;
3370 }
3371 rcu_read_unlock();
3372
3373 return best;
3374 }
3375
pktgen_stop(struct pktgen_thread * t)3376 static void pktgen_stop(struct pktgen_thread *t)
3377 {
3378 struct pktgen_dev *pkt_dev;
3379
3380 func_enter();
3381
3382 rcu_read_lock();
3383
3384 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) {
3385 pktgen_stop_device(pkt_dev);
3386 }
3387
3388 rcu_read_unlock();
3389 }
3390
3391 /*
3392 * one of our devices needs to be removed - find it
3393 * and remove it
3394 */
pktgen_rem_one_if(struct pktgen_thread * t)3395 static void pktgen_rem_one_if(struct pktgen_thread *t)
3396 {
3397 struct list_head *q, *n;
3398 struct pktgen_dev *cur;
3399
3400 func_enter();
3401
3402 list_for_each_safe(q, n, &t->if_list) {
3403 cur = list_entry(q, struct pktgen_dev, list);
3404
3405 if (!cur->removal_mark)
3406 continue;
3407
3408 kfree_skb(cur->skb);
3409 cur->skb = NULL;
3410
3411 pktgen_remove_device(t, cur);
3412
3413 break;
3414 }
3415 }
3416
pktgen_rem_all_ifs(struct pktgen_thread * t)3417 static void pktgen_rem_all_ifs(struct pktgen_thread *t)
3418 {
3419 struct list_head *q, *n;
3420 struct pktgen_dev *cur;
3421
3422 func_enter();
3423
3424 /* Remove all devices, free mem */
3425
3426 list_for_each_safe(q, n, &t->if_list) {
3427 cur = list_entry(q, struct pktgen_dev, list);
3428
3429 kfree_skb(cur->skb);
3430 cur->skb = NULL;
3431
3432 pktgen_remove_device(t, cur);
3433 }
3434 }
3435
pktgen_rem_thread(struct pktgen_thread * t)3436 static void pktgen_rem_thread(struct pktgen_thread *t)
3437 {
3438 /* Remove from the thread list */
3439 remove_proc_entry(t->tsk->comm, t->net->proc_dir);
3440 }
3441
pktgen_resched(struct pktgen_dev * pkt_dev)3442 static void pktgen_resched(struct pktgen_dev *pkt_dev)
3443 {
3444 ktime_t idle_start = ktime_get();
3445 schedule();
3446 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(ktime_get(), idle_start));
3447 }
3448
pktgen_wait_for_skb(struct pktgen_dev * pkt_dev)3449 static void pktgen_wait_for_skb(struct pktgen_dev *pkt_dev)
3450 {
3451 ktime_t idle_start = ktime_get();
3452
3453 while (refcount_read(&(pkt_dev->skb->users)) != 1) {
3454 if (signal_pending(current))
3455 break;
3456
3457 if (need_resched())
3458 pktgen_resched(pkt_dev);
3459 else
3460 cpu_relax();
3461 }
3462 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(ktime_get(), idle_start));
3463 }
3464
pktgen_xmit(struct pktgen_dev * pkt_dev)3465 static void pktgen_xmit(struct pktgen_dev *pkt_dev)
3466 {
3467 bool skb_shared = !!(READ_ONCE(pkt_dev->flags) & F_SHARED);
3468 struct net_device *odev = pkt_dev->odev;
3469 struct netdev_queue *txq;
3470 unsigned int burst = 1;
3471 struct sk_buff *skb;
3472 int clone_skb = 0;
3473 int ret;
3474
3475 /* If 'skb_shared' is false, the read of possible
3476 * new values (if any) for 'burst' and 'clone_skb' will be skipped to
3477 * prevent some concurrent changes from slipping in. And the stabilized
3478 * config will be read in during the next run of pktgen_xmit.
3479 */
3480 if (skb_shared) {
3481 burst = READ_ONCE(pkt_dev->burst);
3482 clone_skb = READ_ONCE(pkt_dev->clone_skb);
3483 }
3484
3485 /* If device is offline, then don't send */
3486 if (unlikely(!netif_running(odev) || !netif_carrier_ok(odev))) {
3487 pktgen_stop_device(pkt_dev);
3488 return;
3489 }
3490
3491 /* This is max DELAY, this has special meaning of
3492 * "never transmit"
3493 */
3494 if (unlikely(pkt_dev->delay == ULLONG_MAX)) {
3495 pkt_dev->next_tx = ktime_add_ns(ktime_get(), ULONG_MAX);
3496 return;
3497 }
3498
3499 /* If no skb or clone count exhausted then get new one */
3500 if (!pkt_dev->skb || (pkt_dev->last_ok &&
3501 ++pkt_dev->clone_count >= clone_skb)) {
3502 /* build a new pkt */
3503 kfree_skb(pkt_dev->skb);
3504
3505 pkt_dev->skb = fill_packet(odev, pkt_dev);
3506 if (pkt_dev->skb == NULL) {
3507 pr_err("ERROR: couldn't allocate skb in fill_packet\n");
3508 schedule();
3509 pkt_dev->clone_count--; /* back out increment, OOM */
3510 return;
3511 }
3512 pkt_dev->last_pkt_size = pkt_dev->skb->len;
3513 pkt_dev->clone_count = 0; /* reset counter */
3514 }
3515
3516 if (pkt_dev->delay && pkt_dev->last_ok)
3517 spin(pkt_dev, pkt_dev->next_tx);
3518
3519 if (pkt_dev->xmit_mode == M_NETIF_RECEIVE) {
3520 skb = pkt_dev->skb;
3521 skb->protocol = eth_type_trans(skb, skb->dev);
3522 if (skb_shared)
3523 refcount_add(burst, &skb->users);
3524 local_bh_disable();
3525 do {
3526 ret = netif_receive_skb(skb);
3527 if (ret == NET_RX_DROP)
3528 pkt_dev->errors++;
3529 pkt_dev->sofar++;
3530 pkt_dev->seq_num++;
3531 if (unlikely(!skb_shared)) {
3532 pkt_dev->skb = NULL;
3533 break;
3534 }
3535 if (refcount_read(&skb->users) != burst) {
3536 /* skb was queued by rps/rfs or taps,
3537 * so cannot reuse this skb
3538 */
3539 WARN_ON(refcount_sub_and_test(burst - 1, &skb->users));
3540 /* get out of the loop and wait
3541 * until skb is consumed
3542 */
3543 break;
3544 }
3545 /* skb was 'freed' by stack, so clean few
3546 * bits and reuse it
3547 */
3548 skb_reset_redirect(skb);
3549 } while (--burst > 0);
3550 goto out; /* Skips xmit_mode M_START_XMIT */
3551 } else if (pkt_dev->xmit_mode == M_QUEUE_XMIT) {
3552 local_bh_disable();
3553 if (skb_shared)
3554 refcount_inc(&pkt_dev->skb->users);
3555
3556 ret = dev_queue_xmit(pkt_dev->skb);
3557
3558 if (!skb_shared && dev_xmit_complete(ret))
3559 pkt_dev->skb = NULL;
3560
3561 switch (ret) {
3562 case NET_XMIT_SUCCESS:
3563 pkt_dev->sofar++;
3564 pkt_dev->seq_num++;
3565 pkt_dev->tx_bytes += pkt_dev->last_pkt_size;
3566 break;
3567 case NET_XMIT_DROP:
3568 case NET_XMIT_CN:
3569 /* These are all valid return codes for a qdisc but
3570 * indicate packets are being dropped or will likely
3571 * be dropped soon.
3572 */
3573 case NETDEV_TX_BUSY:
3574 /* qdisc may call dev_hard_start_xmit directly in cases
3575 * where no queues exist e.g. loopback device, virtual
3576 * devices, etc. In this case we need to handle
3577 * NETDEV_TX_ codes.
3578 */
3579 default:
3580 pkt_dev->errors++;
3581 net_info_ratelimited("%s xmit error: %d\n",
3582 pkt_dev->odevname, ret);
3583 break;
3584 }
3585 goto out;
3586 }
3587
3588 txq = skb_get_tx_queue(odev, pkt_dev->skb);
3589
3590 local_bh_disable();
3591
3592 HARD_TX_LOCK(odev, txq, smp_processor_id());
3593
3594 if (unlikely(netif_xmit_frozen_or_drv_stopped(txq))) {
3595 pkt_dev->last_ok = 0;
3596 goto unlock;
3597 }
3598 if (skb_shared)
3599 refcount_add(burst, &pkt_dev->skb->users);
3600
3601 xmit_more:
3602 ret = netdev_start_xmit(pkt_dev->skb, odev, txq, --burst > 0);
3603
3604 if (!skb_shared && dev_xmit_complete(ret))
3605 pkt_dev->skb = NULL;
3606
3607 switch (ret) {
3608 case NETDEV_TX_OK:
3609 pkt_dev->last_ok = 1;
3610 pkt_dev->sofar++;
3611 pkt_dev->seq_num++;
3612 pkt_dev->tx_bytes += pkt_dev->last_pkt_size;
3613 if (burst > 0 && !netif_xmit_frozen_or_drv_stopped(txq))
3614 goto xmit_more;
3615 break;
3616 case NET_XMIT_DROP:
3617 case NET_XMIT_CN:
3618 /* skb has been consumed */
3619 pkt_dev->errors++;
3620 break;
3621 default: /* Drivers are not supposed to return other values! */
3622 net_info_ratelimited("%s xmit error: %d\n",
3623 pkt_dev->odevname, ret);
3624 pkt_dev->errors++;
3625 fallthrough;
3626 case NETDEV_TX_BUSY:
3627 /* Retry it next time */
3628 if (skb_shared)
3629 refcount_dec(&pkt_dev->skb->users);
3630 pkt_dev->last_ok = 0;
3631 }
3632 if (unlikely(burst))
3633 WARN_ON(refcount_sub_and_test(burst, &pkt_dev->skb->users));
3634 unlock:
3635 HARD_TX_UNLOCK(odev, txq);
3636
3637 out:
3638 local_bh_enable();
3639
3640 /* If pkt_dev->count is zero, then run forever */
3641 if ((pkt_dev->count != 0) && (pkt_dev->sofar >= pkt_dev->count)) {
3642 if (pkt_dev->skb)
3643 pktgen_wait_for_skb(pkt_dev);
3644
3645 /* Done with this */
3646 pktgen_stop_device(pkt_dev);
3647 }
3648 }
3649
3650 /*
3651 * Main loop of the thread goes here
3652 */
3653
pktgen_thread_worker(void * arg)3654 static int pktgen_thread_worker(void *arg)
3655 {
3656 struct pktgen_thread *t = arg;
3657 struct pktgen_dev *pkt_dev = NULL;
3658 int cpu = t->cpu;
3659
3660 WARN_ON_ONCE(smp_processor_id() != cpu);
3661
3662 init_waitqueue_head(&t->queue);
3663 complete(&t->start_done);
3664
3665 pr_debug("starting pktgen/%d: pid=%d\n", cpu, task_pid_nr(current));
3666
3667 set_freezable();
3668
3669 while (!kthread_should_stop()) {
3670 pkt_dev = next_to_run(t);
3671
3672 if (unlikely(!pkt_dev && t->control == 0)) {
3673 if (t->net->pktgen_exiting)
3674 break;
3675 wait_event_freezable_timeout(t->queue,
3676 t->control != 0, HZ / 10);
3677 continue;
3678 }
3679
3680 if (likely(pkt_dev)) {
3681 pktgen_xmit(pkt_dev);
3682
3683 if (need_resched())
3684 pktgen_resched(pkt_dev);
3685 else
3686 cpu_relax();
3687 }
3688
3689 if (t->control & T_STOP) {
3690 pktgen_stop(t);
3691 t->control &= ~(T_STOP);
3692 }
3693
3694 if (t->control & T_RUN) {
3695 pktgen_run(t);
3696 t->control &= ~(T_RUN);
3697 }
3698
3699 if (t->control & T_REMDEVALL) {
3700 pktgen_rem_all_ifs(t);
3701 t->control &= ~(T_REMDEVALL);
3702 }
3703
3704 if (t->control & T_REMDEV) {
3705 pktgen_rem_one_if(t);
3706 t->control &= ~(T_REMDEV);
3707 }
3708
3709 try_to_freeze();
3710 }
3711
3712 pr_debug("%s stopping all device\n", t->tsk->comm);
3713 pktgen_stop(t);
3714
3715 pr_debug("%s removing all device\n", t->tsk->comm);
3716 pktgen_rem_all_ifs(t);
3717
3718 pr_debug("%s removing thread\n", t->tsk->comm);
3719 pktgen_rem_thread(t);
3720
3721 return 0;
3722 }
3723
pktgen_find_dev(struct pktgen_thread * t,const char * ifname,bool exact)3724 static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t,
3725 const char *ifname, bool exact)
3726 {
3727 struct pktgen_dev *p, *pkt_dev = NULL;
3728 size_t len = strlen(ifname);
3729
3730 rcu_read_lock();
3731 list_for_each_entry_rcu(p, &t->if_list, list)
3732 if (strncmp(p->odevname, ifname, len) == 0) {
3733 if (p->odevname[len]) {
3734 if (exact || p->odevname[len] != '@')
3735 continue;
3736 }
3737 pkt_dev = p;
3738 break;
3739 }
3740
3741 rcu_read_unlock();
3742 pr_debug("find_dev(%s) returning %p\n", ifname, pkt_dev);
3743 return pkt_dev;
3744 }
3745
3746 /*
3747 * Adds a dev at front of if_list.
3748 */
3749
add_dev_to_thread(struct pktgen_thread * t,struct pktgen_dev * pkt_dev)3750 static int add_dev_to_thread(struct pktgen_thread *t,
3751 struct pktgen_dev *pkt_dev)
3752 {
3753 int rv = 0;
3754
3755 /* This function cannot be called concurrently, as its called
3756 * under pktgen_thread_lock mutex, but it can run from
3757 * userspace on another CPU than the kthread. The if_lock()
3758 * is used here to sync with concurrent instances of
3759 * _rem_dev_from_if_list() invoked via kthread, which is also
3760 * updating the if_list */
3761 if_lock(t);
3762
3763 if (pkt_dev->pg_thread) {
3764 pr_err("ERROR: already assigned to a thread\n");
3765 rv = -EBUSY;
3766 goto out;
3767 }
3768
3769 pkt_dev->running = 0;
3770 pkt_dev->pg_thread = t;
3771 list_add_rcu(&pkt_dev->list, &t->if_list);
3772
3773 out:
3774 if_unlock(t);
3775 return rv;
3776 }
3777
3778 /* Called under thread lock */
3779
pktgen_add_device(struct pktgen_thread * t,const char * ifname)3780 static int pktgen_add_device(struct pktgen_thread *t, const char *ifname)
3781 {
3782 struct pktgen_dev *pkt_dev;
3783 int err;
3784 int node = cpu_to_node(t->cpu);
3785
3786 /* We don't allow a device to be on several threads */
3787
3788 pkt_dev = __pktgen_NN_threads(t->net, ifname, FIND);
3789 if (pkt_dev) {
3790 pr_err("ERROR: interface already used\n");
3791 return -EBUSY;
3792 }
3793
3794 pkt_dev = kzalloc_node(sizeof(struct pktgen_dev), GFP_KERNEL, node);
3795 if (!pkt_dev)
3796 return -ENOMEM;
3797
3798 strcpy(pkt_dev->odevname, ifname);
3799 pkt_dev->flows = vzalloc_node(array_size(MAX_CFLOWS,
3800 sizeof(struct flow_state)),
3801 node);
3802 if (pkt_dev->flows == NULL) {
3803 kfree(pkt_dev);
3804 return -ENOMEM;
3805 }
3806
3807 pkt_dev->removal_mark = 0;
3808 pkt_dev->nfrags = 0;
3809 pkt_dev->delay = pg_delay_d;
3810 pkt_dev->count = pg_count_d;
3811 pkt_dev->sofar = 0;
3812 pkt_dev->udp_src_min = 9; /* sink port */
3813 pkt_dev->udp_src_max = 9;
3814 pkt_dev->udp_dst_min = 9;
3815 pkt_dev->udp_dst_max = 9;
3816 pkt_dev->vlan_p = 0;
3817 pkt_dev->vlan_cfi = 0;
3818 pkt_dev->vlan_id = 0xffff;
3819 pkt_dev->svlan_p = 0;
3820 pkt_dev->svlan_cfi = 0;
3821 pkt_dev->svlan_id = 0xffff;
3822 pkt_dev->burst = 1;
3823 pkt_dev->node = NUMA_NO_NODE;
3824 pkt_dev->flags = F_SHARED; /* SKB shared by default */
3825
3826 err = pktgen_setup_dev(t->net, pkt_dev, ifname);
3827 if (err)
3828 goto out1;
3829 if (pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING)
3830 pkt_dev->clone_skb = pg_clone_skb_d;
3831
3832 pkt_dev->entry = proc_create_data(ifname, 0600, t->net->proc_dir,
3833 &pktgen_if_proc_ops, pkt_dev);
3834 if (!pkt_dev->entry) {
3835 pr_err("cannot create %s/%s procfs entry\n",
3836 PG_PROC_DIR, ifname);
3837 err = -EINVAL;
3838 goto out2;
3839 }
3840 #ifdef CONFIG_XFRM
3841 pkt_dev->ipsmode = XFRM_MODE_TRANSPORT;
3842 pkt_dev->ipsproto = IPPROTO_ESP;
3843
3844 /* xfrm tunnel mode needs additional dst to extract outer
3845 * ip header protocol/ttl/id field, here create a phony one.
3846 * instead of looking for a valid rt, which definitely hurting
3847 * performance under such circumstance.
3848 */
3849 pkt_dev->dstops.family = AF_INET;
3850 pkt_dev->xdst.u.dst.dev = pkt_dev->odev;
3851 dst_init_metrics(&pkt_dev->xdst.u.dst, pktgen_dst_metrics, false);
3852 pkt_dev->xdst.child = &pkt_dev->xdst.u.dst;
3853 pkt_dev->xdst.u.dst.ops = &pkt_dev->dstops;
3854 #endif
3855
3856 return add_dev_to_thread(t, pkt_dev);
3857 out2:
3858 netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker);
3859 out1:
3860 #ifdef CONFIG_XFRM
3861 free_SAs(pkt_dev);
3862 #endif
3863 vfree(pkt_dev->flows);
3864 kfree(pkt_dev);
3865 return err;
3866 }
3867
pktgen_create_thread(int cpu,struct pktgen_net * pn)3868 static int __net_init pktgen_create_thread(int cpu, struct pktgen_net *pn)
3869 {
3870 struct pktgen_thread *t;
3871 struct proc_dir_entry *pe;
3872 struct task_struct *p;
3873
3874 t = kzalloc_node(sizeof(struct pktgen_thread), GFP_KERNEL,
3875 cpu_to_node(cpu));
3876 if (!t) {
3877 pr_err("ERROR: out of memory, can't create new thread\n");
3878 return -ENOMEM;
3879 }
3880
3881 mutex_init(&t->if_lock);
3882 t->cpu = cpu;
3883
3884 INIT_LIST_HEAD(&t->if_list);
3885
3886 list_add_tail(&t->th_list, &pn->pktgen_threads);
3887 init_completion(&t->start_done);
3888
3889 p = kthread_create_on_node(pktgen_thread_worker,
3890 t,
3891 cpu_to_node(cpu),
3892 "kpktgend_%d", cpu);
3893 if (IS_ERR(p)) {
3894 pr_err("kthread_create_on_node() failed for cpu %d\n", t->cpu);
3895 list_del(&t->th_list);
3896 kfree(t);
3897 return PTR_ERR(p);
3898 }
3899 kthread_bind(p, cpu);
3900 t->tsk = p;
3901
3902 pe = proc_create_data(t->tsk->comm, 0600, pn->proc_dir,
3903 &pktgen_thread_proc_ops, t);
3904 if (!pe) {
3905 pr_err("cannot create %s/%s procfs entry\n",
3906 PG_PROC_DIR, t->tsk->comm);
3907 kthread_stop(p);
3908 list_del(&t->th_list);
3909 kfree(t);
3910 return -EINVAL;
3911 }
3912
3913 t->net = pn;
3914 get_task_struct(p);
3915 wake_up_process(p);
3916 wait_for_completion(&t->start_done);
3917
3918 return 0;
3919 }
3920
3921 /*
3922 * Removes a device from the thread if_list.
3923 */
_rem_dev_from_if_list(struct pktgen_thread * t,struct pktgen_dev * pkt_dev)3924 static void _rem_dev_from_if_list(struct pktgen_thread *t,
3925 struct pktgen_dev *pkt_dev)
3926 {
3927 struct list_head *q, *n;
3928 struct pktgen_dev *p;
3929
3930 if_lock(t);
3931 list_for_each_safe(q, n, &t->if_list) {
3932 p = list_entry(q, struct pktgen_dev, list);
3933 if (p == pkt_dev)
3934 list_del_rcu(&p->list);
3935 }
3936 if_unlock(t);
3937 }
3938
pktgen_remove_device(struct pktgen_thread * t,struct pktgen_dev * pkt_dev)3939 static int pktgen_remove_device(struct pktgen_thread *t,
3940 struct pktgen_dev *pkt_dev)
3941 {
3942 pr_debug("remove_device pkt_dev=%p\n", pkt_dev);
3943
3944 if (pkt_dev->running) {
3945 pr_warn("WARNING: trying to remove a running interface, stopping it now\n");
3946 pktgen_stop_device(pkt_dev);
3947 }
3948
3949 /* Dis-associate from the interface */
3950
3951 if (pkt_dev->odev) {
3952 netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker);
3953 pkt_dev->odev = NULL;
3954 }
3955
3956 /* Remove proc before if_list entry, because add_device uses
3957 * list to determine if interface already exist, avoid race
3958 * with proc_create_data() */
3959 proc_remove(pkt_dev->entry);
3960
3961 /* And update the thread if_list */
3962 _rem_dev_from_if_list(t, pkt_dev);
3963
3964 #ifdef CONFIG_XFRM
3965 free_SAs(pkt_dev);
3966 #endif
3967 vfree(pkt_dev->flows);
3968 if (pkt_dev->page)
3969 put_page(pkt_dev->page);
3970 kfree_rcu(pkt_dev, rcu);
3971 return 0;
3972 }
3973
pg_net_init(struct net * net)3974 static int __net_init pg_net_init(struct net *net)
3975 {
3976 struct pktgen_net *pn = net_generic(net, pg_net_id);
3977 struct proc_dir_entry *pe;
3978 int cpu, ret = 0;
3979
3980 pn->net = net;
3981 INIT_LIST_HEAD(&pn->pktgen_threads);
3982 pn->pktgen_exiting = false;
3983 pn->proc_dir = proc_mkdir(PG_PROC_DIR, pn->net->proc_net);
3984 if (!pn->proc_dir) {
3985 pr_warn("cannot create /proc/net/%s\n", PG_PROC_DIR);
3986 return -ENODEV;
3987 }
3988 pe = proc_create(PGCTRL, 0600, pn->proc_dir, &pktgen_proc_ops);
3989 if (pe == NULL) {
3990 pr_err("cannot create %s procfs entry\n", PGCTRL);
3991 ret = -EINVAL;
3992 goto remove;
3993 }
3994
3995 cpus_read_lock();
3996 for_each_online_cpu(cpu) {
3997 int err;
3998
3999 err = pktgen_create_thread(cpu, pn);
4000 if (err)
4001 pr_warn("Cannot create thread for cpu %d (%d)\n",
4002 cpu, err);
4003 }
4004 cpus_read_unlock();
4005
4006 if (list_empty(&pn->pktgen_threads)) {
4007 pr_err("Initialization failed for all threads\n");
4008 ret = -ENODEV;
4009 goto remove_entry;
4010 }
4011
4012 return 0;
4013
4014 remove_entry:
4015 remove_proc_entry(PGCTRL, pn->proc_dir);
4016 remove:
4017 remove_proc_entry(PG_PROC_DIR, pn->net->proc_net);
4018 return ret;
4019 }
4020
pg_net_exit(struct net * net)4021 static void __net_exit pg_net_exit(struct net *net)
4022 {
4023 struct pktgen_net *pn = net_generic(net, pg_net_id);
4024 struct pktgen_thread *t;
4025 struct list_head *q, *n;
4026 LIST_HEAD(list);
4027
4028 /* Stop all interfaces & threads */
4029 pn->pktgen_exiting = true;
4030
4031 mutex_lock(&pktgen_thread_lock);
4032 list_splice_init(&pn->pktgen_threads, &list);
4033 mutex_unlock(&pktgen_thread_lock);
4034
4035 list_for_each_safe(q, n, &list) {
4036 t = list_entry(q, struct pktgen_thread, th_list);
4037 list_del(&t->th_list);
4038 kthread_stop_put(t->tsk);
4039 kfree(t);
4040 }
4041
4042 remove_proc_entry(PGCTRL, pn->proc_dir);
4043 remove_proc_entry(PG_PROC_DIR, pn->net->proc_net);
4044 }
4045
4046 static struct pernet_operations pg_net_ops = {
4047 .init = pg_net_init,
4048 .exit = pg_net_exit,
4049 .id = &pg_net_id,
4050 .size = sizeof(struct pktgen_net),
4051 };
4052
pg_init(void)4053 static int __init pg_init(void)
4054 {
4055 int ret = 0;
4056
4057 pr_info("%s", version);
4058 ret = register_pernet_subsys(&pg_net_ops);
4059 if (ret)
4060 return ret;
4061 ret = register_netdevice_notifier(&pktgen_notifier_block);
4062 if (ret)
4063 unregister_pernet_subsys(&pg_net_ops);
4064
4065 return ret;
4066 }
4067
pg_cleanup(void)4068 static void __exit pg_cleanup(void)
4069 {
4070 unregister_netdevice_notifier(&pktgen_notifier_block);
4071 unregister_pernet_subsys(&pg_net_ops);
4072 /* Don't need rcu_barrier() due to use of kfree_rcu() */
4073 }
4074
4075 module_init(pg_init);
4076 module_exit(pg_cleanup);
4077
4078 MODULE_AUTHOR("Robert Olsson <robert.olsson@its.uu.se>");
4079 MODULE_DESCRIPTION("Packet Generator tool");
4080 MODULE_LICENSE("GPL");
4081 MODULE_VERSION(VERSION);
4082 module_param(pg_count_d, int, 0);
4083 MODULE_PARM_DESC(pg_count_d, "Default number of packets to inject");
4084 module_param(pg_delay_d, int, 0);
4085 MODULE_PARM_DESC(pg_delay_d, "Default delay between packets (nanoseconds)");
4086 module_param(pg_clone_skb_d, int, 0);
4087 MODULE_PARM_DESC(pg_clone_skb_d, "Default number of copies of the same packet");
4088 module_param(debug, int, 0);
4089 MODULE_PARM_DESC(debug, "Enable debugging of pktgen module");
4090