1 /*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * The User Datagram Protocol (UDP).
7 *
8 * Authors: Ross Biro
9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
11 * Alan Cox, <alan@lxorguk.ukuu.org.uk>
12 * Hirokazu Takahashi, <taka@valinux.co.jp>
13 *
14 * Fixes:
15 * Alan Cox : verify_area() calls
16 * Alan Cox : stopped close while in use off icmp
17 * messages. Not a fix but a botch that
18 * for udp at least is 'valid'.
19 * Alan Cox : Fixed icmp handling properly
20 * Alan Cox : Correct error for oversized datagrams
21 * Alan Cox : Tidied select() semantics.
22 * Alan Cox : udp_err() fixed properly, also now
23 * select and read wake correctly on errors
24 * Alan Cox : udp_send verify_area moved to avoid mem leak
25 * Alan Cox : UDP can count its memory
26 * Alan Cox : send to an unknown connection causes
27 * an ECONNREFUSED off the icmp, but
28 * does NOT close.
29 * Alan Cox : Switched to new sk_buff handlers. No more backlog!
30 * Alan Cox : Using generic datagram code. Even smaller and the PEEK
31 * bug no longer crashes it.
32 * Fred Van Kempen : Net2e support for sk->broadcast.
33 * Alan Cox : Uses skb_free_datagram
34 * Alan Cox : Added get/set sockopt support.
35 * Alan Cox : Broadcasting without option set returns EACCES.
36 * Alan Cox : No wakeup calls. Instead we now use the callbacks.
37 * Alan Cox : Use ip_tos and ip_ttl
38 * Alan Cox : SNMP Mibs
39 * Alan Cox : MSG_DONTROUTE, and 0.0.0.0 support.
40 * Matt Dillon : UDP length checks.
41 * Alan Cox : Smarter af_inet used properly.
42 * Alan Cox : Use new kernel side addressing.
43 * Alan Cox : Incorrect return on truncated datagram receive.
44 * Arnt Gulbrandsen : New udp_send and stuff
45 * Alan Cox : Cache last socket
46 * Alan Cox : Route cache
47 * Jon Peatfield : Minor efficiency fix to sendto().
48 * Mike Shaver : RFC1122 checks.
49 * Alan Cox : Nonblocking error fix.
50 * Willy Konynenberg : Transparent proxying support.
51 * Mike McLagan : Routing by source
52 * David S. Miller : New socket lookup architecture.
53 * Last socket cache retained as it
54 * does have a high hit rate.
55 * Olaf Kirch : Don't linearise iovec on sendmsg.
56 * Andi Kleen : Some cleanups, cache destination entry
57 * for connect.
58 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
59 * Melvin Smith : Check msg_name not msg_namelen in sendto(),
60 * return ENOTCONN for unconnected sockets (POSIX)
61 * Janos Farkas : don't deliver multi/broadcasts to a different
62 * bound-to-device socket
63 * Hirokazu Takahashi : HW checksumming for outgoing UDP
64 * datagrams.
65 * Hirokazu Takahashi : sendfile() on UDP works now.
66 * Arnaldo C. Melo : convert /proc/net/udp to seq_file
67 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
68 * Alexey Kuznetsov: allow both IPv4 and IPv6 sockets to bind
69 * a single port at the same time.
70 * Derek Atkins <derek@ihtfp.com>: Add Encapulation Support
71 * James Chapman : Add L2TP encapsulation type.
72 *
73 *
74 * This program is free software; you can redistribute it and/or
75 * modify it under the terms of the GNU General Public License
76 * as published by the Free Software Foundation; either version
77 * 2 of the License, or (at your option) any later version.
78 */
79
80 #define pr_fmt(fmt) "UDP: " fmt
81
82 #include <asm/uaccess.h>
83 #include <asm/ioctls.h>
84 #include <linux/bootmem.h>
85 #include <linux/highmem.h>
86 #include <linux/swap.h>
87 #include <linux/types.h>
88 #include <linux/fcntl.h>
89 #include <linux/module.h>
90 #include <linux/socket.h>
91 #include <linux/sockios.h>
92 #include <linux/igmp.h>
93 #include <linux/inetdevice.h>
94 #include <linux/in.h>
95 #include <linux/errno.h>
96 #include <linux/timer.h>
97 #include <linux/mm.h>
98 #include <linux/inet.h>
99 #include <linux/netdevice.h>
100 #include <linux/slab.h>
101 #include <net/tcp_states.h>
102 #include <linux/skbuff.h>
103 #include <linux/netdevice.h>
104 #include <linux/proc_fs.h>
105 #include <linux/seq_file.h>
106 #include <net/net_namespace.h>
107 #include <net/icmp.h>
108 #include <net/inet_hashtables.h>
109 #include <net/route.h>
110 #include <net/checksum.h>
111 #include <net/xfrm.h>
112 #include <trace/events/udp.h>
113 #include <linux/static_key.h>
114 #include <trace/events/skb.h>
115 #include <net/busy_poll.h>
116 #include "udp_impl.h"
117
118 struct udp_table udp_table __read_mostly;
119 EXPORT_SYMBOL(udp_table);
120
121 long sysctl_udp_mem[3] __read_mostly;
122 EXPORT_SYMBOL(sysctl_udp_mem);
123
124 int sysctl_udp_rmem_min __read_mostly;
125 EXPORT_SYMBOL(sysctl_udp_rmem_min);
126
127 int sysctl_udp_wmem_min __read_mostly;
128 EXPORT_SYMBOL(sysctl_udp_wmem_min);
129
130 atomic_long_t udp_memory_allocated;
131 EXPORT_SYMBOL(udp_memory_allocated);
132
133 #define MAX_UDP_PORTS 65536
134 #define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN)
135
udp_lib_lport_inuse(struct net * net,__u16 num,const struct udp_hslot * hslot,unsigned long * bitmap,struct sock * sk,int (* saddr_comp)(const struct sock * sk1,const struct sock * sk2),unsigned int log)136 static int udp_lib_lport_inuse(struct net *net, __u16 num,
137 const struct udp_hslot *hslot,
138 unsigned long *bitmap,
139 struct sock *sk,
140 int (*saddr_comp)(const struct sock *sk1,
141 const struct sock *sk2),
142 unsigned int log)
143 {
144 struct sock *sk2;
145 struct hlist_nulls_node *node;
146 kuid_t uid = sock_i_uid(sk);
147
148 sk_nulls_for_each(sk2, node, &hslot->head)
149 if (net_eq(sock_net(sk2), net) &&
150 sk2 != sk &&
151 (bitmap || udp_sk(sk2)->udp_port_hash == num) &&
152 (!sk2->sk_reuse || !sk->sk_reuse) &&
153 (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
154 sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
155 (!sk2->sk_reuseport || !sk->sk_reuseport ||
156 !uid_eq(uid, sock_i_uid(sk2))) &&
157 (*saddr_comp)(sk, sk2)) {
158 if (bitmap)
159 __set_bit(udp_sk(sk2)->udp_port_hash >> log,
160 bitmap);
161 else
162 return 1;
163 }
164 return 0;
165 }
166
167 /*
168 * Note: we still hold spinlock of primary hash chain, so no other writer
169 * can insert/delete a socket with local_port == num
170 */
udp_lib_lport_inuse2(struct net * net,__u16 num,struct udp_hslot * hslot2,struct sock * sk,int (* saddr_comp)(const struct sock * sk1,const struct sock * sk2))171 static int udp_lib_lport_inuse2(struct net *net, __u16 num,
172 struct udp_hslot *hslot2,
173 struct sock *sk,
174 int (*saddr_comp)(const struct sock *sk1,
175 const struct sock *sk2))
176 {
177 struct sock *sk2;
178 struct hlist_nulls_node *node;
179 kuid_t uid = sock_i_uid(sk);
180 int res = 0;
181
182 spin_lock(&hslot2->lock);
183 udp_portaddr_for_each_entry(sk2, node, &hslot2->head)
184 if (net_eq(sock_net(sk2), net) &&
185 sk2 != sk &&
186 (udp_sk(sk2)->udp_port_hash == num) &&
187 (!sk2->sk_reuse || !sk->sk_reuse) &&
188 (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
189 sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
190 (!sk2->sk_reuseport || !sk->sk_reuseport ||
191 !uid_eq(uid, sock_i_uid(sk2))) &&
192 (*saddr_comp)(sk, sk2)) {
193 res = 1;
194 break;
195 }
196 spin_unlock(&hslot2->lock);
197 return res;
198 }
199
200 /**
201 * udp_lib_get_port - UDP/-Lite port lookup for IPv4 and IPv6
202 *
203 * @sk: socket struct in question
204 * @snum: port number to look up
205 * @saddr_comp: AF-dependent comparison of bound local IP addresses
206 * @hash2_nulladdr: AF-dependent hash value in secondary hash chains,
207 * with NULL address
208 */
udp_lib_get_port(struct sock * sk,unsigned short snum,int (* saddr_comp)(const struct sock * sk1,const struct sock * sk2),unsigned int hash2_nulladdr)209 int udp_lib_get_port(struct sock *sk, unsigned short snum,
210 int (*saddr_comp)(const struct sock *sk1,
211 const struct sock *sk2),
212 unsigned int hash2_nulladdr)
213 {
214 struct udp_hslot *hslot, *hslot2;
215 struct udp_table *udptable = sk->sk_prot->h.udp_table;
216 int error = 1;
217 struct net *net = sock_net(sk);
218
219 if (!snum) {
220 int low, high, remaining;
221 unsigned int rand;
222 unsigned short first, last;
223 DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN);
224
225 inet_get_local_port_range(net, &low, &high);
226 remaining = (high - low) + 1;
227
228 rand = prandom_u32();
229 first = reciprocal_scale(rand, remaining) + low;
230 /*
231 * force rand to be an odd multiple of UDP_HTABLE_SIZE
232 */
233 rand = (rand | 1) * (udptable->mask + 1);
234 last = first + udptable->mask + 1;
235 do {
236 hslot = udp_hashslot(udptable, net, first);
237 bitmap_zero(bitmap, PORTS_PER_CHAIN);
238 spin_lock_bh(&hslot->lock);
239 udp_lib_lport_inuse(net, snum, hslot, bitmap, sk,
240 saddr_comp, udptable->log);
241
242 snum = first;
243 /*
244 * Iterate on all possible values of snum for this hash.
245 * Using steps of an odd multiple of UDP_HTABLE_SIZE
246 * give us randomization and full range coverage.
247 */
248 do {
249 if (low <= snum && snum <= high &&
250 !test_bit(snum >> udptable->log, bitmap) &&
251 !inet_is_local_reserved_port(net, snum))
252 goto found;
253 snum += rand;
254 } while (snum != first);
255 spin_unlock_bh(&hslot->lock);
256 } while (++first != last);
257 goto fail;
258 } else {
259 hslot = udp_hashslot(udptable, net, snum);
260 spin_lock_bh(&hslot->lock);
261 if (hslot->count > 10) {
262 int exist;
263 unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum;
264
265 slot2 &= udptable->mask;
266 hash2_nulladdr &= udptable->mask;
267
268 hslot2 = udp_hashslot2(udptable, slot2);
269 if (hslot->count < hslot2->count)
270 goto scan_primary_hash;
271
272 exist = udp_lib_lport_inuse2(net, snum, hslot2,
273 sk, saddr_comp);
274 if (!exist && (hash2_nulladdr != slot2)) {
275 hslot2 = udp_hashslot2(udptable, hash2_nulladdr);
276 exist = udp_lib_lport_inuse2(net, snum, hslot2,
277 sk, saddr_comp);
278 }
279 if (exist)
280 goto fail_unlock;
281 else
282 goto found;
283 }
284 scan_primary_hash:
285 if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk,
286 saddr_comp, 0))
287 goto fail_unlock;
288 }
289 found:
290 inet_sk(sk)->inet_num = snum;
291 udp_sk(sk)->udp_port_hash = snum;
292 udp_sk(sk)->udp_portaddr_hash ^= snum;
293 if (sk_unhashed(sk)) {
294 sk_nulls_add_node_rcu(sk, &hslot->head);
295 hslot->count++;
296 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
297
298 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
299 spin_lock(&hslot2->lock);
300 hlist_nulls_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
301 &hslot2->head);
302 hslot2->count++;
303 spin_unlock(&hslot2->lock);
304 }
305 error = 0;
306 fail_unlock:
307 spin_unlock_bh(&hslot->lock);
308 fail:
309 return error;
310 }
311 EXPORT_SYMBOL(udp_lib_get_port);
312
ipv4_rcv_saddr_equal(const struct sock * sk1,const struct sock * sk2)313 static int ipv4_rcv_saddr_equal(const struct sock *sk1, const struct sock *sk2)
314 {
315 struct inet_sock *inet1 = inet_sk(sk1), *inet2 = inet_sk(sk2);
316
317 return (!ipv6_only_sock(sk2) &&
318 (!inet1->inet_rcv_saddr || !inet2->inet_rcv_saddr ||
319 inet1->inet_rcv_saddr == inet2->inet_rcv_saddr));
320 }
321
udp4_portaddr_hash(struct net * net,__be32 saddr,unsigned int port)322 static unsigned int udp4_portaddr_hash(struct net *net, __be32 saddr,
323 unsigned int port)
324 {
325 return jhash_1word((__force u32)saddr, net_hash_mix(net)) ^ port;
326 }
327
udp_v4_get_port(struct sock * sk,unsigned short snum)328 int udp_v4_get_port(struct sock *sk, unsigned short snum)
329 {
330 unsigned int hash2_nulladdr =
331 udp4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum);
332 unsigned int hash2_partial =
333 udp4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0);
334
335 /* precompute partial secondary hash */
336 udp_sk(sk)->udp_portaddr_hash = hash2_partial;
337 return udp_lib_get_port(sk, snum, ipv4_rcv_saddr_equal, hash2_nulladdr);
338 }
339
compute_score(struct sock * sk,struct net * net,__be32 saddr,unsigned short hnum,__be16 sport,__be32 daddr,__be16 dport,int dif)340 static inline int compute_score(struct sock *sk, struct net *net, __be32 saddr,
341 unsigned short hnum,
342 __be16 sport, __be32 daddr, __be16 dport, int dif)
343 {
344 int score = -1;
345
346 if (net_eq(sock_net(sk), net) && udp_sk(sk)->udp_port_hash == hnum &&
347 !ipv6_only_sock(sk)) {
348 struct inet_sock *inet = inet_sk(sk);
349
350 score = (sk->sk_family == PF_INET ? 2 : 1);
351 if (inet->inet_rcv_saddr) {
352 if (inet->inet_rcv_saddr != daddr)
353 return -1;
354 score += 4;
355 }
356 if (inet->inet_daddr) {
357 if (inet->inet_daddr != saddr)
358 return -1;
359 score += 4;
360 }
361 if (inet->inet_dport) {
362 if (inet->inet_dport != sport)
363 return -1;
364 score += 4;
365 }
366 if (sk->sk_bound_dev_if) {
367 if (sk->sk_bound_dev_if != dif)
368 return -1;
369 score += 4;
370 }
371 }
372 return score;
373 }
374
375 /*
376 * In this second variant, we check (daddr, dport) matches (inet_rcv_sadd, inet_num)
377 */
compute_score2(struct sock * sk,struct net * net,__be32 saddr,__be16 sport,__be32 daddr,unsigned int hnum,int dif)378 static inline int compute_score2(struct sock *sk, struct net *net,
379 __be32 saddr, __be16 sport,
380 __be32 daddr, unsigned int hnum, int dif)
381 {
382 int score = -1;
383
384 if (net_eq(sock_net(sk), net) && !ipv6_only_sock(sk)) {
385 struct inet_sock *inet = inet_sk(sk);
386
387 if (inet->inet_rcv_saddr != daddr)
388 return -1;
389 if (inet->inet_num != hnum)
390 return -1;
391
392 score = (sk->sk_family == PF_INET ? 2 : 1);
393 if (inet->inet_daddr) {
394 if (inet->inet_daddr != saddr)
395 return -1;
396 score += 4;
397 }
398 if (inet->inet_dport) {
399 if (inet->inet_dport != sport)
400 return -1;
401 score += 4;
402 }
403 if (sk->sk_bound_dev_if) {
404 if (sk->sk_bound_dev_if != dif)
405 return -1;
406 score += 4;
407 }
408 }
409 return score;
410 }
411
udp_ehashfn(struct net * net,const __be32 laddr,const __u16 lport,const __be32 faddr,const __be16 fport)412 static unsigned int udp_ehashfn(struct net *net, const __be32 laddr,
413 const __u16 lport, const __be32 faddr,
414 const __be16 fport)
415 {
416 static u32 udp_ehash_secret __read_mostly;
417
418 net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret));
419
420 return __inet_ehashfn(laddr, lport, faddr, fport,
421 udp_ehash_secret + net_hash_mix(net));
422 }
423
424
425 /* called with read_rcu_lock() */
udp4_lib_lookup2(struct net * net,__be32 saddr,__be16 sport,__be32 daddr,unsigned int hnum,int dif,struct udp_hslot * hslot2,unsigned int slot2)426 static struct sock *udp4_lib_lookup2(struct net *net,
427 __be32 saddr, __be16 sport,
428 __be32 daddr, unsigned int hnum, int dif,
429 struct udp_hslot *hslot2, unsigned int slot2)
430 {
431 struct sock *sk, *result;
432 struct hlist_nulls_node *node;
433 int score, badness, matches = 0, reuseport = 0;
434 u32 hash = 0;
435
436 begin:
437 result = NULL;
438 badness = 0;
439 udp_portaddr_for_each_entry_rcu(sk, node, &hslot2->head) {
440 score = compute_score2(sk, net, saddr, sport,
441 daddr, hnum, dif);
442 if (score > badness) {
443 result = sk;
444 badness = score;
445 reuseport = sk->sk_reuseport;
446 if (reuseport) {
447 hash = udp_ehashfn(net, daddr, hnum,
448 saddr, sport);
449 matches = 1;
450 }
451 } else if (score == badness && reuseport) {
452 matches++;
453 if (reciprocal_scale(hash, matches) == 0)
454 result = sk;
455 hash = next_pseudo_random32(hash);
456 }
457 }
458 /*
459 * if the nulls value we got at the end of this lookup is
460 * not the expected one, we must restart lookup.
461 * We probably met an item that was moved to another chain.
462 */
463 if (get_nulls_value(node) != slot2)
464 goto begin;
465 if (result) {
466 if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
467 result = NULL;
468 else if (unlikely(compute_score2(result, net, saddr, sport,
469 daddr, hnum, dif) < badness)) {
470 sock_put(result);
471 goto begin;
472 }
473 }
474 return result;
475 }
476
477 /* UDP is nearly always wildcards out the wazoo, it makes no sense to try
478 * harder than this. -DaveM
479 */
__udp4_lib_lookup(struct net * net,__be32 saddr,__be16 sport,__be32 daddr,__be16 dport,int dif,struct udp_table * udptable)480 struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
481 __be16 sport, __be32 daddr, __be16 dport,
482 int dif, struct udp_table *udptable)
483 {
484 struct sock *sk, *result;
485 struct hlist_nulls_node *node;
486 unsigned short hnum = ntohs(dport);
487 unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask);
488 struct udp_hslot *hslot2, *hslot = &udptable->hash[slot];
489 int score, badness, matches = 0, reuseport = 0;
490 u32 hash = 0;
491
492 rcu_read_lock();
493 if (hslot->count > 10) {
494 hash2 = udp4_portaddr_hash(net, daddr, hnum);
495 slot2 = hash2 & udptable->mask;
496 hslot2 = &udptable->hash2[slot2];
497 if (hslot->count < hslot2->count)
498 goto begin;
499
500 result = udp4_lib_lookup2(net, saddr, sport,
501 daddr, hnum, dif,
502 hslot2, slot2);
503 if (!result) {
504 hash2 = udp4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
505 slot2 = hash2 & udptable->mask;
506 hslot2 = &udptable->hash2[slot2];
507 if (hslot->count < hslot2->count)
508 goto begin;
509
510 result = udp4_lib_lookup2(net, saddr, sport,
511 htonl(INADDR_ANY), hnum, dif,
512 hslot2, slot2);
513 }
514 rcu_read_unlock();
515 return result;
516 }
517 begin:
518 result = NULL;
519 badness = 0;
520 sk_nulls_for_each_rcu(sk, node, &hslot->head) {
521 score = compute_score(sk, net, saddr, hnum, sport,
522 daddr, dport, dif);
523 if (score > badness) {
524 result = sk;
525 badness = score;
526 reuseport = sk->sk_reuseport;
527 if (reuseport) {
528 hash = udp_ehashfn(net, daddr, hnum,
529 saddr, sport);
530 matches = 1;
531 }
532 } else if (score == badness && reuseport) {
533 matches++;
534 if (reciprocal_scale(hash, matches) == 0)
535 result = sk;
536 hash = next_pseudo_random32(hash);
537 }
538 }
539 /*
540 * if the nulls value we got at the end of this lookup is
541 * not the expected one, we must restart lookup.
542 * We probably met an item that was moved to another chain.
543 */
544 if (get_nulls_value(node) != slot)
545 goto begin;
546
547 if (result) {
548 if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
549 result = NULL;
550 else if (unlikely(compute_score(result, net, saddr, hnum, sport,
551 daddr, dport, dif) < badness)) {
552 sock_put(result);
553 goto begin;
554 }
555 }
556 rcu_read_unlock();
557 return result;
558 }
559 EXPORT_SYMBOL_GPL(__udp4_lib_lookup);
560
__udp4_lib_lookup_skb(struct sk_buff * skb,__be16 sport,__be16 dport,struct udp_table * udptable)561 static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb,
562 __be16 sport, __be16 dport,
563 struct udp_table *udptable)
564 {
565 const struct iphdr *iph = ip_hdr(skb);
566
567 return __udp4_lib_lookup(dev_net(skb_dst(skb)->dev), iph->saddr, sport,
568 iph->daddr, dport, inet_iif(skb),
569 udptable);
570 }
571
udp4_lib_lookup(struct net * net,__be32 saddr,__be16 sport,__be32 daddr,__be16 dport,int dif)572 struct sock *udp4_lib_lookup(struct net *net, __be32 saddr, __be16 sport,
573 __be32 daddr, __be16 dport, int dif)
574 {
575 return __udp4_lib_lookup(net, saddr, sport, daddr, dport, dif, &udp_table);
576 }
577 EXPORT_SYMBOL_GPL(udp4_lib_lookup);
578
__udp_is_mcast_sock(struct net * net,struct sock * sk,__be16 loc_port,__be32 loc_addr,__be16 rmt_port,__be32 rmt_addr,int dif,unsigned short hnum)579 static inline bool __udp_is_mcast_sock(struct net *net, struct sock *sk,
580 __be16 loc_port, __be32 loc_addr,
581 __be16 rmt_port, __be32 rmt_addr,
582 int dif, unsigned short hnum)
583 {
584 struct inet_sock *inet = inet_sk(sk);
585
586 if (!net_eq(sock_net(sk), net) ||
587 udp_sk(sk)->udp_port_hash != hnum ||
588 (inet->inet_daddr && inet->inet_daddr != rmt_addr) ||
589 (inet->inet_dport != rmt_port && inet->inet_dport) ||
590 (inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) ||
591 ipv6_only_sock(sk) ||
592 (sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif))
593 return false;
594 if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif))
595 return false;
596 return true;
597 }
598
599 /*
600 * This routine is called by the ICMP module when it gets some
601 * sort of error condition. If err < 0 then the socket should
602 * be closed and the error returned to the user. If err > 0
603 * it's just the icmp type << 8 | icmp code.
604 * Header points to the ip header of the error packet. We move
605 * on past this. Then (as it used to claim before adjustment)
606 * header points to the first 8 bytes of the udp header. We need
607 * to find the appropriate port.
608 */
609
__udp4_lib_err(struct sk_buff * skb,u32 info,struct udp_table * udptable)610 void __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable)
611 {
612 struct inet_sock *inet;
613 const struct iphdr *iph = (const struct iphdr *)skb->data;
614 struct udphdr *uh = (struct udphdr *)(skb->data+(iph->ihl<<2));
615 const int type = icmp_hdr(skb)->type;
616 const int code = icmp_hdr(skb)->code;
617 struct sock *sk;
618 int harderr;
619 int err;
620 struct net *net = dev_net(skb->dev);
621
622 sk = __udp4_lib_lookup(net, iph->daddr, uh->dest,
623 iph->saddr, uh->source, skb->dev->ifindex, udptable);
624 if (sk == NULL) {
625 ICMP_INC_STATS_BH(net, ICMP_MIB_INERRORS);
626 return; /* No socket for error */
627 }
628
629 err = 0;
630 harderr = 0;
631 inet = inet_sk(sk);
632
633 switch (type) {
634 default:
635 case ICMP_TIME_EXCEEDED:
636 err = EHOSTUNREACH;
637 break;
638 case ICMP_SOURCE_QUENCH:
639 goto out;
640 case ICMP_PARAMETERPROB:
641 err = EPROTO;
642 harderr = 1;
643 break;
644 case ICMP_DEST_UNREACH:
645 if (code == ICMP_FRAG_NEEDED) { /* Path MTU discovery */
646 ipv4_sk_update_pmtu(skb, sk, info);
647 if (inet->pmtudisc != IP_PMTUDISC_DONT) {
648 err = EMSGSIZE;
649 harderr = 1;
650 break;
651 }
652 goto out;
653 }
654 err = EHOSTUNREACH;
655 if (code <= NR_ICMP_UNREACH) {
656 harderr = icmp_err_convert[code].fatal;
657 err = icmp_err_convert[code].errno;
658 }
659 break;
660 case ICMP_REDIRECT:
661 ipv4_sk_redirect(skb, sk);
662 goto out;
663 }
664
665 /*
666 * RFC1122: OK. Passes ICMP errors back to application, as per
667 * 4.1.3.3.
668 */
669 if (!inet->recverr) {
670 if (!harderr || sk->sk_state != TCP_ESTABLISHED)
671 goto out;
672 } else
673 ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1));
674
675 sk->sk_err = err;
676 sk->sk_error_report(sk);
677 out:
678 sock_put(sk);
679 }
680
udp_err(struct sk_buff * skb,u32 info)681 void udp_err(struct sk_buff *skb, u32 info)
682 {
683 __udp4_lib_err(skb, info, &udp_table);
684 }
685
686 /*
687 * Throw away all pending data and cancel the corking. Socket is locked.
688 */
udp_flush_pending_frames(struct sock * sk)689 void udp_flush_pending_frames(struct sock *sk)
690 {
691 struct udp_sock *up = udp_sk(sk);
692
693 if (up->pending) {
694 up->len = 0;
695 up->pending = 0;
696 ip_flush_pending_frames(sk);
697 }
698 }
699 EXPORT_SYMBOL(udp_flush_pending_frames);
700
701 /**
702 * udp4_hwcsum - handle outgoing HW checksumming
703 * @skb: sk_buff containing the filled-in UDP header
704 * (checksum field must be zeroed out)
705 * @src: source IP address
706 * @dst: destination IP address
707 */
udp4_hwcsum(struct sk_buff * skb,__be32 src,__be32 dst)708 void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst)
709 {
710 struct udphdr *uh = udp_hdr(skb);
711 int offset = skb_transport_offset(skb);
712 int len = skb->len - offset;
713 int hlen = len;
714 __wsum csum = 0;
715
716 if (!skb_has_frag_list(skb)) {
717 /*
718 * Only one fragment on the socket.
719 */
720 skb->csum_start = skb_transport_header(skb) - skb->head;
721 skb->csum_offset = offsetof(struct udphdr, check);
722 uh->check = ~csum_tcpudp_magic(src, dst, len,
723 IPPROTO_UDP, 0);
724 } else {
725 struct sk_buff *frags;
726
727 /*
728 * HW-checksum won't work as there are two or more
729 * fragments on the socket so that all csums of sk_buffs
730 * should be together
731 */
732 skb_walk_frags(skb, frags) {
733 csum = csum_add(csum, frags->csum);
734 hlen -= frags->len;
735 }
736
737 csum = skb_checksum(skb, offset, hlen, csum);
738 skb->ip_summed = CHECKSUM_NONE;
739
740 uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum);
741 if (uh->check == 0)
742 uh->check = CSUM_MANGLED_0;
743 }
744 }
745 EXPORT_SYMBOL_GPL(udp4_hwcsum);
746
747 /* Function to set UDP checksum for an IPv4 UDP packet. This is intended
748 * for the simple case like when setting the checksum for a UDP tunnel.
749 */
udp_set_csum(bool nocheck,struct sk_buff * skb,__be32 saddr,__be32 daddr,int len)750 void udp_set_csum(bool nocheck, struct sk_buff *skb,
751 __be32 saddr, __be32 daddr, int len)
752 {
753 struct udphdr *uh = udp_hdr(skb);
754
755 if (nocheck)
756 uh->check = 0;
757 else if (skb_is_gso(skb))
758 uh->check = ~udp_v4_check(len, saddr, daddr, 0);
759 else if (skb_dst(skb) && skb_dst(skb)->dev &&
760 (skb_dst(skb)->dev->features & NETIF_F_V4_CSUM)) {
761
762 BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL);
763
764 skb->ip_summed = CHECKSUM_PARTIAL;
765 skb->csum_start = skb_transport_header(skb) - skb->head;
766 skb->csum_offset = offsetof(struct udphdr, check);
767 uh->check = ~udp_v4_check(len, saddr, daddr, 0);
768 } else {
769 __wsum csum;
770
771 BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL);
772
773 uh->check = 0;
774 csum = skb_checksum(skb, 0, len, 0);
775 uh->check = udp_v4_check(len, saddr, daddr, csum);
776 if (uh->check == 0)
777 uh->check = CSUM_MANGLED_0;
778
779 skb->ip_summed = CHECKSUM_UNNECESSARY;
780 }
781 }
782 EXPORT_SYMBOL(udp_set_csum);
783
udp_send_skb(struct sk_buff * skb,struct flowi4 * fl4)784 static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4)
785 {
786 struct sock *sk = skb->sk;
787 struct inet_sock *inet = inet_sk(sk);
788 struct udphdr *uh;
789 int err = 0;
790 int is_udplite = IS_UDPLITE(sk);
791 int offset = skb_transport_offset(skb);
792 int len = skb->len - offset;
793 __wsum csum = 0;
794
795 /*
796 * Create a UDP header
797 */
798 uh = udp_hdr(skb);
799 uh->source = inet->inet_sport;
800 uh->dest = fl4->fl4_dport;
801 uh->len = htons(len);
802 uh->check = 0;
803
804 if (is_udplite) /* UDP-Lite */
805 csum = udplite_csum(skb);
806
807 else if (sk->sk_no_check_tx && !skb_is_gso(skb)) { /* UDP csum off */
808
809 skb->ip_summed = CHECKSUM_NONE;
810 goto send;
811
812 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
813
814 udp4_hwcsum(skb, fl4->saddr, fl4->daddr);
815 goto send;
816
817 } else
818 csum = udp_csum(skb);
819
820 /* add protocol-dependent pseudo-header */
821 uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len,
822 sk->sk_protocol, csum);
823 if (uh->check == 0)
824 uh->check = CSUM_MANGLED_0;
825
826 send:
827 err = ip_send_skb(sock_net(sk), skb);
828 if (err) {
829 if (err == -ENOBUFS && !inet->recverr) {
830 UDP_INC_STATS_USER(sock_net(sk),
831 UDP_MIB_SNDBUFERRORS, is_udplite);
832 err = 0;
833 }
834 } else
835 UDP_INC_STATS_USER(sock_net(sk),
836 UDP_MIB_OUTDATAGRAMS, is_udplite);
837 return err;
838 }
839
840 /*
841 * Push out all pending data as one UDP datagram. Socket is locked.
842 */
udp_push_pending_frames(struct sock * sk)843 int udp_push_pending_frames(struct sock *sk)
844 {
845 struct udp_sock *up = udp_sk(sk);
846 struct inet_sock *inet = inet_sk(sk);
847 struct flowi4 *fl4 = &inet->cork.fl.u.ip4;
848 struct sk_buff *skb;
849 int err = 0;
850
851 skb = ip_finish_skb(sk, fl4);
852 if (!skb)
853 goto out;
854
855 err = udp_send_skb(skb, fl4);
856
857 out:
858 up->len = 0;
859 up->pending = 0;
860 return err;
861 }
862 EXPORT_SYMBOL(udp_push_pending_frames);
863
udp_sendmsg(struct kiocb * iocb,struct sock * sk,struct msghdr * msg,size_t len)864 int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
865 size_t len)
866 {
867 struct inet_sock *inet = inet_sk(sk);
868 struct udp_sock *up = udp_sk(sk);
869 struct flowi4 fl4_stack;
870 struct flowi4 *fl4;
871 int ulen = len;
872 struct ipcm_cookie ipc;
873 struct rtable *rt = NULL;
874 int free = 0;
875 int connected = 0;
876 __be32 daddr, faddr, saddr;
877 __be16 dport;
878 u8 tos;
879 int err, is_udplite = IS_UDPLITE(sk);
880 int corkreq = up->corkflag || msg->msg_flags&MSG_MORE;
881 int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);
882 struct sk_buff *skb;
883 struct ip_options_data opt_copy;
884
885 if (len > 0xFFFF)
886 return -EMSGSIZE;
887
888 /*
889 * Check the flags.
890 */
891
892 if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */
893 return -EOPNOTSUPP;
894
895 ipc.opt = NULL;
896 ipc.tx_flags = 0;
897 ipc.ttl = 0;
898 ipc.tos = -1;
899
900 getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
901
902 fl4 = &inet->cork.fl.u.ip4;
903 if (up->pending) {
904 /*
905 * There are pending frames.
906 * The socket lock must be held while it's corked.
907 */
908 lock_sock(sk);
909 if (likely(up->pending)) {
910 if (unlikely(up->pending != AF_INET)) {
911 release_sock(sk);
912 return -EINVAL;
913 }
914 goto do_append_data;
915 }
916 release_sock(sk);
917 }
918 ulen += sizeof(struct udphdr);
919
920 /*
921 * Get and verify the address.
922 */
923 if (msg->msg_name) {
924 DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name);
925 if (msg->msg_namelen < sizeof(*usin))
926 return -EINVAL;
927 if (usin->sin_family != AF_INET) {
928 if (usin->sin_family != AF_UNSPEC)
929 return -EAFNOSUPPORT;
930 }
931
932 daddr = usin->sin_addr.s_addr;
933 dport = usin->sin_port;
934 if (dport == 0)
935 return -EINVAL;
936 } else {
937 if (sk->sk_state != TCP_ESTABLISHED)
938 return -EDESTADDRREQ;
939 daddr = inet->inet_daddr;
940 dport = inet->inet_dport;
941 /* Open fast path for connected socket.
942 Route will not be used, if at least one option is set.
943 */
944 connected = 1;
945 }
946 ipc.addr = inet->inet_saddr;
947
948 ipc.oif = sk->sk_bound_dev_if;
949
950 sock_tx_timestamp(sk, &ipc.tx_flags);
951
952 if (msg->msg_controllen) {
953 err = ip_cmsg_send(sock_net(sk), msg, &ipc,
954 sk->sk_family == AF_INET6);
955 if (err)
956 return err;
957 if (ipc.opt)
958 free = 1;
959 connected = 0;
960 }
961 if (!ipc.opt) {
962 struct ip_options_rcu *inet_opt;
963
964 rcu_read_lock();
965 inet_opt = rcu_dereference(inet->inet_opt);
966 if (inet_opt) {
967 memcpy(&opt_copy, inet_opt,
968 sizeof(*inet_opt) + inet_opt->opt.optlen);
969 ipc.opt = &opt_copy.opt;
970 }
971 rcu_read_unlock();
972 }
973
974 saddr = ipc.addr;
975 ipc.addr = faddr = daddr;
976
977 if (ipc.opt && ipc.opt->opt.srr) {
978 if (!daddr)
979 return -EINVAL;
980 faddr = ipc.opt->opt.faddr;
981 connected = 0;
982 }
983 tos = get_rttos(&ipc, inet);
984 if (sock_flag(sk, SOCK_LOCALROUTE) ||
985 (msg->msg_flags & MSG_DONTROUTE) ||
986 (ipc.opt && ipc.opt->opt.is_strictroute)) {
987 tos |= RTO_ONLINK;
988 connected = 0;
989 }
990
991 if (ipv4_is_multicast(daddr)) {
992 if (!ipc.oif)
993 ipc.oif = inet->mc_index;
994 if (!saddr)
995 saddr = inet->mc_addr;
996 connected = 0;
997 } else if (!ipc.oif)
998 ipc.oif = inet->uc_index;
999
1000 if (connected)
1001 rt = (struct rtable *)sk_dst_check(sk, 0);
1002
1003 if (rt == NULL) {
1004 struct net *net = sock_net(sk);
1005
1006 fl4 = &fl4_stack;
1007 flowi4_init_output(fl4, ipc.oif, sk->sk_mark, tos,
1008 RT_SCOPE_UNIVERSE, sk->sk_protocol,
1009 inet_sk_flowi_flags(sk),
1010 faddr, saddr, dport, inet->inet_sport,
1011 sk->sk_uid);
1012
1013 security_sk_classify_flow(sk, flowi4_to_flowi(fl4));
1014 rt = ip_route_output_flow(net, fl4, sk);
1015 if (IS_ERR(rt)) {
1016 err = PTR_ERR(rt);
1017 rt = NULL;
1018 if (err == -ENETUNREACH)
1019 IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
1020 goto out;
1021 }
1022
1023 err = -EACCES;
1024 if ((rt->rt_flags & RTCF_BROADCAST) &&
1025 !sock_flag(sk, SOCK_BROADCAST))
1026 goto out;
1027 if (connected)
1028 sk_dst_set(sk, dst_clone(&rt->dst));
1029 }
1030
1031 if (msg->msg_flags&MSG_CONFIRM)
1032 goto do_confirm;
1033 back_from_confirm:
1034
1035 saddr = fl4->saddr;
1036 if (!ipc.addr)
1037 daddr = ipc.addr = fl4->daddr;
1038
1039 /* Lockless fast path for the non-corking case. */
1040 if (!corkreq) {
1041 skb = ip_make_skb(sk, fl4, getfrag, msg->msg_iov, ulen,
1042 sizeof(struct udphdr), &ipc, &rt,
1043 msg->msg_flags);
1044 err = PTR_ERR(skb);
1045 if (!IS_ERR_OR_NULL(skb))
1046 err = udp_send_skb(skb, fl4);
1047 goto out;
1048 }
1049
1050 lock_sock(sk);
1051 if (unlikely(up->pending)) {
1052 /* The socket is already corked while preparing it. */
1053 /* ... which is an evident application bug. --ANK */
1054 release_sock(sk);
1055
1056 LIMIT_NETDEBUG(KERN_DEBUG pr_fmt("cork app bug 2\n"));
1057 err = -EINVAL;
1058 goto out;
1059 }
1060 /*
1061 * Now cork the socket to pend data.
1062 */
1063 fl4 = &inet->cork.fl.u.ip4;
1064 fl4->daddr = daddr;
1065 fl4->saddr = saddr;
1066 fl4->fl4_dport = dport;
1067 fl4->fl4_sport = inet->inet_sport;
1068 up->pending = AF_INET;
1069
1070 do_append_data:
1071 up->len += ulen;
1072 err = ip_append_data(sk, fl4, getfrag, msg->msg_iov, ulen,
1073 sizeof(struct udphdr), &ipc, &rt,
1074 corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
1075 if (err)
1076 udp_flush_pending_frames(sk);
1077 else if (!corkreq)
1078 err = udp_push_pending_frames(sk);
1079 else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))
1080 up->pending = 0;
1081 release_sock(sk);
1082
1083 out:
1084 ip_rt_put(rt);
1085 if (free)
1086 kfree(ipc.opt);
1087 if (!err)
1088 return len;
1089 /*
1090 * ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space. Reporting
1091 * ENOBUFS might not be good (it's not tunable per se), but otherwise
1092 * we don't have a good statistic (IpOutDiscards but it can be too many
1093 * things). We could add another new stat but at least for now that
1094 * seems like overkill.
1095 */
1096 if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
1097 UDP_INC_STATS_USER(sock_net(sk),
1098 UDP_MIB_SNDBUFERRORS, is_udplite);
1099 }
1100 return err;
1101
1102 do_confirm:
1103 dst_confirm(&rt->dst);
1104 if (!(msg->msg_flags&MSG_PROBE) || len)
1105 goto back_from_confirm;
1106 err = 0;
1107 goto out;
1108 }
1109 EXPORT_SYMBOL(udp_sendmsg);
1110
udp_sendpage(struct sock * sk,struct page * page,int offset,size_t size,int flags)1111 int udp_sendpage(struct sock *sk, struct page *page, int offset,
1112 size_t size, int flags)
1113 {
1114 struct inet_sock *inet = inet_sk(sk);
1115 struct udp_sock *up = udp_sk(sk);
1116 int ret;
1117
1118 if (flags & MSG_SENDPAGE_NOTLAST)
1119 flags |= MSG_MORE;
1120
1121 if (!up->pending) {
1122 struct msghdr msg = { .msg_flags = flags|MSG_MORE };
1123
1124 /* Call udp_sendmsg to specify destination address which
1125 * sendpage interface can't pass.
1126 * This will succeed only when the socket is connected.
1127 */
1128 ret = udp_sendmsg(NULL, sk, &msg, 0);
1129 if (ret < 0)
1130 return ret;
1131 }
1132
1133 lock_sock(sk);
1134
1135 if (unlikely(!up->pending)) {
1136 release_sock(sk);
1137
1138 LIMIT_NETDEBUG(KERN_DEBUG pr_fmt("udp cork app bug 3\n"));
1139 return -EINVAL;
1140 }
1141
1142 ret = ip_append_page(sk, &inet->cork.fl.u.ip4,
1143 page, offset, size, flags);
1144 if (ret == -EOPNOTSUPP) {
1145 release_sock(sk);
1146 return sock_no_sendpage(sk->sk_socket, page, offset,
1147 size, flags);
1148 }
1149 if (ret < 0) {
1150 udp_flush_pending_frames(sk);
1151 goto out;
1152 }
1153
1154 up->len += size;
1155 if (!(up->corkflag || (flags&MSG_MORE)))
1156 ret = udp_push_pending_frames(sk);
1157 if (!ret)
1158 ret = size;
1159 out:
1160 release_sock(sk);
1161 return ret;
1162 }
1163
1164
1165 /**
1166 * first_packet_length - return length of first packet in receive queue
1167 * @sk: socket
1168 *
1169 * Drops all bad checksum frames, until a valid one is found.
1170 * Returns the length of found skb, or 0 if none is found.
1171 */
first_packet_length(struct sock * sk)1172 static unsigned int first_packet_length(struct sock *sk)
1173 {
1174 struct sk_buff_head list_kill, *rcvq = &sk->sk_receive_queue;
1175 struct sk_buff *skb;
1176 unsigned int res;
1177
1178 __skb_queue_head_init(&list_kill);
1179
1180 spin_lock_bh(&rcvq->lock);
1181 while ((skb = skb_peek(rcvq)) != NULL &&
1182 udp_lib_checksum_complete(skb)) {
1183 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_CSUMERRORS,
1184 IS_UDPLITE(sk));
1185 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,
1186 IS_UDPLITE(sk));
1187 atomic_inc(&sk->sk_drops);
1188 __skb_unlink(skb, rcvq);
1189 __skb_queue_tail(&list_kill, skb);
1190 }
1191 res = skb ? skb->len : 0;
1192 spin_unlock_bh(&rcvq->lock);
1193
1194 if (!skb_queue_empty(&list_kill)) {
1195 bool slow = lock_sock_fast(sk);
1196
1197 __skb_queue_purge(&list_kill);
1198 sk_mem_reclaim_partial(sk);
1199 unlock_sock_fast(sk, slow);
1200 }
1201 return res;
1202 }
1203
1204 /*
1205 * IOCTL requests applicable to the UDP protocol
1206 */
1207
udp_ioctl(struct sock * sk,int cmd,unsigned long arg)1208 int udp_ioctl(struct sock *sk, int cmd, unsigned long arg)
1209 {
1210 switch (cmd) {
1211 case SIOCOUTQ:
1212 {
1213 int amount = sk_wmem_alloc_get(sk);
1214
1215 return put_user(amount, (int __user *)arg);
1216 }
1217
1218 case SIOCINQ:
1219 {
1220 unsigned int amount = first_packet_length(sk);
1221
1222 if (amount)
1223 /*
1224 * We will only return the amount
1225 * of this packet since that is all
1226 * that will be read.
1227 */
1228 amount -= sizeof(struct udphdr);
1229
1230 return put_user(amount, (int __user *)arg);
1231 }
1232
1233 default:
1234 return -ENOIOCTLCMD;
1235 }
1236
1237 return 0;
1238 }
1239 EXPORT_SYMBOL(udp_ioctl);
1240
1241 /*
1242 * This should be easy, if there is something there we
1243 * return it, otherwise we block.
1244 */
1245
udp_recvmsg(struct kiocb * iocb,struct sock * sk,struct msghdr * msg,size_t len,int noblock,int flags,int * addr_len)1246 int udp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
1247 size_t len, int noblock, int flags, int *addr_len)
1248 {
1249 struct inet_sock *inet = inet_sk(sk);
1250 DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name);
1251 struct sk_buff *skb;
1252 unsigned int ulen, copied;
1253 int peeked, off = 0;
1254 int err;
1255 int is_udplite = IS_UDPLITE(sk);
1256 bool checksum_valid = false;
1257 bool slow;
1258
1259 if (flags & MSG_ERRQUEUE)
1260 return ip_recv_error(sk, msg, len, addr_len);
1261
1262 try_again:
1263 skb = __skb_recv_datagram(sk, flags | (noblock ? MSG_DONTWAIT : 0),
1264 &peeked, &off, &err);
1265 if (!skb)
1266 goto out;
1267
1268 ulen = skb->len - sizeof(struct udphdr);
1269 copied = len;
1270 if (copied > ulen)
1271 copied = ulen;
1272 else if (copied < ulen)
1273 msg->msg_flags |= MSG_TRUNC;
1274
1275 /*
1276 * If checksum is needed at all, try to do it while copying the
1277 * data. If the data is truncated, or if we only want a partial
1278 * coverage checksum (UDP-Lite), do it before the copy.
1279 */
1280
1281 if (copied < ulen || UDP_SKB_CB(skb)->partial_cov) {
1282 checksum_valid = !udp_lib_checksum_complete(skb);
1283 if (!checksum_valid)
1284 goto csum_copy_err;
1285 }
1286
1287 if (checksum_valid || skb_csum_unnecessary(skb))
1288 err = skb_copy_datagram_iovec(skb, sizeof(struct udphdr),
1289 msg->msg_iov, copied);
1290 else {
1291 err = skb_copy_and_csum_datagram_iovec(skb,
1292 sizeof(struct udphdr),
1293 msg->msg_iov);
1294
1295 if (err == -EINVAL)
1296 goto csum_copy_err;
1297 }
1298
1299 if (unlikely(err)) {
1300 trace_kfree_skb(skb, udp_recvmsg);
1301 if (!peeked) {
1302 atomic_inc(&sk->sk_drops);
1303 UDP_INC_STATS_USER(sock_net(sk),
1304 UDP_MIB_INERRORS, is_udplite);
1305 }
1306 goto out_free;
1307 }
1308
1309 if (!peeked)
1310 UDP_INC_STATS_USER(sock_net(sk),
1311 UDP_MIB_INDATAGRAMS, is_udplite);
1312
1313 sock_recv_ts_and_drops(msg, sk, skb);
1314
1315 /* Copy the address. */
1316 if (sin) {
1317 sin->sin_family = AF_INET;
1318 sin->sin_port = udp_hdr(skb)->source;
1319 sin->sin_addr.s_addr = ip_hdr(skb)->saddr;
1320 memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
1321 *addr_len = sizeof(*sin);
1322 }
1323 if (inet->cmsg_flags)
1324 ip_cmsg_recv(msg, skb);
1325
1326 err = copied;
1327 if (flags & MSG_TRUNC)
1328 err = ulen;
1329
1330 out_free:
1331 skb_free_datagram_locked(sk, skb);
1332 out:
1333 return err;
1334
1335 csum_copy_err:
1336 slow = lock_sock_fast(sk);
1337 if (!skb_kill_datagram(sk, skb, flags)) {
1338 UDP_INC_STATS_USER(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
1339 UDP_INC_STATS_USER(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
1340 }
1341 unlock_sock_fast(sk, slow);
1342
1343 /* starting over for a new packet, but check if we need to yield */
1344 cond_resched();
1345 msg->msg_flags &= ~MSG_TRUNC;
1346 goto try_again;
1347 }
1348
1349
udp_disconnect(struct sock * sk,int flags)1350 int udp_disconnect(struct sock *sk, int flags)
1351 {
1352 struct inet_sock *inet = inet_sk(sk);
1353 /*
1354 * 1003.1g - break association.
1355 */
1356
1357 sk->sk_state = TCP_CLOSE;
1358 inet->inet_daddr = 0;
1359 inet->inet_dport = 0;
1360 sock_rps_reset_rxhash(sk);
1361 sk->sk_bound_dev_if = 0;
1362 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
1363 inet_reset_saddr(sk);
1364
1365 if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) {
1366 sk->sk_prot->unhash(sk);
1367 inet->inet_sport = 0;
1368 }
1369 sk_dst_reset(sk);
1370 return 0;
1371 }
1372 EXPORT_SYMBOL(udp_disconnect);
1373
udp_lib_unhash(struct sock * sk)1374 void udp_lib_unhash(struct sock *sk)
1375 {
1376 if (sk_hashed(sk)) {
1377 struct udp_table *udptable = sk->sk_prot->h.udp_table;
1378 struct udp_hslot *hslot, *hslot2;
1379
1380 hslot = udp_hashslot(udptable, sock_net(sk),
1381 udp_sk(sk)->udp_port_hash);
1382 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
1383
1384 spin_lock_bh(&hslot->lock);
1385 if (sk_nulls_del_node_init_rcu(sk)) {
1386 hslot->count--;
1387 inet_sk(sk)->inet_num = 0;
1388 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
1389
1390 spin_lock(&hslot2->lock);
1391 hlist_nulls_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
1392 hslot2->count--;
1393 spin_unlock(&hslot2->lock);
1394 }
1395 spin_unlock_bh(&hslot->lock);
1396 }
1397 }
1398 EXPORT_SYMBOL(udp_lib_unhash);
1399
1400 /*
1401 * inet_rcv_saddr was changed, we must rehash secondary hash
1402 */
udp_lib_rehash(struct sock * sk,u16 newhash)1403 void udp_lib_rehash(struct sock *sk, u16 newhash)
1404 {
1405 if (sk_hashed(sk)) {
1406 struct udp_table *udptable = sk->sk_prot->h.udp_table;
1407 struct udp_hslot *hslot, *hslot2, *nhslot2;
1408
1409 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
1410 nhslot2 = udp_hashslot2(udptable, newhash);
1411 udp_sk(sk)->udp_portaddr_hash = newhash;
1412 if (hslot2 != nhslot2) {
1413 hslot = udp_hashslot(udptable, sock_net(sk),
1414 udp_sk(sk)->udp_port_hash);
1415 /* we must lock primary chain too */
1416 spin_lock_bh(&hslot->lock);
1417
1418 spin_lock(&hslot2->lock);
1419 hlist_nulls_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
1420 hslot2->count--;
1421 spin_unlock(&hslot2->lock);
1422
1423 spin_lock(&nhslot2->lock);
1424 hlist_nulls_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
1425 &nhslot2->head);
1426 nhslot2->count++;
1427 spin_unlock(&nhslot2->lock);
1428
1429 spin_unlock_bh(&hslot->lock);
1430 }
1431 }
1432 }
1433 EXPORT_SYMBOL(udp_lib_rehash);
1434
udp_v4_rehash(struct sock * sk)1435 static void udp_v4_rehash(struct sock *sk)
1436 {
1437 u16 new_hash = udp4_portaddr_hash(sock_net(sk),
1438 inet_sk(sk)->inet_rcv_saddr,
1439 inet_sk(sk)->inet_num);
1440 udp_lib_rehash(sk, new_hash);
1441 }
1442
__udp_queue_rcv_skb(struct sock * sk,struct sk_buff * skb)1443 static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
1444 {
1445 int rc;
1446
1447 if (inet_sk(sk)->inet_daddr) {
1448 sock_rps_save_rxhash(sk, skb);
1449 sk_mark_napi_id(sk, skb);
1450 }
1451
1452 rc = sock_queue_rcv_skb(sk, skb);
1453 if (rc < 0) {
1454 int is_udplite = IS_UDPLITE(sk);
1455
1456 /* Note that an ENOMEM error is charged twice */
1457 if (rc == -ENOMEM)
1458 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,
1459 is_udplite);
1460 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
1461 kfree_skb(skb);
1462 trace_udp_fail_queue_rcv_skb(rc, sk);
1463 return -1;
1464 }
1465
1466 return 0;
1467
1468 }
1469
1470 static struct static_key udp_encap_needed __read_mostly;
udp_encap_enable(void)1471 void udp_encap_enable(void)
1472 {
1473 if (!static_key_enabled(&udp_encap_needed))
1474 static_key_slow_inc(&udp_encap_needed);
1475 }
1476 EXPORT_SYMBOL(udp_encap_enable);
1477
1478 /* returns:
1479 * -1: error
1480 * 0: success
1481 * >0: "udp encap" protocol resubmission
1482 *
1483 * Note that in the success and error cases, the skb is assumed to
1484 * have either been requeued or freed.
1485 */
udp_queue_rcv_skb(struct sock * sk,struct sk_buff * skb)1486 int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
1487 {
1488 struct udp_sock *up = udp_sk(sk);
1489 int rc;
1490 int is_udplite = IS_UDPLITE(sk);
1491
1492 /*
1493 * Charge it to the socket, dropping if the queue is full.
1494 */
1495 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
1496 goto drop;
1497 nf_reset(skb);
1498
1499 if (static_key_false(&udp_encap_needed) && up->encap_type) {
1500 int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
1501
1502 /*
1503 * This is an encapsulation socket so pass the skb to
1504 * the socket's udp_encap_rcv() hook. Otherwise, just
1505 * fall through and pass this up the UDP socket.
1506 * up->encap_rcv() returns the following value:
1507 * =0 if skb was successfully passed to the encap
1508 * handler or was discarded by it.
1509 * >0 if skb should be passed on to UDP.
1510 * <0 if skb should be resubmitted as proto -N
1511 */
1512
1513 /* if we're overly short, let UDP handle it */
1514 encap_rcv = ACCESS_ONCE(up->encap_rcv);
1515 if (skb->len > sizeof(struct udphdr) && encap_rcv != NULL) {
1516 int ret;
1517
1518 /* Verify checksum before giving to encap */
1519 if (udp_lib_checksum_complete(skb))
1520 goto csum_error;
1521
1522 ret = encap_rcv(sk, skb);
1523 if (ret <= 0) {
1524 UDP_INC_STATS_BH(sock_net(sk),
1525 UDP_MIB_INDATAGRAMS,
1526 is_udplite);
1527 return -ret;
1528 }
1529 }
1530
1531 /* FALLTHROUGH -- it's a UDP Packet */
1532 }
1533
1534 /*
1535 * UDP-Lite specific tests, ignored on UDP sockets
1536 */
1537 if ((is_udplite & UDPLITE_RECV_CC) && UDP_SKB_CB(skb)->partial_cov) {
1538
1539 /*
1540 * MIB statistics other than incrementing the error count are
1541 * disabled for the following two types of errors: these depend
1542 * on the application settings, not on the functioning of the
1543 * protocol stack as such.
1544 *
1545 * RFC 3828 here recommends (sec 3.3): "There should also be a
1546 * way ... to ... at least let the receiving application block
1547 * delivery of packets with coverage values less than a value
1548 * provided by the application."
1549 */
1550 if (up->pcrlen == 0) { /* full coverage was set */
1551 LIMIT_NETDEBUG(KERN_WARNING "UDPLite: partial coverage %d while full coverage %d requested\n",
1552 UDP_SKB_CB(skb)->cscov, skb->len);
1553 goto drop;
1554 }
1555 /* The next case involves violating the min. coverage requested
1556 * by the receiver. This is subtle: if receiver wants x and x is
1557 * greater than the buffersize/MTU then receiver will complain
1558 * that it wants x while sender emits packets of smaller size y.
1559 * Therefore the above ...()->partial_cov statement is essential.
1560 */
1561 if (UDP_SKB_CB(skb)->cscov < up->pcrlen) {
1562 LIMIT_NETDEBUG(KERN_WARNING "UDPLite: coverage %d too small, need min %d\n",
1563 UDP_SKB_CB(skb)->cscov, up->pcrlen);
1564 goto drop;
1565 }
1566 }
1567
1568 if (rcu_access_pointer(sk->sk_filter) &&
1569 udp_lib_checksum_complete(skb))
1570 goto csum_error;
1571
1572
1573 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
1574 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,
1575 is_udplite);
1576 goto drop;
1577 }
1578
1579 rc = 0;
1580
1581 ipv4_pktinfo_prepare(sk, skb);
1582 bh_lock_sock(sk);
1583 if (!sock_owned_by_user(sk))
1584 rc = __udp_queue_rcv_skb(sk, skb);
1585 else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) {
1586 bh_unlock_sock(sk);
1587 goto drop;
1588 }
1589 bh_unlock_sock(sk);
1590
1591 return rc;
1592
1593 csum_error:
1594 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
1595 drop:
1596 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
1597 atomic_inc(&sk->sk_drops);
1598 kfree_skb(skb);
1599 return -1;
1600 }
1601
1602
flush_stack(struct sock ** stack,unsigned int count,struct sk_buff * skb,unsigned int final)1603 static void flush_stack(struct sock **stack, unsigned int count,
1604 struct sk_buff *skb, unsigned int final)
1605 {
1606 unsigned int i;
1607 struct sk_buff *skb1 = NULL;
1608 struct sock *sk;
1609
1610 for (i = 0; i < count; i++) {
1611 sk = stack[i];
1612 if (likely(skb1 == NULL))
1613 skb1 = (i == final) ? skb : skb_clone(skb, GFP_ATOMIC);
1614
1615 if (!skb1) {
1616 atomic_inc(&sk->sk_drops);
1617 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,
1618 IS_UDPLITE(sk));
1619 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,
1620 IS_UDPLITE(sk));
1621 }
1622
1623 if (skb1 && udp_queue_rcv_skb(sk, skb1) <= 0)
1624 skb1 = NULL;
1625
1626 sock_put(sk);
1627 }
1628 if (unlikely(skb1))
1629 kfree_skb(skb1);
1630 }
1631
1632 /* For TCP sockets, sk_rx_dst is protected by socket lock
1633 * For UDP, we use xchg() to guard against concurrent changes.
1634 */
udp_sk_rx_dst_set(struct sock * sk,struct dst_entry * dst)1635 static void udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
1636 {
1637 struct dst_entry *old;
1638
1639 dst_hold(dst);
1640 old = xchg(&sk->sk_rx_dst, dst);
1641 dst_release(old);
1642 }
1643
1644 /*
1645 * Multicasts and broadcasts go to each listener.
1646 *
1647 * Note: called only from the BH handler context.
1648 */
__udp4_lib_mcast_deliver(struct net * net,struct sk_buff * skb,struct udphdr * uh,__be32 saddr,__be32 daddr,struct udp_table * udptable)1649 static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
1650 struct udphdr *uh,
1651 __be32 saddr, __be32 daddr,
1652 struct udp_table *udptable)
1653 {
1654 struct sock *sk, *stack[256 / sizeof(struct sock *)];
1655 struct hlist_nulls_node *node;
1656 unsigned short hnum = ntohs(uh->dest);
1657 struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);
1658 int dif = skb->dev->ifindex;
1659 unsigned int count = 0, offset = offsetof(typeof(*sk), sk_nulls_node);
1660 unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
1661
1662 if (use_hash2) {
1663 hash2_any = udp4_portaddr_hash(net, htonl(INADDR_ANY), hnum) &
1664 udp_table.mask;
1665 hash2 = udp4_portaddr_hash(net, daddr, hnum) & udp_table.mask;
1666 start_lookup:
1667 hslot = &udp_table.hash2[hash2];
1668 offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
1669 }
1670
1671 spin_lock(&hslot->lock);
1672 sk_nulls_for_each_entry_offset(sk, node, &hslot->head, offset) {
1673 if (__udp_is_mcast_sock(net, sk,
1674 uh->dest, daddr,
1675 uh->source, saddr,
1676 dif, hnum)) {
1677 if (unlikely(count == ARRAY_SIZE(stack))) {
1678 flush_stack(stack, count, skb, ~0);
1679 count = 0;
1680 }
1681 stack[count++] = sk;
1682 sock_hold(sk);
1683 }
1684 }
1685
1686 spin_unlock(&hslot->lock);
1687
1688 /* Also lookup *:port if we are using hash2 and haven't done so yet. */
1689 if (use_hash2 && hash2 != hash2_any) {
1690 hash2 = hash2_any;
1691 goto start_lookup;
1692 }
1693
1694 /*
1695 * do the slow work with no lock held
1696 */
1697 if (count) {
1698 flush_stack(stack, count, skb, count - 1);
1699 } else {
1700 kfree_skb(skb);
1701 }
1702 return 0;
1703 }
1704
1705 /* Initialize UDP checksum. If exited with zero value (success),
1706 * CHECKSUM_UNNECESSARY means, that no more checks are required.
1707 * Otherwise, csum completion requires chacksumming packet body,
1708 * including udp header and folding it to skb->csum.
1709 */
udp4_csum_init(struct sk_buff * skb,struct udphdr * uh,int proto)1710 static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh,
1711 int proto)
1712 {
1713 int err;
1714
1715 UDP_SKB_CB(skb)->partial_cov = 0;
1716 UDP_SKB_CB(skb)->cscov = skb->len;
1717
1718 if (proto == IPPROTO_UDPLITE) {
1719 err = udplite_checksum_init(skb, uh);
1720 if (err)
1721 return err;
1722 }
1723
1724 return skb_checksum_init_zero_check(skb, proto, uh->check,
1725 inet_compute_pseudo);
1726 }
1727
1728 /*
1729 * All we need to do is get the socket, and then do a checksum.
1730 */
1731
__udp4_lib_rcv(struct sk_buff * skb,struct udp_table * udptable,int proto)1732 int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
1733 int proto)
1734 {
1735 struct sock *sk;
1736 struct udphdr *uh;
1737 unsigned short ulen;
1738 struct rtable *rt = skb_rtable(skb);
1739 __be32 saddr, daddr;
1740 struct net *net = dev_net(skb->dev);
1741
1742 /*
1743 * Validate the packet.
1744 */
1745 if (!pskb_may_pull(skb, sizeof(struct udphdr)))
1746 goto drop; /* No space for header. */
1747
1748 uh = udp_hdr(skb);
1749 ulen = ntohs(uh->len);
1750 saddr = ip_hdr(skb)->saddr;
1751 daddr = ip_hdr(skb)->daddr;
1752
1753 if (ulen > skb->len)
1754 goto short_packet;
1755
1756 if (proto == IPPROTO_UDP) {
1757 /* UDP validates ulen. */
1758 if (ulen < sizeof(*uh) || pskb_trim_rcsum(skb, ulen))
1759 goto short_packet;
1760 uh = udp_hdr(skb);
1761 }
1762
1763 if (udp4_csum_init(skb, uh, proto))
1764 goto csum_error;
1765
1766 sk = skb_steal_sock(skb);
1767 if (sk) {
1768 struct dst_entry *dst = skb_dst(skb);
1769 int ret;
1770
1771 if (unlikely(sk->sk_rx_dst != dst))
1772 udp_sk_rx_dst_set(sk, dst);
1773
1774 ret = udp_queue_rcv_skb(sk, skb);
1775 sock_put(sk);
1776 /* a return value > 0 means to resubmit the input, but
1777 * it wants the return to be -protocol, or 0
1778 */
1779 if (ret > 0)
1780 return -ret;
1781 return 0;
1782 } else {
1783 if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST))
1784 return __udp4_lib_mcast_deliver(net, skb, uh,
1785 saddr, daddr, udptable);
1786
1787 sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest, udptable);
1788 }
1789
1790 if (sk != NULL) {
1791 int ret;
1792
1793 if (udp_sk(sk)->convert_csum && uh->check && !IS_UDPLITE(sk))
1794 skb_checksum_try_convert(skb, IPPROTO_UDP, uh->check,
1795 inet_compute_pseudo);
1796
1797 ret = udp_queue_rcv_skb(sk, skb);
1798 sock_put(sk);
1799
1800 /* a return value > 0 means to resubmit the input, but
1801 * it wants the return to be -protocol, or 0
1802 */
1803 if (ret > 0)
1804 return -ret;
1805 return 0;
1806 }
1807
1808 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
1809 goto drop;
1810 nf_reset(skb);
1811
1812 /* No socket. Drop packet silently, if checksum is wrong */
1813 if (udp_lib_checksum_complete(skb))
1814 goto csum_error;
1815
1816 UDP_INC_STATS_BH(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE);
1817 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
1818
1819 /*
1820 * Hmm. We got an UDP packet to a port to which we
1821 * don't wanna listen. Ignore it.
1822 */
1823 kfree_skb(skb);
1824 return 0;
1825
1826 short_packet:
1827 LIMIT_NETDEBUG(KERN_DEBUG "UDP%s: short packet: From %pI4:%u %d/%d to %pI4:%u\n",
1828 proto == IPPROTO_UDPLITE ? "Lite" : "",
1829 &saddr, ntohs(uh->source),
1830 ulen, skb->len,
1831 &daddr, ntohs(uh->dest));
1832 goto drop;
1833
1834 csum_error:
1835 /*
1836 * RFC1122: OK. Discards the bad packet silently (as far as
1837 * the network is concerned, anyway) as per 4.1.3.4 (MUST).
1838 */
1839 LIMIT_NETDEBUG(KERN_DEBUG "UDP%s: bad checksum. From %pI4:%u to %pI4:%u ulen %d\n",
1840 proto == IPPROTO_UDPLITE ? "Lite" : "",
1841 &saddr, ntohs(uh->source), &daddr, ntohs(uh->dest),
1842 ulen);
1843 UDP_INC_STATS_BH(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE);
1844 drop:
1845 UDP_INC_STATS_BH(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE);
1846 kfree_skb(skb);
1847 return 0;
1848 }
1849
1850 /* We can only early demux multicast if there is a single matching socket.
1851 * If more than one socket found returns NULL
1852 */
__udp4_lib_mcast_demux_lookup(struct net * net,__be16 loc_port,__be32 loc_addr,__be16 rmt_port,__be32 rmt_addr,int dif)1853 static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net,
1854 __be16 loc_port, __be32 loc_addr,
1855 __be16 rmt_port, __be32 rmt_addr,
1856 int dif)
1857 {
1858 struct sock *sk, *result;
1859 struct hlist_nulls_node *node;
1860 unsigned short hnum = ntohs(loc_port);
1861 unsigned int count, slot = udp_hashfn(net, hnum, udp_table.mask);
1862 struct udp_hslot *hslot = &udp_table.hash[slot];
1863
1864 /* Do not bother scanning a too big list */
1865 if (hslot->count > 10)
1866 return NULL;
1867
1868 rcu_read_lock();
1869 begin:
1870 count = 0;
1871 result = NULL;
1872 sk_nulls_for_each_rcu(sk, node, &hslot->head) {
1873 if (__udp_is_mcast_sock(net, sk,
1874 loc_port, loc_addr,
1875 rmt_port, rmt_addr,
1876 dif, hnum)) {
1877 result = sk;
1878 ++count;
1879 }
1880 }
1881 /*
1882 * if the nulls value we got at the end of this lookup is
1883 * not the expected one, we must restart lookup.
1884 * We probably met an item that was moved to another chain.
1885 */
1886 if (get_nulls_value(node) != slot)
1887 goto begin;
1888
1889 if (result) {
1890 if (count != 1 ||
1891 unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
1892 result = NULL;
1893 else if (unlikely(!__udp_is_mcast_sock(net, result,
1894 loc_port, loc_addr,
1895 rmt_port, rmt_addr,
1896 dif, hnum))) {
1897 sock_put(result);
1898 result = NULL;
1899 }
1900 }
1901 rcu_read_unlock();
1902 return result;
1903 }
1904
1905 /* For unicast we should only early demux connected sockets or we can
1906 * break forwarding setups. The chains here can be long so only check
1907 * if the first socket is an exact match and if not move on.
1908 */
__udp4_lib_demux_lookup(struct net * net,__be16 loc_port,__be32 loc_addr,__be16 rmt_port,__be32 rmt_addr,int dif)1909 static struct sock *__udp4_lib_demux_lookup(struct net *net,
1910 __be16 loc_port, __be32 loc_addr,
1911 __be16 rmt_port, __be32 rmt_addr,
1912 int dif)
1913 {
1914 struct sock *sk, *result;
1915 struct hlist_nulls_node *node;
1916 unsigned short hnum = ntohs(loc_port);
1917 unsigned int hash2 = udp4_portaddr_hash(net, loc_addr, hnum);
1918 unsigned int slot2 = hash2 & udp_table.mask;
1919 struct udp_hslot *hslot2 = &udp_table.hash2[slot2];
1920 INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr);
1921 const __portpair ports = INET_COMBINED_PORTS(rmt_port, hnum);
1922
1923 rcu_read_lock();
1924 result = NULL;
1925 udp_portaddr_for_each_entry_rcu(sk, node, &hslot2->head) {
1926 if (INET_MATCH(sk, net, acookie,
1927 rmt_addr, loc_addr, ports, dif))
1928 result = sk;
1929 /* Only check first socket in chain */
1930 break;
1931 }
1932
1933 if (result) {
1934 if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
1935 result = NULL;
1936 else if (unlikely(!INET_MATCH(sk, net, acookie,
1937 rmt_addr, loc_addr,
1938 ports, dif))) {
1939 sock_put(result);
1940 result = NULL;
1941 }
1942 }
1943 rcu_read_unlock();
1944 return result;
1945 }
1946
udp_v4_early_demux(struct sk_buff * skb)1947 void udp_v4_early_demux(struct sk_buff *skb)
1948 {
1949 struct net *net = dev_net(skb->dev);
1950 const struct iphdr *iph;
1951 const struct udphdr *uh;
1952 struct sock *sk;
1953 struct dst_entry *dst;
1954 int dif = skb->dev->ifindex;
1955 int ours;
1956
1957 /* validate the packet */
1958 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct udphdr)))
1959 return;
1960
1961 iph = ip_hdr(skb);
1962 uh = udp_hdr(skb);
1963
1964 if (skb->pkt_type == PACKET_BROADCAST ||
1965 skb->pkt_type == PACKET_MULTICAST) {
1966 struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
1967
1968 if (!in_dev)
1969 return;
1970
1971 /* we are supposed to accept bcast packets */
1972 if (skb->pkt_type == PACKET_MULTICAST) {
1973 ours = ip_check_mc_rcu(in_dev, iph->daddr, iph->saddr,
1974 iph->protocol);
1975 if (!ours)
1976 return;
1977 }
1978
1979 sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr,
1980 uh->source, iph->saddr, dif);
1981 } else if (skb->pkt_type == PACKET_HOST) {
1982 sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr,
1983 uh->source, iph->saddr, dif);
1984 } else {
1985 return;
1986 }
1987
1988 if (!sk)
1989 return;
1990
1991 skb->sk = sk;
1992 skb->destructor = sock_efree;
1993 dst = READ_ONCE(sk->sk_rx_dst);
1994
1995 if (dst)
1996 dst = dst_check(dst, 0);
1997 if (dst) {
1998 /* DST_NOCACHE can not be used without taking a reference */
1999 if (dst->flags & DST_NOCACHE) {
2000 if (likely(atomic_inc_not_zero(&dst->__refcnt)))
2001 skb_dst_set(skb, dst);
2002 } else {
2003 skb_dst_set_noref(skb, dst);
2004 }
2005 }
2006 }
2007
udp_rcv(struct sk_buff * skb)2008 int udp_rcv(struct sk_buff *skb)
2009 {
2010 return __udp4_lib_rcv(skb, &udp_table, IPPROTO_UDP);
2011 }
2012
udp_destroy_sock(struct sock * sk)2013 void udp_destroy_sock(struct sock *sk)
2014 {
2015 struct udp_sock *up = udp_sk(sk);
2016 bool slow = lock_sock_fast(sk);
2017 udp_flush_pending_frames(sk);
2018 unlock_sock_fast(sk, slow);
2019 if (static_key_false(&udp_encap_needed) && up->encap_type) {
2020 void (*encap_destroy)(struct sock *sk);
2021 encap_destroy = ACCESS_ONCE(up->encap_destroy);
2022 if (encap_destroy)
2023 encap_destroy(sk);
2024 }
2025 }
2026
2027 /*
2028 * Socket option code for UDP
2029 */
udp_lib_setsockopt(struct sock * sk,int level,int optname,char __user * optval,unsigned int optlen,int (* push_pending_frames)(struct sock *))2030 int udp_lib_setsockopt(struct sock *sk, int level, int optname,
2031 char __user *optval, unsigned int optlen,
2032 int (*push_pending_frames)(struct sock *))
2033 {
2034 struct udp_sock *up = udp_sk(sk);
2035 int val, valbool;
2036 int err = 0;
2037 int is_udplite = IS_UDPLITE(sk);
2038
2039 if (optlen < sizeof(int))
2040 return -EINVAL;
2041
2042 if (get_user(val, (int __user *)optval))
2043 return -EFAULT;
2044
2045 valbool = val ? 1 : 0;
2046
2047 switch (optname) {
2048 case UDP_CORK:
2049 if (val != 0) {
2050 up->corkflag = 1;
2051 } else {
2052 up->corkflag = 0;
2053 lock_sock(sk);
2054 (*push_pending_frames)(sk);
2055 release_sock(sk);
2056 }
2057 break;
2058
2059 case UDP_ENCAP:
2060 switch (val) {
2061 case 0:
2062 case UDP_ENCAP_ESPINUDP:
2063 case UDP_ENCAP_ESPINUDP_NON_IKE:
2064 up->encap_rcv = xfrm4_udp_encap_rcv;
2065 /* FALLTHROUGH */
2066 case UDP_ENCAP_L2TPINUDP:
2067 up->encap_type = val;
2068 udp_encap_enable();
2069 break;
2070 default:
2071 err = -ENOPROTOOPT;
2072 break;
2073 }
2074 break;
2075
2076 case UDP_NO_CHECK6_TX:
2077 up->no_check6_tx = valbool;
2078 break;
2079
2080 case UDP_NO_CHECK6_RX:
2081 up->no_check6_rx = valbool;
2082 break;
2083
2084 /*
2085 * UDP-Lite's partial checksum coverage (RFC 3828).
2086 */
2087 /* The sender sets actual checksum coverage length via this option.
2088 * The case coverage > packet length is handled by send module. */
2089 case UDPLITE_SEND_CSCOV:
2090 if (!is_udplite) /* Disable the option on UDP sockets */
2091 return -ENOPROTOOPT;
2092 if (val != 0 && val < 8) /* Illegal coverage: use default (8) */
2093 val = 8;
2094 else if (val > USHRT_MAX)
2095 val = USHRT_MAX;
2096 up->pcslen = val;
2097 up->pcflag |= UDPLITE_SEND_CC;
2098 break;
2099
2100 /* The receiver specifies a minimum checksum coverage value. To make
2101 * sense, this should be set to at least 8 (as done below). If zero is
2102 * used, this again means full checksum coverage. */
2103 case UDPLITE_RECV_CSCOV:
2104 if (!is_udplite) /* Disable the option on UDP sockets */
2105 return -ENOPROTOOPT;
2106 if (val != 0 && val < 8) /* Avoid silly minimal values. */
2107 val = 8;
2108 else if (val > USHRT_MAX)
2109 val = USHRT_MAX;
2110 up->pcrlen = val;
2111 up->pcflag |= UDPLITE_RECV_CC;
2112 break;
2113
2114 default:
2115 err = -ENOPROTOOPT;
2116 break;
2117 }
2118
2119 return err;
2120 }
2121 EXPORT_SYMBOL(udp_lib_setsockopt);
2122
udp_setsockopt(struct sock * sk,int level,int optname,char __user * optval,unsigned int optlen)2123 int udp_setsockopt(struct sock *sk, int level, int optname,
2124 char __user *optval, unsigned int optlen)
2125 {
2126 if (level == SOL_UDP || level == SOL_UDPLITE)
2127 return udp_lib_setsockopt(sk, level, optname, optval, optlen,
2128 udp_push_pending_frames);
2129 return ip_setsockopt(sk, level, optname, optval, optlen);
2130 }
2131
2132 #ifdef CONFIG_COMPAT
compat_udp_setsockopt(struct sock * sk,int level,int optname,char __user * optval,unsigned int optlen)2133 int compat_udp_setsockopt(struct sock *sk, int level, int optname,
2134 char __user *optval, unsigned int optlen)
2135 {
2136 if (level == SOL_UDP || level == SOL_UDPLITE)
2137 return udp_lib_setsockopt(sk, level, optname, optval, optlen,
2138 udp_push_pending_frames);
2139 return compat_ip_setsockopt(sk, level, optname, optval, optlen);
2140 }
2141 #endif
2142
udp_lib_getsockopt(struct sock * sk,int level,int optname,char __user * optval,int __user * optlen)2143 int udp_lib_getsockopt(struct sock *sk, int level, int optname,
2144 char __user *optval, int __user *optlen)
2145 {
2146 struct udp_sock *up = udp_sk(sk);
2147 int val, len;
2148
2149 if (get_user(len, optlen))
2150 return -EFAULT;
2151
2152 len = min_t(unsigned int, len, sizeof(int));
2153
2154 if (len < 0)
2155 return -EINVAL;
2156
2157 switch (optname) {
2158 case UDP_CORK:
2159 val = up->corkflag;
2160 break;
2161
2162 case UDP_ENCAP:
2163 val = up->encap_type;
2164 break;
2165
2166 case UDP_NO_CHECK6_TX:
2167 val = up->no_check6_tx;
2168 break;
2169
2170 case UDP_NO_CHECK6_RX:
2171 val = up->no_check6_rx;
2172 break;
2173
2174 /* The following two cannot be changed on UDP sockets, the return is
2175 * always 0 (which corresponds to the full checksum coverage of UDP). */
2176 case UDPLITE_SEND_CSCOV:
2177 val = up->pcslen;
2178 break;
2179
2180 case UDPLITE_RECV_CSCOV:
2181 val = up->pcrlen;
2182 break;
2183
2184 default:
2185 return -ENOPROTOOPT;
2186 }
2187
2188 if (put_user(len, optlen))
2189 return -EFAULT;
2190 if (copy_to_user(optval, &val, len))
2191 return -EFAULT;
2192 return 0;
2193 }
2194 EXPORT_SYMBOL(udp_lib_getsockopt);
2195
udp_getsockopt(struct sock * sk,int level,int optname,char __user * optval,int __user * optlen)2196 int udp_getsockopt(struct sock *sk, int level, int optname,
2197 char __user *optval, int __user *optlen)
2198 {
2199 if (level == SOL_UDP || level == SOL_UDPLITE)
2200 return udp_lib_getsockopt(sk, level, optname, optval, optlen);
2201 return ip_getsockopt(sk, level, optname, optval, optlen);
2202 }
2203
2204 #ifdef CONFIG_COMPAT
compat_udp_getsockopt(struct sock * sk,int level,int optname,char __user * optval,int __user * optlen)2205 int compat_udp_getsockopt(struct sock *sk, int level, int optname,
2206 char __user *optval, int __user *optlen)
2207 {
2208 if (level == SOL_UDP || level == SOL_UDPLITE)
2209 return udp_lib_getsockopt(sk, level, optname, optval, optlen);
2210 return compat_ip_getsockopt(sk, level, optname, optval, optlen);
2211 }
2212 #endif
2213 /**
2214 * udp_poll - wait for a UDP event.
2215 * @file - file struct
2216 * @sock - socket
2217 * @wait - poll table
2218 *
2219 * This is same as datagram poll, except for the special case of
2220 * blocking sockets. If application is using a blocking fd
2221 * and a packet with checksum error is in the queue;
2222 * then it could get return from select indicating data available
2223 * but then block when reading it. Add special case code
2224 * to work around these arguably broken applications.
2225 */
udp_poll(struct file * file,struct socket * sock,poll_table * wait)2226 unsigned int udp_poll(struct file *file, struct socket *sock, poll_table *wait)
2227 {
2228 unsigned int mask = datagram_poll(file, sock, wait);
2229 struct sock *sk = sock->sk;
2230
2231 sock_rps_record_flow(sk);
2232
2233 /* Check for false positives due to checksum errors */
2234 if ((mask & POLLRDNORM) && !(file->f_flags & O_NONBLOCK) &&
2235 !(sk->sk_shutdown & RCV_SHUTDOWN) && !first_packet_length(sk))
2236 mask &= ~(POLLIN | POLLRDNORM);
2237
2238 return mask;
2239
2240 }
2241 EXPORT_SYMBOL(udp_poll);
2242
udp_abort(struct sock * sk,int err)2243 int udp_abort(struct sock *sk, int err)
2244 {
2245 lock_sock(sk);
2246
2247 sk->sk_err = err;
2248 sk->sk_error_report(sk);
2249 udp_disconnect(sk, 0);
2250
2251 release_sock(sk);
2252
2253 return 0;
2254 }
2255 EXPORT_SYMBOL_GPL(udp_abort);
2256
2257 struct proto udp_prot = {
2258 .name = "UDP",
2259 .owner = THIS_MODULE,
2260 .close = udp_lib_close,
2261 .connect = ip4_datagram_connect,
2262 .disconnect = udp_disconnect,
2263 .ioctl = udp_ioctl,
2264 .destroy = udp_destroy_sock,
2265 .setsockopt = udp_setsockopt,
2266 .getsockopt = udp_getsockopt,
2267 .sendmsg = udp_sendmsg,
2268 .recvmsg = udp_recvmsg,
2269 .sendpage = udp_sendpage,
2270 .backlog_rcv = __udp_queue_rcv_skb,
2271 .release_cb = ip4_datagram_release_cb,
2272 .hash = udp_lib_hash,
2273 .unhash = udp_lib_unhash,
2274 .rehash = udp_v4_rehash,
2275 .get_port = udp_v4_get_port,
2276 .memory_allocated = &udp_memory_allocated,
2277 .sysctl_mem = sysctl_udp_mem,
2278 .sysctl_wmem = &sysctl_udp_wmem_min,
2279 .sysctl_rmem = &sysctl_udp_rmem_min,
2280 .obj_size = sizeof(struct udp_sock),
2281 .slab_flags = SLAB_DESTROY_BY_RCU,
2282 .h.udp_table = &udp_table,
2283 #ifdef CONFIG_COMPAT
2284 .compat_setsockopt = compat_udp_setsockopt,
2285 .compat_getsockopt = compat_udp_getsockopt,
2286 #endif
2287 .clear_sk = sk_prot_clear_portaddr_nulls,
2288 .diag_destroy = udp_abort,
2289 };
2290 EXPORT_SYMBOL(udp_prot);
2291
2292 /* ------------------------------------------------------------------------ */
2293 #ifdef CONFIG_PROC_FS
2294
udp_get_first(struct seq_file * seq,int start)2295 static struct sock *udp_get_first(struct seq_file *seq, int start)
2296 {
2297 struct sock *sk;
2298 struct udp_iter_state *state = seq->private;
2299 struct net *net = seq_file_net(seq);
2300
2301 for (state->bucket = start; state->bucket <= state->udp_table->mask;
2302 ++state->bucket) {
2303 struct hlist_nulls_node *node;
2304 struct udp_hslot *hslot = &state->udp_table->hash[state->bucket];
2305
2306 if (hlist_nulls_empty(&hslot->head))
2307 continue;
2308
2309 spin_lock_bh(&hslot->lock);
2310 sk_nulls_for_each(sk, node, &hslot->head) {
2311 if (!net_eq(sock_net(sk), net))
2312 continue;
2313 if (sk->sk_family == state->family)
2314 goto found;
2315 }
2316 spin_unlock_bh(&hslot->lock);
2317 }
2318 sk = NULL;
2319 found:
2320 return sk;
2321 }
2322
udp_get_next(struct seq_file * seq,struct sock * sk)2323 static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk)
2324 {
2325 struct udp_iter_state *state = seq->private;
2326 struct net *net = seq_file_net(seq);
2327
2328 do {
2329 sk = sk_nulls_next(sk);
2330 } while (sk && (!net_eq(sock_net(sk), net) || sk->sk_family != state->family));
2331
2332 if (!sk) {
2333 if (state->bucket <= state->udp_table->mask)
2334 spin_unlock_bh(&state->udp_table->hash[state->bucket].lock);
2335 return udp_get_first(seq, state->bucket + 1);
2336 }
2337 return sk;
2338 }
2339
udp_get_idx(struct seq_file * seq,loff_t pos)2340 static struct sock *udp_get_idx(struct seq_file *seq, loff_t pos)
2341 {
2342 struct sock *sk = udp_get_first(seq, 0);
2343
2344 if (sk)
2345 while (pos && (sk = udp_get_next(seq, sk)) != NULL)
2346 --pos;
2347 return pos ? NULL : sk;
2348 }
2349
udp_seq_start(struct seq_file * seq,loff_t * pos)2350 static void *udp_seq_start(struct seq_file *seq, loff_t *pos)
2351 {
2352 struct udp_iter_state *state = seq->private;
2353 state->bucket = MAX_UDP_PORTS;
2354
2355 return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN;
2356 }
2357
udp_seq_next(struct seq_file * seq,void * v,loff_t * pos)2358 static void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2359 {
2360 struct sock *sk;
2361
2362 if (v == SEQ_START_TOKEN)
2363 sk = udp_get_idx(seq, 0);
2364 else
2365 sk = udp_get_next(seq, v);
2366
2367 ++*pos;
2368 return sk;
2369 }
2370
udp_seq_stop(struct seq_file * seq,void * v)2371 static void udp_seq_stop(struct seq_file *seq, void *v)
2372 {
2373 struct udp_iter_state *state = seq->private;
2374
2375 if (state->bucket <= state->udp_table->mask)
2376 spin_unlock_bh(&state->udp_table->hash[state->bucket].lock);
2377 }
2378
udp_seq_open(struct inode * inode,struct file * file)2379 int udp_seq_open(struct inode *inode, struct file *file)
2380 {
2381 struct udp_seq_afinfo *afinfo = PDE_DATA(inode);
2382 struct udp_iter_state *s;
2383 int err;
2384
2385 err = seq_open_net(inode, file, &afinfo->seq_ops,
2386 sizeof(struct udp_iter_state));
2387 if (err < 0)
2388 return err;
2389
2390 s = ((struct seq_file *)file->private_data)->private;
2391 s->family = afinfo->family;
2392 s->udp_table = afinfo->udp_table;
2393 return err;
2394 }
2395 EXPORT_SYMBOL(udp_seq_open);
2396
2397 /* ------------------------------------------------------------------------ */
udp_proc_register(struct net * net,struct udp_seq_afinfo * afinfo)2398 int udp_proc_register(struct net *net, struct udp_seq_afinfo *afinfo)
2399 {
2400 struct proc_dir_entry *p;
2401 int rc = 0;
2402
2403 afinfo->seq_ops.start = udp_seq_start;
2404 afinfo->seq_ops.next = udp_seq_next;
2405 afinfo->seq_ops.stop = udp_seq_stop;
2406
2407 p = proc_create_data(afinfo->name, S_IRUGO, net->proc_net,
2408 afinfo->seq_fops, afinfo);
2409 if (!p)
2410 rc = -ENOMEM;
2411 return rc;
2412 }
2413 EXPORT_SYMBOL(udp_proc_register);
2414
udp_proc_unregister(struct net * net,struct udp_seq_afinfo * afinfo)2415 void udp_proc_unregister(struct net *net, struct udp_seq_afinfo *afinfo)
2416 {
2417 remove_proc_entry(afinfo->name, net->proc_net);
2418 }
2419 EXPORT_SYMBOL(udp_proc_unregister);
2420
2421 /* ------------------------------------------------------------------------ */
udp4_format_sock(struct sock * sp,struct seq_file * f,int bucket)2422 static void udp4_format_sock(struct sock *sp, struct seq_file *f,
2423 int bucket)
2424 {
2425 struct inet_sock *inet = inet_sk(sp);
2426 __be32 dest = inet->inet_daddr;
2427 __be32 src = inet->inet_rcv_saddr;
2428 __u16 destp = ntohs(inet->inet_dport);
2429 __u16 srcp = ntohs(inet->inet_sport);
2430
2431 seq_printf(f, "%5d: %08X:%04X %08X:%04X"
2432 " %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %d",
2433 bucket, src, srcp, dest, destp, sp->sk_state,
2434 sk_wmem_alloc_get(sp),
2435 sk_rmem_alloc_get(sp),
2436 0, 0L, 0,
2437 from_kuid_munged(seq_user_ns(f), sock_i_uid(sp)),
2438 0, sock_i_ino(sp),
2439 atomic_read(&sp->sk_refcnt), sp,
2440 atomic_read(&sp->sk_drops));
2441 }
2442
udp4_seq_show(struct seq_file * seq,void * v)2443 int udp4_seq_show(struct seq_file *seq, void *v)
2444 {
2445 seq_setwidth(seq, 127);
2446 if (v == SEQ_START_TOKEN)
2447 seq_puts(seq, " sl local_address rem_address st tx_queue "
2448 "rx_queue tr tm->when retrnsmt uid timeout "
2449 "inode ref pointer drops");
2450 else {
2451 struct udp_iter_state *state = seq->private;
2452
2453 udp4_format_sock(v, seq, state->bucket);
2454 }
2455 seq_pad(seq, '\n');
2456 return 0;
2457 }
2458
2459 static const struct file_operations udp_afinfo_seq_fops = {
2460 .owner = THIS_MODULE,
2461 .open = udp_seq_open,
2462 .read = seq_read,
2463 .llseek = seq_lseek,
2464 .release = seq_release_net
2465 };
2466
2467 /* ------------------------------------------------------------------------ */
2468 static struct udp_seq_afinfo udp4_seq_afinfo = {
2469 .name = "udp",
2470 .family = AF_INET,
2471 .udp_table = &udp_table,
2472 .seq_fops = &udp_afinfo_seq_fops,
2473 .seq_ops = {
2474 .show = udp4_seq_show,
2475 },
2476 };
2477
udp4_proc_init_net(struct net * net)2478 static int __net_init udp4_proc_init_net(struct net *net)
2479 {
2480 return udp_proc_register(net, &udp4_seq_afinfo);
2481 }
2482
udp4_proc_exit_net(struct net * net)2483 static void __net_exit udp4_proc_exit_net(struct net *net)
2484 {
2485 udp_proc_unregister(net, &udp4_seq_afinfo);
2486 }
2487
2488 static struct pernet_operations udp4_net_ops = {
2489 .init = udp4_proc_init_net,
2490 .exit = udp4_proc_exit_net,
2491 };
2492
udp4_proc_init(void)2493 int __init udp4_proc_init(void)
2494 {
2495 return register_pernet_subsys(&udp4_net_ops);
2496 }
2497
udp4_proc_exit(void)2498 void udp4_proc_exit(void)
2499 {
2500 unregister_pernet_subsys(&udp4_net_ops);
2501 }
2502 #endif /* CONFIG_PROC_FS */
2503
2504 static __initdata unsigned long uhash_entries;
set_uhash_entries(char * str)2505 static int __init set_uhash_entries(char *str)
2506 {
2507 ssize_t ret;
2508
2509 if (!str)
2510 return 0;
2511
2512 ret = kstrtoul(str, 0, &uhash_entries);
2513 if (ret)
2514 return 0;
2515
2516 if (uhash_entries && uhash_entries < UDP_HTABLE_SIZE_MIN)
2517 uhash_entries = UDP_HTABLE_SIZE_MIN;
2518 return 1;
2519 }
2520 __setup("uhash_entries=", set_uhash_entries);
2521
udp_table_init(struct udp_table * table,const char * name)2522 void __init udp_table_init(struct udp_table *table, const char *name)
2523 {
2524 unsigned int i;
2525
2526 table->hash = alloc_large_system_hash(name,
2527 2 * sizeof(struct udp_hslot),
2528 uhash_entries,
2529 21, /* one slot per 2 MB */
2530 0,
2531 &table->log,
2532 &table->mask,
2533 UDP_HTABLE_SIZE_MIN,
2534 64 * 1024);
2535
2536 table->hash2 = table->hash + (table->mask + 1);
2537 for (i = 0; i <= table->mask; i++) {
2538 INIT_HLIST_NULLS_HEAD(&table->hash[i].head, i);
2539 table->hash[i].count = 0;
2540 spin_lock_init(&table->hash[i].lock);
2541 }
2542 for (i = 0; i <= table->mask; i++) {
2543 INIT_HLIST_NULLS_HEAD(&table->hash2[i].head, i);
2544 table->hash2[i].count = 0;
2545 spin_lock_init(&table->hash2[i].lock);
2546 }
2547 }
2548
udp_init(void)2549 void __init udp_init(void)
2550 {
2551 unsigned long limit;
2552
2553 udp_table_init(&udp_table, "UDP");
2554 limit = nr_free_buffer_pages() / 8;
2555 limit = max(limit, 128UL);
2556 sysctl_udp_mem[0] = limit / 4 * 3;
2557 sysctl_udp_mem[1] = limit;
2558 sysctl_udp_mem[2] = sysctl_udp_mem[0] * 2;
2559
2560 sysctl_udp_rmem_min = SK_MEM_QUANTUM;
2561 sysctl_udp_wmem_min = SK_MEM_QUANTUM;
2562 }
2563