Lines Matching refs:po
191 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
195 static void *packet_previous_frame(struct packet_sock *po,
247 static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
248 static void __fanout_link(struct sock *sk, struct packet_sock *po);
286 static struct net_device *packet_cached_dev_get(struct packet_sock *po) in packet_cached_dev_get() argument
291 dev = rcu_dereference(po->cached_dev); in packet_cached_dev_get()
299 static void packet_cached_dev_assign(struct packet_sock *po, in packet_cached_dev_assign() argument
302 rcu_assign_pointer(po->cached_dev, dev); in packet_cached_dev_assign()
305 static void packet_cached_dev_reset(struct packet_sock *po) in packet_cached_dev_reset() argument
307 RCU_INIT_POINTER(po->cached_dev, NULL); in packet_cached_dev_reset()
310 static bool packet_use_direct_xmit(const struct packet_sock *po) in packet_use_direct_xmit() argument
312 return po->xmit == packet_direct_xmit; in packet_use_direct_xmit()
342 struct packet_sock *po = pkt_sk(sk); in __register_prot_hook() local
344 if (!po->running) { in __register_prot_hook()
345 if (po->fanout) in __register_prot_hook()
346 __fanout_link(sk, po); in __register_prot_hook()
348 dev_add_pack(&po->prot_hook); in __register_prot_hook()
351 po->running = 1; in __register_prot_hook()
369 struct packet_sock *po = pkt_sk(sk); in __unregister_prot_hook() local
371 lockdep_assert_held_once(&po->bind_lock); in __unregister_prot_hook()
373 po->running = 0; in __unregister_prot_hook()
375 if (po->fanout) in __unregister_prot_hook()
376 __fanout_unlink(sk, po); in __unregister_prot_hook()
378 __dev_remove_pack(&po->prot_hook); in __unregister_prot_hook()
383 spin_unlock(&po->bind_lock); in __unregister_prot_hook()
385 spin_lock(&po->bind_lock); in __unregister_prot_hook()
391 struct packet_sock *po = pkt_sk(sk); in unregister_prot_hook() local
393 if (po->running) in unregister_prot_hook()
404 static void __packet_set_status(struct packet_sock *po, void *frame, int status) in __packet_set_status() argument
409 switch (po->tp_version) { in __packet_set_status()
427 static int __packet_get_status(struct packet_sock *po, void *frame) in __packet_get_status() argument
434 switch (po->tp_version) { in __packet_get_status()
465 static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame, in __packet_set_timestamp() argument
472 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp))) in __packet_set_timestamp()
476 switch (po->tp_version) { in __packet_set_timestamp()
498 static void *packet_lookup_frame(struct packet_sock *po, in packet_lookup_frame() argument
512 if (status != __packet_get_status(po, h.raw)) in packet_lookup_frame()
518 static void *packet_current_frame(struct packet_sock *po, in packet_current_frame() argument
522 return packet_lookup_frame(po, rb, rb->head, status); in packet_current_frame()
530 static void prb_shutdown_retire_blk_timer(struct packet_sock *po, in prb_shutdown_retire_blk_timer() argument
535 pkc = GET_PBDQC_FROM_RB(&po->rx_ring); in prb_shutdown_retire_blk_timer()
544 static void prb_init_blk_timer(struct packet_sock *po, in prb_init_blk_timer() argument
549 pkc->retire_blk_timer.data = (long)po; in prb_init_blk_timer()
554 static void prb_setup_retire_blk_timer(struct packet_sock *po) in prb_setup_retire_blk_timer() argument
558 pkc = GET_PBDQC_FROM_RB(&po->rx_ring); in prb_setup_retire_blk_timer()
559 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired); in prb_setup_retire_blk_timer()
562 static int prb_calc_retire_blk_tmo(struct packet_sock *po, in prb_calc_retire_blk_tmo() argument
571 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex); in prb_calc_retire_blk_tmo()
610 static void init_prb_bdqc(struct packet_sock *po, in init_prb_bdqc() argument
626 p1->hdrlen = po->tp_hdrlen; in init_prb_bdqc()
627 p1->version = po->tp_version; in init_prb_bdqc()
629 po->stats.stats3.tp_freeze_q_cnt = 0; in init_prb_bdqc()
633 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po, in init_prb_bdqc()
640 prb_setup_retire_blk_timer(po); in init_prb_bdqc()
679 struct packet_sock *po = (struct packet_sock *)data; in prb_retire_rx_blk_timer_expired() local
680 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring); in prb_retire_rx_blk_timer_expired()
684 spin_lock(&po->sk.sk_receive_queue.lock); in prb_retire_rx_blk_timer_expired()
714 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO); in prb_retire_rx_blk_timer_expired()
715 if (!prb_dispatch_next_block(pkc, po)) in prb_retire_rx_blk_timer_expired()
747 spin_unlock(&po->sk.sk_receive_queue.lock); in prb_retire_rx_blk_timer_expired()
795 struct packet_sock *po, unsigned int stat) in prb_close_block() argument
801 struct sock *sk = &po->sk; in prb_close_block()
803 if (po->stats.stats3.tp_drops) in prb_close_block()
908 struct packet_sock *po) in prb_freeze_queue() argument
911 po->stats.stats3.tp_freeze_q_cnt++; in prb_freeze_queue()
923 struct packet_sock *po) in prb_dispatch_next_block() argument
934 prb_freeze_queue(pkc, po); in prb_dispatch_next_block()
948 struct packet_sock *po, unsigned int status) in prb_retire_current_block() argument
969 prb_close_block(pkc, pbd, po, status); in prb_retire_current_block()
1047 static void *__packet_lookup_frame_in_block(struct packet_sock *po, in __packet_lookup_frame_in_block() argument
1057 pkc = GET_PBDQC_FROM_RB(&po->rx_ring); in __packet_lookup_frame_in_block()
1092 prb_retire_current_block(pkc, po, 0); in __packet_lookup_frame_in_block()
1095 curr = (char *)prb_dispatch_next_block(pkc, po); in __packet_lookup_frame_in_block()
1109 static void *packet_current_rx_frame(struct packet_sock *po, in packet_current_rx_frame() argument
1114 switch (po->tp_version) { in packet_current_rx_frame()
1117 curr = packet_lookup_frame(po, &po->rx_ring, in packet_current_rx_frame()
1118 po->rx_ring.head, status); in packet_current_rx_frame()
1121 return __packet_lookup_frame_in_block(po, skb, status, len); in packet_current_rx_frame()
1129 static void *prb_lookup_block(struct packet_sock *po, in prb_lookup_block() argument
1153 static void *__prb_previous_block(struct packet_sock *po, in __prb_previous_block() argument
1158 return prb_lookup_block(po, rb, previous, status); in __prb_previous_block()
1161 static void *packet_previous_rx_frame(struct packet_sock *po, in packet_previous_rx_frame() argument
1165 if (po->tp_version <= TPACKET_V2) in packet_previous_rx_frame()
1166 return packet_previous_frame(po, rb, status); in packet_previous_rx_frame()
1168 return __prb_previous_block(po, rb, status); in packet_previous_rx_frame()
1171 static void packet_increment_rx_head(struct packet_sock *po, in packet_increment_rx_head() argument
1174 switch (po->tp_version) { in packet_increment_rx_head()
1186 static void *packet_previous_frame(struct packet_sock *po, in packet_previous_frame() argument
1191 return packet_lookup_frame(po, rb, previous, status); in packet_previous_frame()
1224 static int packet_alloc_pending(struct packet_sock *po) in packet_alloc_pending() argument
1226 po->rx_ring.pending_refcnt = NULL; in packet_alloc_pending()
1228 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int); in packet_alloc_pending()
1229 if (unlikely(po->tx_ring.pending_refcnt == NULL)) in packet_alloc_pending()
1235 static void packet_free_pending(struct packet_sock *po) in packet_free_pending() argument
1237 free_percpu(po->tx_ring.pending_refcnt); in packet_free_pending()
1245 static bool __tpacket_has_room(struct packet_sock *po, int pow_off) in __tpacket_has_room() argument
1249 len = po->rx_ring.frame_max + 1; in __tpacket_has_room()
1250 idx = po->rx_ring.head; in __tpacket_has_room()
1255 return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL); in __tpacket_has_room()
1258 static bool __tpacket_v3_has_room(struct packet_sock *po, int pow_off) in __tpacket_v3_has_room() argument
1262 len = po->rx_ring.prb_bdqc.knum_blocks; in __tpacket_v3_has_room()
1263 idx = po->rx_ring.prb_bdqc.kactive_blk_num; in __tpacket_v3_has_room()
1268 return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL); in __tpacket_v3_has_room()
1271 static int __packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb) in __packet_rcv_has_room() argument
1273 struct sock *sk = &po->sk; in __packet_rcv_has_room()
1276 if (po->prot_hook.func != tpacket_rcv) { in __packet_rcv_has_room()
1287 if (po->tp_version == TPACKET_V3) { in __packet_rcv_has_room()
1288 if (__tpacket_v3_has_room(po, ROOM_POW_OFF)) in __packet_rcv_has_room()
1290 else if (__tpacket_v3_has_room(po, 0)) in __packet_rcv_has_room()
1293 if (__tpacket_has_room(po, ROOM_POW_OFF)) in __packet_rcv_has_room()
1295 else if (__tpacket_has_room(po, 0)) in __packet_rcv_has_room()
1302 static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb) in packet_rcv_has_room() argument
1307 spin_lock_bh(&po->sk.sk_receive_queue.lock); in packet_rcv_has_room()
1308 ret = __packet_rcv_has_room(po, skb); in packet_rcv_has_room()
1310 if (po->pressure == has_room) in packet_rcv_has_room()
1311 po->pressure = !has_room; in packet_rcv_has_room()
1312 spin_unlock_bh(&po->sk.sk_receive_queue.lock); in packet_rcv_has_room()
1332 static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb) in fanout_flow_is_huge() argument
1339 if (po->rollover->history[i] == rxhash) in fanout_flow_is_huge()
1342 po->rollover->history[prandom_u32() % ROLLOVER_HLEN] = rxhash; in fanout_flow_is_huge()
1381 struct packet_sock *po, *po_next, *po_skip = NULL; in fanout_demux_rollover() local
1384 po = pkt_sk(f->arr[idx]); in fanout_demux_rollover()
1387 room = packet_rcv_has_room(po, skb); in fanout_demux_rollover()
1389 (room == ROOM_LOW && !fanout_flow_is_huge(po, skb))) in fanout_demux_rollover()
1391 po_skip = po; in fanout_demux_rollover()
1394 i = j = min_t(int, po->rollover->sock, num - 1); in fanout_demux_rollover()
1400 po->rollover->sock = i; in fanout_demux_rollover()
1401 atomic_long_inc(&po->rollover->num); in fanout_demux_rollover()
1403 atomic_long_inc(&po->rollover->num_huge); in fanout_demux_rollover()
1411 atomic_long_inc(&po->rollover->num_failed); in fanout_demux_rollover()
1449 struct packet_sock *po; in packet_rcv_fanout() local
1491 po = pkt_sk(f->arr[idx]); in packet_rcv_fanout()
1492 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev); in packet_rcv_fanout()
1499 static void __fanout_link(struct sock *sk, struct packet_sock *po) in __fanout_link() argument
1501 struct packet_fanout *f = po->fanout; in __fanout_link()
1512 static void __fanout_unlink(struct sock *sk, struct packet_sock *po) in __fanout_unlink() argument
1514 struct packet_fanout *f = po->fanout; in __fanout_unlink()
1566 static int fanout_set_data_cbpf(struct packet_sock *po, char __user *data, in fanout_set_data_cbpf() argument
1573 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED)) in fanout_set_data_cbpf()
1584 __fanout_set_data_bpf(po->fanout, new); in fanout_set_data_cbpf()
1588 static int fanout_set_data_ebpf(struct packet_sock *po, char __user *data, in fanout_set_data_ebpf() argument
1594 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED)) in fanout_set_data_ebpf()
1605 __fanout_set_data_bpf(po->fanout, new); in fanout_set_data_ebpf()
1609 static int fanout_set_data(struct packet_sock *po, char __user *data, in fanout_set_data() argument
1612 switch (po->fanout->type) { in fanout_set_data()
1614 return fanout_set_data_cbpf(po, data, len); in fanout_set_data()
1616 return fanout_set_data_ebpf(po, data, len); in fanout_set_data()
1634 struct packet_sock *po = pkt_sk(sk); in fanout_add() local
1659 if (po->fanout) in fanout_add()
1697 match->prot_hook.type = po->prot_hook.type; in fanout_add()
1698 match->prot_hook.dev = po->prot_hook.dev; in fanout_add()
1706 spin_lock(&po->bind_lock); in fanout_add()
1707 if (po->running && in fanout_add()
1709 match->prot_hook.type == po->prot_hook.type && in fanout_add()
1710 match->prot_hook.dev == po->prot_hook.dev) { in fanout_add()
1713 __dev_remove_pack(&po->prot_hook); in fanout_add()
1714 po->fanout = match; in fanout_add()
1715 po->rollover = rollover; in fanout_add()
1718 __fanout_link(sk, po); in fanout_add()
1722 spin_unlock(&po->bind_lock); in fanout_add()
1742 struct packet_sock *po = pkt_sk(sk); in fanout_release() local
1746 f = po->fanout; in fanout_release()
1748 po->fanout = NULL; in fanout_release()
2034 struct packet_sock *po; in packet_rcv() local
2044 po = pkt_sk(sk); in packet_rcv()
2096 if (unlikely(po->origdev)) in packet_rcv()
2119 po->stats.stats1.tp_packets++; in packet_rcv()
2129 po->stats.stats1.tp_drops++; in packet_rcv()
2150 struct packet_sock *po; in tpacket_rcv() local
2175 po = pkt_sk(sk); in tpacket_rcv()
2206 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 + in tpacket_rcv()
2207 po->tp_reserve; in tpacket_rcv()
2210 netoff = TPACKET_ALIGN(po->tp_hdrlen + in tpacket_rcv()
2212 po->tp_reserve; in tpacket_rcv()
2213 if (po->has_vnet_hdr) { in tpacket_rcv()
2219 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
2220 if (macoff + snaplen > po->rx_ring.frame_size) { in tpacket_rcv()
2221 if (po->copy_thresh && in tpacket_rcv()
2232 snaplen = po->rx_ring.frame_size - macoff; in tpacket_rcv()
2239 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) { in tpacket_rcv()
2242 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff; in tpacket_rcv()
2248 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len; in tpacket_rcv()
2253 h.raw = packet_current_rx_frame(po, skb, in tpacket_rcv()
2257 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
2258 packet_increment_rx_head(po, &po->rx_ring); in tpacket_rcv()
2265 if (po->stats.stats1.tp_drops) in tpacket_rcv()
2268 po->stats.stats1.tp_packets++; in tpacket_rcv()
2285 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp))) in tpacket_rcv()
2290 switch (po->tp_version) { in tpacket_rcv()
2342 if (unlikely(po->origdev)) in tpacket_rcv()
2350 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
2362 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
2363 __packet_set_status(po, h.raw, status); in tpacket_rcv()
2366 prb_clear_blk_fill_status(&po->rx_ring); in tpacket_rcv()
2383 po->stats.stats1.tp_drops++; in tpacket_rcv()
2393 struct packet_sock *po = pkt_sk(skb->sk); in tpacket_destruct_skb() local
2395 if (likely(po->tx_ring.pg_vec)) { in tpacket_destruct_skb()
2400 packet_dec_pending(&po->tx_ring); in tpacket_destruct_skb()
2402 ts = __packet_set_timestamp(po, ph, skb); in tpacket_destruct_skb()
2403 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts); in tpacket_destruct_skb()
2496 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, in tpacket_fill_skb() argument
2503 struct socket *sock = po->sk.sk_socket; in tpacket_fill_skb()
2511 skb->priority = po->sk.sk_priority; in tpacket_fill_skb()
2512 skb->mark = po->sk.sk_mark; in tpacket_fill_skb()
2513 sock_tx_timestamp(&po->sk, sockc->tsflags, &skb_shinfo(skb)->tx_flags); in tpacket_fill_skb()
2550 atomic_add(to_write, &po->sk.sk_wmem_alloc); in tpacket_fill_skb()
2577 static int tpacket_parse_header(struct packet_sock *po, void *frame, in tpacket_parse_header() argument
2585 switch (po->tp_version) { in tpacket_parse_header()
2598 if (unlikely(po->tp_tx_has_off)) { in tpacket_parse_header()
2601 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll); in tpacket_parse_header()
2602 off_max = po->tx_ring.frame_size - tp_len; in tpacket_parse_header()
2603 if (po->sk.sk_type == SOCK_DGRAM) { in tpacket_parse_header()
2604 switch (po->tp_version) { in tpacket_parse_header()
2613 switch (po->tp_version) { in tpacket_parse_header()
2625 off = po->tp_hdrlen - sizeof(struct sockaddr_ll); in tpacket_parse_header()
2632 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) in tpacket_snd() argument
2650 mutex_lock(&po->pg_vec_lock); in tpacket_snd()
2653 dev = packet_cached_dev_get(po); in tpacket_snd()
2654 proto = po->num; in tpacket_snd()
2666 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex); in tpacket_snd()
2676 sockc.tsflags = po->sk.sk_tsflags; in tpacket_snd()
2678 err = sock_cmsg_send(&po->sk, msg, &sockc); in tpacket_snd()
2683 if (po->sk.sk_socket->type == SOCK_RAW) in tpacket_snd()
2685 size_max = po->tx_ring.frame_size in tpacket_snd()
2686 - (po->tp_hdrlen - sizeof(struct sockaddr_ll)); in tpacket_snd()
2688 if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !po->has_vnet_hdr) in tpacket_snd()
2692 ph = packet_current_frame(po, &po->tx_ring, in tpacket_snd()
2701 tp_len = tpacket_parse_header(po, ph, size_max, &data); in tpacket_snd()
2708 if (po->has_vnet_hdr) { in tpacket_snd()
2721 skb = sock_alloc_send_skb(&po->sk, in tpacket_snd()
2732 tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto, in tpacket_snd()
2736 !po->has_vnet_hdr && in tpacket_snd()
2742 if (po->tp_loss) { in tpacket_snd()
2743 __packet_set_status(po, ph, in tpacket_snd()
2745 packet_increment_head(&po->tx_ring); in tpacket_snd()
2755 if (po->has_vnet_hdr && packet_snd_vnet_gso(skb, vnet_hdr)) { in tpacket_snd()
2763 __packet_set_status(po, ph, TP_STATUS_SENDING); in tpacket_snd()
2764 packet_inc_pending(&po->tx_ring); in tpacket_snd()
2767 err = po->xmit(skb); in tpacket_snd()
2770 if (err && __packet_get_status(po, ph) == in tpacket_snd()
2782 packet_increment_head(&po->tx_ring); in tpacket_snd()
2791 (need_wait && packet_read_pending(&po->tx_ring)))); in tpacket_snd()
2797 __packet_set_status(po, ph, status); in tpacket_snd()
2802 mutex_unlock(&po->pg_vec_lock); in tpacket_snd()
2842 struct packet_sock *po = pkt_sk(sk); in packet_snd() local
2852 dev = packet_cached_dev_get(po); in packet_snd()
2853 proto = po->num; in packet_snd()
2883 if (po->has_vnet_hdr) { in packet_snd()
2960 err = po->xmit(skb); in packet_snd()
2980 struct packet_sock *po = pkt_sk(sk); in packet_sendmsg() local
2982 if (po->tx_ring.pg_vec) in packet_sendmsg()
2983 return tpacket_snd(po, msg); in packet_sendmsg()
2996 struct packet_sock *po; in packet_release() local
3005 po = pkt_sk(sk); in packet_release()
3015 spin_lock(&po->bind_lock); in packet_release()
3017 packet_cached_dev_reset(po); in packet_release()
3019 if (po->prot_hook.dev) { in packet_release()
3020 dev_put(po->prot_hook.dev); in packet_release()
3021 po->prot_hook.dev = NULL; in packet_release()
3023 spin_unlock(&po->bind_lock); in packet_release()
3028 if (po->rx_ring.pg_vec) { in packet_release()
3033 if (po->tx_ring.pg_vec) { in packet_release()
3044 kfree(po->rollover); in packet_release()
3057 packet_free_pending(po); in packet_release()
3071 struct packet_sock *po = pkt_sk(sk); in packet_do_bind() local
3080 spin_lock(&po->bind_lock); in packet_do_bind()
3083 if (po->fanout) { in packet_do_bind()
3105 proto_curr = po->prot_hook.type; in packet_do_bind()
3106 dev_curr = po->prot_hook.dev; in packet_do_bind()
3111 if (po->running) { in packet_do_bind()
3116 po->num = 0; in packet_do_bind()
3119 dev_curr = po->prot_hook.dev; in packet_do_bind()
3125 BUG_ON(po->running); in packet_do_bind()
3126 po->num = proto; in packet_do_bind()
3127 po->prot_hook.type = proto; in packet_do_bind()
3131 po->prot_hook.dev = NULL; in packet_do_bind()
3132 po->ifindex = -1; in packet_do_bind()
3133 packet_cached_dev_reset(po); in packet_do_bind()
3135 po->prot_hook.dev = dev; in packet_do_bind()
3136 po->ifindex = dev ? dev->ifindex : 0; in packet_do_bind()
3137 packet_cached_dev_assign(po, dev); in packet_do_bind()
3156 spin_unlock(&po->bind_lock); in packet_do_bind()
3218 struct packet_sock *po; in packet_create() local
3241 po = pkt_sk(sk); in packet_create()
3243 po->num = proto; in packet_create()
3244 po->xmit = dev_queue_xmit; in packet_create()
3246 err = packet_alloc_pending(po); in packet_create()
3250 packet_cached_dev_reset(po); in packet_create()
3259 spin_lock_init(&po->bind_lock); in packet_create()
3260 mutex_init(&po->pg_vec_lock); in packet_create()
3261 po->rollover = NULL; in packet_create()
3262 po->prot_hook.func = packet_rcv; in packet_create()
3265 po->prot_hook.func = packet_rcv_spkt; in packet_create()
3267 po->prot_hook.af_packet_priv = sk; in packet_create()
3270 po->prot_hook.type = proto; in packet_create()
3455 struct packet_sock *po = pkt_sk(sk); in packet_getname() local
3462 sll->sll_ifindex = po->ifindex; in packet_getname()
3463 sll->sll_protocol = po->num; in packet_getname()
3466 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex); in packet_getname()
3528 struct packet_sock *po = pkt_sk(sk); in packet_mc_add() local
3550 for (ml = po->mclist; ml; ml = ml->next) { in packet_mc_add()
3568 i->next = po->mclist; in packet_mc_add()
3569 po->mclist = i; in packet_mc_add()
3572 po->mclist = i->next; in packet_mc_add()
3609 struct packet_sock *po = pkt_sk(sk); in packet_flush_mclist() local
3612 if (!po->mclist) in packet_flush_mclist()
3616 while ((ml = po->mclist) != NULL) { in packet_flush_mclist()
3619 po->mclist = ml->next; in packet_flush_mclist()
3632 struct packet_sock *po = pkt_sk(sk); in packet_setsockopt() local
3667 switch (po->tp_version) { in packet_setsockopt()
3718 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { in packet_setsockopt()
3721 po->tp_version = val; in packet_setsockopt()
3738 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { in packet_setsockopt()
3741 po->tp_reserve = val; in packet_setsockopt()
3757 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { in packet_setsockopt()
3760 po->tp_loss = !!val; in packet_setsockopt()
3776 po->auxdata = !!val; in packet_setsockopt()
3790 po->origdev = !!val; in packet_setsockopt()
3806 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { in packet_setsockopt()
3809 po->has_vnet_hdr = !!val; in packet_setsockopt()
3824 po->tp_tstamp = val; in packet_setsockopt()
3840 if (!po->fanout) in packet_setsockopt()
3843 return fanout_set_data(po, optval, optlen); in packet_setsockopt()
3855 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { in packet_setsockopt()
3858 po->tp_tx_has_off = !!val; in packet_setsockopt()
3873 po->xmit = val ? packet_direct_xmit : dev_queue_xmit; in packet_setsockopt()
3887 struct packet_sock *po = pkt_sk(sk); in packet_getsockopt() local
3904 memcpy(&st, &po->stats, sizeof(st)); in packet_getsockopt()
3905 memset(&po->stats, 0, sizeof(po->stats)); in packet_getsockopt()
3908 if (po->tp_version == TPACKET_V3) { in packet_getsockopt()
3920 val = po->auxdata; in packet_getsockopt()
3923 val = po->origdev; in packet_getsockopt()
3926 val = po->has_vnet_hdr; in packet_getsockopt()
3929 val = po->tp_version; in packet_getsockopt()
3953 val = po->tp_reserve; in packet_getsockopt()
3956 val = po->tp_loss; in packet_getsockopt()
3959 val = po->tp_tstamp; in packet_getsockopt()
3962 val = (po->fanout ? in packet_getsockopt()
3963 ((u32)po->fanout->id | in packet_getsockopt()
3964 ((u32)po->fanout->type << 16) | in packet_getsockopt()
3965 ((u32)po->fanout->flags << 24)) : in packet_getsockopt()
3969 if (!po->rollover) in packet_getsockopt()
3971 rstats.tp_all = atomic_long_read(&po->rollover->num); in packet_getsockopt()
3972 rstats.tp_huge = atomic_long_read(&po->rollover->num_huge); in packet_getsockopt()
3973 rstats.tp_failed = atomic_long_read(&po->rollover->num_failed); in packet_getsockopt()
3978 val = po->tp_tx_has_off; in packet_getsockopt()
3981 val = packet_use_direct_xmit(po); in packet_getsockopt()
4001 struct packet_sock *po = pkt_sk(sock->sk); in compat_packet_setsockopt() local
4007 po->fanout && po->fanout->type == PACKET_FANOUT_CBPF) { in compat_packet_setsockopt()
4027 struct packet_sock *po = pkt_sk(sk); in packet_notifier() local
4031 if (po->mclist) in packet_notifier()
4032 packet_dev_mclist_delete(dev, &po->mclist); in packet_notifier()
4036 if (dev->ifindex == po->ifindex) { in packet_notifier()
4037 spin_lock(&po->bind_lock); in packet_notifier()
4038 if (po->running) { in packet_notifier()
4045 packet_cached_dev_reset(po); in packet_notifier()
4046 po->ifindex = -1; in packet_notifier()
4047 if (po->prot_hook.dev) in packet_notifier()
4048 dev_put(po->prot_hook.dev); in packet_notifier()
4049 po->prot_hook.dev = NULL; in packet_notifier()
4051 spin_unlock(&po->bind_lock); in packet_notifier()
4055 if (dev->ifindex == po->ifindex) { in packet_notifier()
4056 spin_lock(&po->bind_lock); in packet_notifier()
4057 if (po->num) in packet_notifier()
4059 spin_unlock(&po->bind_lock); in packet_notifier()
4126 struct packet_sock *po = pkt_sk(sk); in packet_poll() local
4130 if (po->rx_ring.pg_vec) { in packet_poll()
4131 if (!packet_previous_rx_frame(po, &po->rx_ring, in packet_poll()
4135 if (po->pressure && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL) in packet_poll()
4136 po->pressure = 0; in packet_poll()
4139 if (po->tx_ring.pg_vec) { in packet_poll()
4140 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE)) in packet_poll()
4249 struct packet_sock *po = pkt_sk(sk); in packet_set_ring() local
4259 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) { in packet_set_ring()
4264 rb = tx_ring ? &po->tx_ring : &po->rx_ring; in packet_set_ring()
4269 if (atomic_read(&po->mapped)) in packet_set_ring()
4281 switch (po->tp_version) { in packet_set_ring()
4283 po->tp_hdrlen = TPACKET_HDRLEN; in packet_set_ring()
4286 po->tp_hdrlen = TPACKET2_HDRLEN; in packet_set_ring()
4289 po->tp_hdrlen = TPACKET3_HDRLEN; in packet_set_ring()
4298 if (po->tp_version >= TPACKET_V3 && in packet_set_ring()
4302 if (unlikely(req->tp_frame_size < po->tp_hdrlen + in packet_set_ring()
4303 po->tp_reserve)) in packet_set_ring()
4322 switch (po->tp_version) { in packet_set_ring()
4328 init_prb_bdqc(po, rb, pg_vec, req_u); in packet_set_ring()
4343 spin_lock(&po->bind_lock); in packet_set_ring()
4344 was_running = po->running; in packet_set_ring()
4345 num = po->num; in packet_set_ring()
4347 po->num = 0; in packet_set_ring()
4350 spin_unlock(&po->bind_lock); in packet_set_ring()
4355 mutex_lock(&po->pg_vec_lock); in packet_set_ring()
4356 if (closing || atomic_read(&po->mapped) == 0) { in packet_set_ring()
4369 po->prot_hook.func = (po->rx_ring.pg_vec) ? in packet_set_ring()
4372 if (atomic_read(&po->mapped)) in packet_set_ring()
4374 atomic_read(&po->mapped)); in packet_set_ring()
4376 mutex_unlock(&po->pg_vec_lock); in packet_set_ring()
4378 spin_lock(&po->bind_lock); in packet_set_ring()
4380 po->num = num; in packet_set_ring()
4383 spin_unlock(&po->bind_lock); in packet_set_ring()
4384 if (pg_vec && (po->tp_version > TPACKET_V2)) { in packet_set_ring()
4387 prb_shutdown_retire_blk_timer(po, rb_queue); in packet_set_ring()
4400 struct packet_sock *po = pkt_sk(sk); in packet_mmap() local
4410 mutex_lock(&po->pg_vec_lock); in packet_mmap()
4413 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) { in packet_mmap()
4429 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) { in packet_mmap()
4449 atomic_inc(&po->mapped); in packet_mmap()
4454 mutex_unlock(&po->pg_vec_lock); in packet_mmap()
4542 const struct packet_sock *po = pkt_sk(s); in packet_seq_show() local
4549 ntohs(po->num), in packet_seq_show()
4550 po->ifindex, in packet_seq_show()
4551 po->running, in packet_seq_show()