Lines Matching refs:psock
53 static void kcm_abort_tx_psock(struct kcm_psock *psock, int err, in kcm_abort_tx_psock() argument
56 struct sock *csk = psock->sk; in kcm_abort_tx_psock()
57 struct kcm_mux *mux = psock->mux; in kcm_abort_tx_psock()
63 if (psock->tx_stopped) { in kcm_abort_tx_psock()
68 psock->tx_stopped = 1; in kcm_abort_tx_psock()
69 KCM_STATS_INCR(psock->stats.tx_aborts); in kcm_abort_tx_psock()
71 if (!psock->tx_kcm) { in kcm_abort_tx_psock()
73 list_del(&psock->psock_avail_list); in kcm_abort_tx_psock()
82 queue_work(kcm_wq, &psock->tx_kcm->tx_work); in kcm_abort_tx_psock()
93 struct kcm_psock *psock) in kcm_update_rx_mux_stats() argument
96 psock->strp.stats.bytes - in kcm_update_rx_mux_stats()
97 psock->saved_rx_bytes); in kcm_update_rx_mux_stats()
99 psock->strp.stats.msgs - psock->saved_rx_msgs; in kcm_update_rx_mux_stats()
100 psock->saved_rx_msgs = psock->strp.stats.msgs; in kcm_update_rx_mux_stats()
101 psock->saved_rx_bytes = psock->strp.stats.bytes; in kcm_update_rx_mux_stats()
105 struct kcm_psock *psock) in kcm_update_tx_mux_stats() argument
108 psock->stats.tx_bytes - psock->saved_tx_bytes); in kcm_update_tx_mux_stats()
110 psock->stats.tx_msgs - psock->saved_tx_msgs; in kcm_update_tx_mux_stats()
111 psock->saved_tx_msgs = psock->stats.tx_msgs; in kcm_update_tx_mux_stats()
112 psock->saved_tx_bytes = psock->stats.tx_bytes; in kcm_update_tx_mux_stats()
124 struct kcm_psock *psock; in kcm_rcv_ready() local
140 psock = list_first_entry(&mux->psocks_ready, struct kcm_psock, in kcm_rcv_ready()
143 if (kcm_queue_rcv_skb(&kcm->sk, psock->ready_rx_msg)) { in kcm_rcv_ready()
152 list_del(&psock->psock_ready_list); in kcm_rcv_ready()
153 psock->ready_rx_msg = NULL; in kcm_rcv_ready()
157 strp_unpause(&psock->strp); in kcm_rcv_ready()
158 strp_check_rcv(&psock->strp); in kcm_rcv_ready()
250 static struct kcm_sock *reserve_rx_kcm(struct kcm_psock *psock, in reserve_rx_kcm() argument
253 struct kcm_mux *mux = psock->mux; in reserve_rx_kcm()
256 WARN_ON(psock->ready_rx_msg); in reserve_rx_kcm()
258 if (psock->rx_kcm) in reserve_rx_kcm()
259 return psock->rx_kcm; in reserve_rx_kcm()
263 if (psock->rx_kcm) { in reserve_rx_kcm()
265 return psock->rx_kcm; in reserve_rx_kcm()
268 kcm_update_rx_mux_stats(mux, psock); in reserve_rx_kcm()
271 psock->ready_rx_msg = head; in reserve_rx_kcm()
272 strp_pause(&psock->strp); in reserve_rx_kcm()
273 list_add_tail(&psock->psock_ready_list, in reserve_rx_kcm()
284 psock->rx_kcm = kcm; in reserve_rx_kcm()
285 kcm->rx_psock = psock; in reserve_rx_kcm()
300 static void unreserve_rx_kcm(struct kcm_psock *psock, in unreserve_rx_kcm() argument
303 struct kcm_sock *kcm = psock->rx_kcm; in unreserve_rx_kcm()
304 struct kcm_mux *mux = psock->mux; in unreserve_rx_kcm()
311 psock->rx_kcm = NULL; in unreserve_rx_kcm()
344 struct kcm_psock *psock; in psock_data_ready() local
348 psock = (struct kcm_psock *)sk->sk_user_data; in psock_data_ready()
349 if (likely(psock)) in psock_data_ready()
350 strp_data_ready(&psock->strp); in psock_data_ready()
358 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp); in kcm_rcv_strparser() local
362 kcm = reserve_rx_kcm(psock, skb); in kcm_rcv_strparser()
372 unreserve_rx_kcm(psock, false); in kcm_rcv_strparser()
379 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp); in kcm_parse_func_strparser() local
380 struct bpf_prog *prog = psock->bpf_prog; in kcm_parse_func_strparser()
389 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp); in kcm_read_sock_done() local
391 unreserve_rx_kcm(psock, true); in kcm_read_sock_done()
408 struct kcm_psock *psock; in psock_write_space() local
414 psock = (struct kcm_psock *)sk->sk_user_data; in psock_write_space()
415 if (unlikely(!psock)) in psock_write_space()
417 mux = psock->mux; in psock_write_space()
422 kcm = psock->tx_kcm; in psock_write_space()
437 struct kcm_psock *psock; in reserve_psock() local
439 psock = kcm->tx_psock; in reserve_psock()
443 if (psock) { in reserve_psock()
445 if (unlikely(psock->tx_stopped)) in reserve_psock()
456 psock = kcm->tx_psock; in reserve_psock()
457 if (unlikely(psock)) { in reserve_psock()
464 psock = list_first_entry(&mux->psocks_avail, in reserve_psock()
467 list_del(&psock->psock_avail_list); in reserve_psock()
472 kcm->tx_psock = psock; in reserve_psock()
473 psock->tx_kcm = kcm; in reserve_psock()
474 KCM_STATS_INCR(psock->stats.reserved); in reserve_psock()
483 return psock; in reserve_psock()
487 static void psock_now_avail(struct kcm_psock *psock) in psock_now_avail() argument
489 struct kcm_mux *mux = psock->mux; in psock_now_avail()
493 list_add_tail(&psock->psock_avail_list, in psock_now_avail()
501 psock->tx_kcm = kcm; in psock_now_avail()
508 kcm->tx_psock = psock; in psock_now_avail()
509 KCM_STATS_INCR(psock->stats.reserved); in psock_now_avail()
517 struct kcm_psock *psock; in unreserve_psock() local
522 psock = kcm->tx_psock; in unreserve_psock()
524 if (WARN_ON(!psock)) { in unreserve_psock()
531 kcm_update_tx_mux_stats(mux, psock); in unreserve_psock()
536 psock->tx_kcm = NULL; in unreserve_psock()
537 KCM_STATS_INCR(psock->stats.unreserved); in unreserve_psock()
539 if (unlikely(psock->tx_stopped)) { in unreserve_psock()
540 if (psock->done) { in unreserve_psock()
542 list_del(&psock->psock_list); in unreserve_psock()
544 sock_put(psock->sk); in unreserve_psock()
545 fput(psock->sk->sk_socket->file); in unreserve_psock()
546 kmem_cache_free(kcm_psockp, psock); in unreserve_psock()
556 psock_now_avail(psock); in unreserve_psock()
576 struct kcm_psock *psock; in kcm_write_msgs() local
584 psock = kcm->tx_psock; in kcm_write_msgs()
585 if (unlikely(psock && psock->tx_stopped)) { in kcm_write_msgs()
605 if (WARN_ON(!psock)) { in kcm_write_msgs()
618 psock = reserve_psock(kcm); in kcm_write_msgs()
619 if (!psock) in kcm_write_msgs()
645 ret = kernel_sendpage(psock->sk->sk_socket, in kcm_write_msgs()
669 kcm_abort_tx_psock(psock, ret ? -ret : EPIPE, in kcm_write_msgs()
682 KCM_STATS_ADD(psock->stats.tx_bytes, ret); in kcm_write_msgs()
704 KCM_STATS_INCR(psock->stats.tx_msgs); in kcm_write_msgs()
1375 struct kcm_psock *psock = NULL, *tpsock; in kcm_attach() local
1404 psock = kmem_cache_zalloc(kcm_psockp, GFP_KERNEL); in kcm_attach()
1405 if (!psock) { in kcm_attach()
1410 psock->mux = mux; in kcm_attach()
1411 psock->sk = csk; in kcm_attach()
1412 psock->bpf_prog = prog; in kcm_attach()
1414 err = strp_init(&psock->strp, csk, &cb); in kcm_attach()
1416 kmem_cache_free(kcm_psockp, psock); in kcm_attach()
1427 strp_stop(&psock->strp); in kcm_attach()
1428 strp_done(&psock->strp); in kcm_attach()
1429 kmem_cache_free(kcm_psockp, psock); in kcm_attach()
1434 psock->save_data_ready = csk->sk_data_ready; in kcm_attach()
1435 psock->save_write_space = csk->sk_write_space; in kcm_attach()
1436 psock->save_state_change = csk->sk_state_change; in kcm_attach()
1437 csk->sk_user_data = psock; in kcm_attach()
1456 list_add(&psock->psock_list, head); in kcm_attach()
1457 psock->index = index; in kcm_attach()
1461 psock_now_avail(psock); in kcm_attach()
1465 strp_check_rcv(&psock->strp); in kcm_attach()
1503 static void kcm_unattach(struct kcm_psock *psock) in kcm_unattach() argument
1505 struct sock *csk = psock->sk; in kcm_unattach()
1506 struct kcm_mux *mux = psock->mux; in kcm_unattach()
1515 csk->sk_data_ready = psock->save_data_ready; in kcm_unattach()
1516 csk->sk_write_space = psock->save_write_space; in kcm_unattach()
1517 csk->sk_state_change = psock->save_state_change; in kcm_unattach()
1518 strp_stop(&psock->strp); in kcm_unattach()
1520 if (WARN_ON(psock->rx_kcm)) { in kcm_unattach()
1531 if (psock->ready_rx_msg) { in kcm_unattach()
1532 list_del(&psock->psock_ready_list); in kcm_unattach()
1533 kfree_skb(psock->ready_rx_msg); in kcm_unattach()
1534 psock->ready_rx_msg = NULL; in kcm_unattach()
1544 strp_done(&psock->strp); in kcm_unattach()
1547 bpf_prog_put(psock->bpf_prog); in kcm_unattach()
1551 aggregate_psock_stats(&psock->stats, &mux->aggregate_psock_stats); in kcm_unattach()
1552 save_strp_stats(&psock->strp, &mux->aggregate_strp_stats); in kcm_unattach()
1556 if (psock->tx_kcm) { in kcm_unattach()
1568 kcm_abort_tx_psock(psock, EPIPE, false); in kcm_unattach()
1571 if (!psock->tx_kcm) { in kcm_unattach()
1575 psock->done = 1; in kcm_unattach()
1581 queue_work(kcm_wq, &psock->tx_kcm->tx_work); in kcm_unattach()
1585 if (!psock->tx_stopped) in kcm_unattach()
1586 list_del(&psock->psock_avail_list); in kcm_unattach()
1587 list_del(&psock->psock_list); in kcm_unattach()
1593 kmem_cache_free(kcm_psockp, psock); in kcm_unattach()
1603 struct kcm_psock *psock; in kcm_unattach_ioctl() local
1622 list_for_each_entry(psock, &mux->psocks, psock_list) { in kcm_unattach_ioctl()
1623 if (psock->sk != csk) in kcm_unattach_ioctl()
1628 if (psock->unattaching || WARN_ON(psock->done)) { in kcm_unattach_ioctl()
1633 psock->unattaching = 1; in kcm_unattach_ioctl()
1638 kcm_unattach(psock); in kcm_unattach_ioctl()
1751 struct kcm_psock *psock, *tmp_psock; in release_mux() local
1754 list_for_each_entry_safe(psock, tmp_psock, in release_mux()
1756 if (!WARN_ON(psock->unattaching)) in release_mux()
1757 kcm_unattach(psock); in release_mux()
1833 struct kcm_psock *psock; in kcm_release() local
1875 psock = kcm->tx_psock; in kcm_release()
1876 if (psock) { in kcm_release()
1881 kcm_abort_tx_psock(psock, EPIPE, false); in kcm_release()