92 static void rds_recv_rcvbuf_delta(struct rds_sock *rs, struct sock *sk,  in rds_recv_rcvbuf_delta()  argument
101 	rs->rs_rcv_bytes += delta;  in rds_recv_rcvbuf_delta()
108 	if (rs->rs_transport->t_type == RDS_TRANS_LOOP)  in rds_recv_rcvbuf_delta()
111 	now_congested = rs->rs_rcv_bytes > rds_sk_rcvbuf(rs);  in rds_recv_rcvbuf_delta()
113 	rdsdebug("rs %p (%pI6c:%u) recv bytes %d buf %d "  in rds_recv_rcvbuf_delta()
115 	  rs, &rs->rs_bound_addr,  in rds_recv_rcvbuf_delta()
116 	  ntohs(rs->rs_bound_port), rs->rs_rcv_bytes,  in rds_recv_rcvbuf_delta()
117 	  rds_sk_rcvbuf(rs), now_congested, delta);  in rds_recv_rcvbuf_delta()
120 	if (!rs->rs_congested && now_congested) {  in rds_recv_rcvbuf_delta()
121 		rs->rs_congested = 1;  in rds_recv_rcvbuf_delta()
128 	else if (rs->rs_congested && (rs->rs_rcv_bytes < (rds_sk_rcvbuf(rs)/2))) {  in rds_recv_rcvbuf_delta()
129 		rs->rs_congested = 0;  in rds_recv_rcvbuf_delta()
170 static void rds_recv_incoming_exthdrs(struct rds_incoming *inc, struct rds_sock *rs)  in rds_recv_incoming_exthdrs()  argument
188 			rds_rdma_unuse(rs, be32_to_cpu(buffer.rdma.h_rdma_rkey), 0);  in rds_recv_incoming_exthdrs()
291 	struct rds_sock *rs = NULL;  in rds_recv_incoming()  local
367 	rs = rds_find_bound(daddr, inc->i_hdr.h_dport, conn->c_bound_if);  in rds_recv_incoming()
368 	if (!rs) {  in rds_recv_incoming()
374 	rds_recv_incoming_exthdrs(inc, rs);  in rds_recv_incoming()
377 	sk = rds_rs_to_sk(rs);  in rds_recv_incoming()
380 	write_lock_irqsave(&rs->rs_recv_lock, flags);  in rds_recv_incoming()
382 		rdsdebug("adding inc %p to rs %p's recv queue\n", inc, rs);  in rds_recv_incoming()
384 		rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,  in rds_recv_incoming()
391 		list_add_tail(&inc->i_item, &rs->rs_recv_queue);  in rds_recv_incoming()
396 	write_unlock_irqrestore(&rs->rs_recv_lock, flags);  in rds_recv_incoming()
399 	if (rs)  in rds_recv_incoming()
400 		rds_sock_put(rs);  in rds_recv_incoming()
408 static int rds_next_incoming(struct rds_sock *rs, struct rds_incoming **inc)  in rds_next_incoming()  argument
413 		read_lock_irqsave(&rs->rs_recv_lock, flags);  in rds_next_incoming()
414 		if (!list_empty(&rs->rs_recv_queue)) {  in rds_next_incoming()
415 			*inc = list_entry(rs->rs_recv_queue.next,  in rds_next_incoming()
420 		read_unlock_irqrestore(&rs->rs_recv_lock, flags);  in rds_next_incoming()
426 static int rds_still_queued(struct rds_sock *rs, struct rds_incoming *inc,  in rds_still_queued()  argument
429 	struct sock *sk = rds_rs_to_sk(rs);  in rds_still_queued()
433 	write_lock_irqsave(&rs->rs_recv_lock, flags);  in rds_still_queued()
438 			rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,  in rds_still_queued()
445 	write_unlock_irqrestore(&rs->rs_recv_lock, flags);  in rds_still_queued()
447 	rdsdebug("inc %p rs %p still %d dropped %d\n", inc, rs, ret, drop);  in rds_still_queued()
455 int rds_notify_queue_get(struct rds_sock *rs, struct msghdr *msghdr)  in rds_notify_queue_get()  argument
479 	spin_lock_irqsave(&rs->rs_lock, flags);  in rds_notify_queue_get()
480 	while (!list_empty(&rs->rs_notify_queue) && count < max_messages) {  in rds_notify_queue_get()
481 		notifier = list_entry(rs->rs_notify_queue.next,  in rds_notify_queue_get()
486 	spin_unlock_irqrestore(&rs->rs_lock, flags);  in rds_notify_queue_get()
512 		spin_lock_irqsave(&rs->rs_lock, flags);  in rds_notify_queue_get()
513 		list_splice(&copy, &rs->rs_notify_queue);  in rds_notify_queue_get()
514 		spin_unlock_irqrestore(&rs->rs_lock, flags);  in rds_notify_queue_get()
523 static int rds_notify_cong(struct rds_sock *rs, struct msghdr *msghdr)  in rds_notify_cong()  argument
525 	uint64_t notify = rs->rs_cong_notify;  in rds_notify_cong()
534 	spin_lock_irqsave(&rs->rs_lock, flags);  in rds_notify_cong()
535 	rs->rs_cong_notify &= ~notify;  in rds_notify_cong()
536 	spin_unlock_irqrestore(&rs->rs_lock, flags);  in rds_notify_cong()
545 			 struct rds_sock *rs)  in rds_cmsg_recv()  argument
557 	    sock_flag(rds_rs_to_sk(rs), SOCK_RCVTSTAMP)) {  in rds_cmsg_recv()
565 	if (rs->rs_rx_traces) {  in rds_cmsg_recv()
571 		t.rx_traces =  rs->rs_rx_traces;  in rds_cmsg_recv()
572 		for (i = 0; i < rs->rs_rx_traces; i++) {  in rds_cmsg_recv()
573 			j = rs->rs_rx_trace[i];  in rds_cmsg_recv()
589 static bool rds_recvmsg_zcookie(struct rds_sock *rs, struct msghdr *msg)  in rds_recvmsg_zcookie()  argument
591 	struct rds_msg_zcopy_queue *q = &rs->rs_zcookie_queue;  in rds_recvmsg_zcookie()
599 	if (!sock_flag(rds_rs_to_sk(rs), SOCK_ZEROCOPY) ||  in rds_recvmsg_zcookie()
628 	struct rds_sock *rs = rds_sk_to_rs(sk);  in rds_recvmsg()  local
647 		if (!list_empty(&rs->rs_notify_queue)) {  in rds_recvmsg()
648 			ret = rds_notify_queue_get(rs, msg);  in rds_recvmsg()
652 		if (rs->rs_cong_notify) {  in rds_recvmsg()
653 			ret = rds_notify_cong(rs, msg);  in rds_recvmsg()
657 		if (!rds_next_incoming(rs, &inc)) {  in rds_recvmsg()
659 				bool reaped = rds_recvmsg_zcookie(rs, msg);  in rds_recvmsg()
666 					(!list_empty(&rs->rs_notify_queue) ||  in rds_recvmsg()
667 					 rs->rs_cong_notify ||  in rds_recvmsg()
668 					 rds_next_incoming(rs, &inc)), timeo);  in rds_recvmsg()
692 		if (!rds_still_queued(rs, inc, !(msg_flags & MSG_PEEK))) {  in rds_recvmsg()
706 		if (rds_cmsg_recv(inc, msg, rs)) {  in rds_recvmsg()
710 		rds_recvmsg_zcookie(rs, msg);  in rds_recvmsg()
731 				sin6->sin6_scope_id = rs->rs_bound_scope_id;  in rds_recvmsg()
750 void rds_clear_recv_queue(struct rds_sock *rs)  in rds_clear_recv_queue()  argument
752 	struct sock *sk = rds_rs_to_sk(rs);  in rds_clear_recv_queue()
756 	write_lock_irqsave(&rs->rs_recv_lock, flags);  in rds_clear_recv_queue()
757 	list_for_each_entry_safe(inc, tmp, &rs->rs_recv_queue, i_item) {  in rds_clear_recv_queue()
758 		rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,  in rds_clear_recv_queue()
764 	write_unlock_irqrestore(&rs->rs_recv_lock, flags);  in rds_clear_recv_queue()