Lines Matching refs:xprt
70 static void xprt_init(struct rpc_xprt *xprt, struct net *net);
71 static __be32 xprt_alloc_xid(struct rpc_xprt *xprt);
72 static void xprt_destroy(struct rpc_xprt *xprt);
216 static void xprt_clear_locked(struct rpc_xprt *xprt) in xprt_clear_locked() argument
218 xprt->snd_task = NULL; in xprt_clear_locked()
219 if (!test_bit(XPRT_CLOSE_WAIT, &xprt->state)) { in xprt_clear_locked()
221 clear_bit(XPRT_LOCKED, &xprt->state); in xprt_clear_locked()
224 queue_work(xprtiod_workqueue, &xprt->task_cleanup); in xprt_clear_locked()
236 int xprt_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task) in xprt_reserve_xprt() argument
240 if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) { in xprt_reserve_xprt()
241 if (task == xprt->snd_task) in xprt_reserve_xprt()
245 if (test_bit(XPRT_WRITE_SPACE, &xprt->state)) in xprt_reserve_xprt()
247 xprt->snd_task = task; in xprt_reserve_xprt()
250 trace_xprt_reserve_xprt(xprt, task); in xprt_reserve_xprt()
254 xprt_clear_locked(xprt); in xprt_reserve_xprt()
258 rpc_sleep_on_timeout(&xprt->sending, task, NULL, in xprt_reserve_xprt()
261 rpc_sleep_on(&xprt->sending, task, NULL); in xprt_reserve_xprt()
267 xprt_need_congestion_window_wait(struct rpc_xprt *xprt) in xprt_need_congestion_window_wait() argument
269 return test_bit(XPRT_CWND_WAIT, &xprt->state); in xprt_need_congestion_window_wait()
273 xprt_set_congestion_window_wait(struct rpc_xprt *xprt) in xprt_set_congestion_window_wait() argument
275 if (!list_empty(&xprt->xmit_queue)) { in xprt_set_congestion_window_wait()
277 if (list_first_entry(&xprt->xmit_queue, struct rpc_rqst, in xprt_set_congestion_window_wait()
281 set_bit(XPRT_CWND_WAIT, &xprt->state); in xprt_set_congestion_window_wait()
285 xprt_test_and_clear_congestion_window_wait(struct rpc_xprt *xprt) in xprt_test_and_clear_congestion_window_wait() argument
287 if (!RPCXPRT_CONGESTED(xprt)) in xprt_test_and_clear_congestion_window_wait()
288 clear_bit(XPRT_CWND_WAIT, &xprt->state); in xprt_test_and_clear_congestion_window_wait()
300 int xprt_reserve_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task) in xprt_reserve_xprt_cong() argument
304 if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) { in xprt_reserve_xprt_cong()
305 if (task == xprt->snd_task) in xprt_reserve_xprt_cong()
310 xprt->snd_task = task; in xprt_reserve_xprt_cong()
313 if (test_bit(XPRT_WRITE_SPACE, &xprt->state)) in xprt_reserve_xprt_cong()
315 if (!xprt_need_congestion_window_wait(xprt)) { in xprt_reserve_xprt_cong()
316 xprt->snd_task = task; in xprt_reserve_xprt_cong()
320 xprt_clear_locked(xprt); in xprt_reserve_xprt_cong()
324 rpc_sleep_on_timeout(&xprt->sending, task, NULL, in xprt_reserve_xprt_cong()
327 rpc_sleep_on(&xprt->sending, task, NULL); in xprt_reserve_xprt_cong()
330 trace_xprt_reserve_cong(xprt, task); in xprt_reserve_xprt_cong()
335 static inline int xprt_lock_write(struct rpc_xprt *xprt, struct rpc_task *task) in xprt_lock_write() argument
339 if (test_bit(XPRT_LOCKED, &xprt->state) && xprt->snd_task == task) in xprt_lock_write()
341 spin_lock(&xprt->transport_lock); in xprt_lock_write()
342 retval = xprt->ops->reserve_xprt(xprt, task); in xprt_lock_write()
343 spin_unlock(&xprt->transport_lock); in xprt_lock_write()
349 struct rpc_xprt *xprt = data; in __xprt_lock_write_func() local
351 xprt->snd_task = task; in __xprt_lock_write_func()
355 static void __xprt_lock_write_next(struct rpc_xprt *xprt) in __xprt_lock_write_next() argument
357 if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) in __xprt_lock_write_next()
359 if (test_bit(XPRT_WRITE_SPACE, &xprt->state)) in __xprt_lock_write_next()
361 if (rpc_wake_up_first_on_wq(xprtiod_workqueue, &xprt->sending, in __xprt_lock_write_next()
362 __xprt_lock_write_func, xprt)) in __xprt_lock_write_next()
365 xprt_clear_locked(xprt); in __xprt_lock_write_next()
368 static void __xprt_lock_write_next_cong(struct rpc_xprt *xprt) in __xprt_lock_write_next_cong() argument
370 if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) in __xprt_lock_write_next_cong()
372 if (test_bit(XPRT_WRITE_SPACE, &xprt->state)) in __xprt_lock_write_next_cong()
374 if (xprt_need_congestion_window_wait(xprt)) in __xprt_lock_write_next_cong()
376 if (rpc_wake_up_first_on_wq(xprtiod_workqueue, &xprt->sending, in __xprt_lock_write_next_cong()
377 __xprt_lock_write_func, xprt)) in __xprt_lock_write_next_cong()
380 xprt_clear_locked(xprt); in __xprt_lock_write_next_cong()
390 void xprt_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task) in xprt_release_xprt() argument
392 if (xprt->snd_task == task) { in xprt_release_xprt()
393 xprt_clear_locked(xprt); in xprt_release_xprt()
394 __xprt_lock_write_next(xprt); in xprt_release_xprt()
396 trace_xprt_release_xprt(xprt, task); in xprt_release_xprt()
408 void xprt_release_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task) in xprt_release_xprt_cong() argument
410 if (xprt->snd_task == task) { in xprt_release_xprt_cong()
411 xprt_clear_locked(xprt); in xprt_release_xprt_cong()
412 __xprt_lock_write_next_cong(xprt); in xprt_release_xprt_cong()
414 trace_xprt_release_cong(xprt, task); in xprt_release_xprt_cong()
418 static inline void xprt_release_write(struct rpc_xprt *xprt, struct rpc_task *task) in xprt_release_write() argument
420 if (xprt->snd_task != task) in xprt_release_write()
422 spin_lock(&xprt->transport_lock); in xprt_release_write()
423 xprt->ops->release_xprt(xprt, task); in xprt_release_write()
424 spin_unlock(&xprt->transport_lock); in xprt_release_write()
432 __xprt_get_cong(struct rpc_xprt *xprt, struct rpc_rqst *req) in __xprt_get_cong() argument
436 trace_xprt_get_cong(xprt, req->rq_task); in __xprt_get_cong()
437 if (RPCXPRT_CONGESTED(xprt)) { in __xprt_get_cong()
438 xprt_set_congestion_window_wait(xprt); in __xprt_get_cong()
442 xprt->cong += RPC_CWNDSCALE; in __xprt_get_cong()
451 __xprt_put_cong(struct rpc_xprt *xprt, struct rpc_rqst *req) in __xprt_put_cong() argument
456 xprt->cong -= RPC_CWNDSCALE; in __xprt_put_cong()
457 xprt_test_and_clear_congestion_window_wait(xprt); in __xprt_put_cong()
458 trace_xprt_put_cong(xprt, req->rq_task); in __xprt_put_cong()
459 __xprt_lock_write_next_cong(xprt); in __xprt_put_cong()
470 xprt_request_get_cong(struct rpc_xprt *xprt, struct rpc_rqst *req) in xprt_request_get_cong() argument
476 spin_lock(&xprt->transport_lock); in xprt_request_get_cong()
477 ret = __xprt_get_cong(xprt, req) != 0; in xprt_request_get_cong()
478 spin_unlock(&xprt->transport_lock); in xprt_request_get_cong()
497 static void xprt_clear_congestion_window_wait_locked(struct rpc_xprt *xprt) in xprt_clear_congestion_window_wait_locked() argument
499 if (test_and_clear_bit(XPRT_CWND_WAIT, &xprt->state)) in xprt_clear_congestion_window_wait_locked()
500 __xprt_lock_write_next_cong(xprt); in xprt_clear_congestion_window_wait_locked()
508 xprt_clear_congestion_window_wait(struct rpc_xprt *xprt) in xprt_clear_congestion_window_wait() argument
510 if (test_and_clear_bit(XPRT_CWND_WAIT, &xprt->state)) { in xprt_clear_congestion_window_wait()
511 spin_lock(&xprt->transport_lock); in xprt_clear_congestion_window_wait()
512 __xprt_lock_write_next_cong(xprt); in xprt_clear_congestion_window_wait()
513 spin_unlock(&xprt->transport_lock); in xprt_clear_congestion_window_wait()
533 void xprt_adjust_cwnd(struct rpc_xprt *xprt, struct rpc_task *task, int result) in xprt_adjust_cwnd() argument
536 unsigned long cwnd = xprt->cwnd; in xprt_adjust_cwnd()
538 if (result >= 0 && cwnd <= xprt->cong) { in xprt_adjust_cwnd()
542 if (cwnd > RPC_MAXCWND(xprt)) in xprt_adjust_cwnd()
543 cwnd = RPC_MAXCWND(xprt); in xprt_adjust_cwnd()
544 __xprt_lock_write_next_cong(xprt); in xprt_adjust_cwnd()
551 xprt->cong, xprt->cwnd, cwnd); in xprt_adjust_cwnd()
552 xprt->cwnd = cwnd; in xprt_adjust_cwnd()
553 __xprt_put_cong(xprt, req); in xprt_adjust_cwnd()
563 void xprt_wake_pending_tasks(struct rpc_xprt *xprt, int status) in xprt_wake_pending_tasks() argument
566 rpc_wake_up_status(&xprt->pending, status); in xprt_wake_pending_tasks()
568 rpc_wake_up(&xprt->pending); in xprt_wake_pending_tasks()
580 void xprt_wait_for_buffer_space(struct rpc_xprt *xprt) in xprt_wait_for_buffer_space() argument
582 set_bit(XPRT_WRITE_SPACE, &xprt->state); in xprt_wait_for_buffer_space()
587 xprt_clear_write_space_locked(struct rpc_xprt *xprt) in xprt_clear_write_space_locked() argument
589 if (test_and_clear_bit(XPRT_WRITE_SPACE, &xprt->state)) { in xprt_clear_write_space_locked()
590 __xprt_lock_write_next(xprt); in xprt_clear_write_space_locked()
592 "xprt %p\n", xprt); in xprt_clear_write_space_locked()
604 bool xprt_write_space(struct rpc_xprt *xprt) in xprt_write_space() argument
608 if (!test_bit(XPRT_WRITE_SPACE, &xprt->state)) in xprt_write_space()
610 spin_lock(&xprt->transport_lock); in xprt_write_space()
611 ret = xprt_clear_write_space_locked(xprt); in xprt_write_space()
612 spin_unlock(&xprt->transport_lock); in xprt_write_space()
652 struct rpc_xprt *xprt = req->rq_xprt; in xprt_init_majortimeo() local
654 if (likely(xprt && xprt_connected(xprt))) in xprt_init_majortimeo()
670 struct rpc_xprt *xprt = req->rq_xprt; in xprt_adjust_timeout() local
689 spin_lock(&xprt->transport_lock); in xprt_adjust_timeout()
691 spin_unlock(&xprt->transport_lock); in xprt_adjust_timeout()
705 struct rpc_xprt *xprt = in xprt_autoclose() local
709 trace_xprt_disconnect_auto(xprt); in xprt_autoclose()
710 clear_bit(XPRT_CLOSE_WAIT, &xprt->state); in xprt_autoclose()
711 xprt->ops->close(xprt); in xprt_autoclose()
712 xprt_release_write(xprt, NULL); in xprt_autoclose()
713 wake_up_bit(&xprt->state, XPRT_LOCKED); in xprt_autoclose()
722 void xprt_disconnect_done(struct rpc_xprt *xprt) in xprt_disconnect_done() argument
724 trace_xprt_disconnect_done(xprt); in xprt_disconnect_done()
725 spin_lock(&xprt->transport_lock); in xprt_disconnect_done()
726 xprt_clear_connected(xprt); in xprt_disconnect_done()
727 xprt_clear_write_space_locked(xprt); in xprt_disconnect_done()
728 xprt_clear_congestion_window_wait_locked(xprt); in xprt_disconnect_done()
729 xprt_wake_pending_tasks(xprt, -ENOTCONN); in xprt_disconnect_done()
730 spin_unlock(&xprt->transport_lock); in xprt_disconnect_done()
739 void xprt_force_disconnect(struct rpc_xprt *xprt) in xprt_force_disconnect() argument
741 trace_xprt_disconnect_force(xprt); in xprt_force_disconnect()
744 spin_lock(&xprt->transport_lock); in xprt_force_disconnect()
745 set_bit(XPRT_CLOSE_WAIT, &xprt->state); in xprt_force_disconnect()
747 if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0) in xprt_force_disconnect()
748 queue_work(xprtiod_workqueue, &xprt->task_cleanup); in xprt_force_disconnect()
749 else if (xprt->snd_task && !test_bit(XPRT_SND_IS_COOKIE, &xprt->state)) in xprt_force_disconnect()
750 rpc_wake_up_queued_task_set_status(&xprt->pending, in xprt_force_disconnect()
751 xprt->snd_task, -ENOTCONN); in xprt_force_disconnect()
752 spin_unlock(&xprt->transport_lock); in xprt_force_disconnect()
757 xprt_connect_cookie(struct rpc_xprt *xprt) in xprt_connect_cookie() argument
759 return READ_ONCE(xprt->connect_cookie); in xprt_connect_cookie()
766 struct rpc_xprt *xprt = req->rq_xprt; in xprt_request_retransmit_after_disconnect() local
768 return req->rq_connect_cookie != xprt_connect_cookie(xprt) || in xprt_request_retransmit_after_disconnect()
769 !xprt_connected(xprt); in xprt_request_retransmit_after_disconnect()
783 void xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie) in xprt_conditional_disconnect() argument
786 spin_lock(&xprt->transport_lock); in xprt_conditional_disconnect()
787 if (cookie != xprt->connect_cookie) in xprt_conditional_disconnect()
789 if (test_bit(XPRT_CLOSING, &xprt->state)) in xprt_conditional_disconnect()
791 set_bit(XPRT_CLOSE_WAIT, &xprt->state); in xprt_conditional_disconnect()
793 if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0) in xprt_conditional_disconnect()
794 queue_work(xprtiod_workqueue, &xprt->task_cleanup); in xprt_conditional_disconnect()
795 xprt_wake_pending_tasks(xprt, -EAGAIN); in xprt_conditional_disconnect()
797 spin_unlock(&xprt->transport_lock); in xprt_conditional_disconnect()
801 xprt_has_timer(const struct rpc_xprt *xprt) in xprt_has_timer() argument
803 return xprt->idle_timeout != 0; in xprt_has_timer()
807 xprt_schedule_autodisconnect(struct rpc_xprt *xprt) in xprt_schedule_autodisconnect() argument
808 __must_hold(&xprt->transport_lock) in xprt_schedule_autodisconnect()
810 xprt->last_used = jiffies; in xprt_schedule_autodisconnect()
811 if (RB_EMPTY_ROOT(&xprt->recv_queue) && xprt_has_timer(xprt)) in xprt_schedule_autodisconnect()
812 mod_timer(&xprt->timer, xprt->last_used + xprt->idle_timeout); in xprt_schedule_autodisconnect()
818 struct rpc_xprt *xprt = from_timer(xprt, t, timer); in xprt_init_autodisconnect() local
820 if (!RB_EMPTY_ROOT(&xprt->recv_queue)) in xprt_init_autodisconnect()
823 xprt->last_used = jiffies; in xprt_init_autodisconnect()
824 if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) in xprt_init_autodisconnect()
826 queue_work(xprtiod_workqueue, &xprt->task_cleanup); in xprt_init_autodisconnect()
829 bool xprt_lock_connect(struct rpc_xprt *xprt, in xprt_lock_connect() argument
835 spin_lock(&xprt->transport_lock); in xprt_lock_connect()
836 if (!test_bit(XPRT_LOCKED, &xprt->state)) in xprt_lock_connect()
838 if (xprt->snd_task != task) in xprt_lock_connect()
840 set_bit(XPRT_SND_IS_COOKIE, &xprt->state); in xprt_lock_connect()
841 xprt->snd_task = cookie; in xprt_lock_connect()
844 spin_unlock(&xprt->transport_lock); in xprt_lock_connect()
849 void xprt_unlock_connect(struct rpc_xprt *xprt, void *cookie) in xprt_unlock_connect() argument
851 spin_lock(&xprt->transport_lock); in xprt_unlock_connect()
852 if (xprt->snd_task != cookie) in xprt_unlock_connect()
854 if (!test_bit(XPRT_LOCKED, &xprt->state)) in xprt_unlock_connect()
856 xprt->snd_task =NULL; in xprt_unlock_connect()
857 clear_bit(XPRT_SND_IS_COOKIE, &xprt->state); in xprt_unlock_connect()
858 xprt->ops->release_xprt(xprt, NULL); in xprt_unlock_connect()
859 xprt_schedule_autodisconnect(xprt); in xprt_unlock_connect()
861 spin_unlock(&xprt->transport_lock); in xprt_unlock_connect()
862 wake_up_bit(&xprt->state, XPRT_LOCKED); in xprt_unlock_connect()
873 struct rpc_xprt *xprt = task->tk_rqstp->rq_xprt; in xprt_connect() local
875 trace_xprt_connect(xprt); in xprt_connect()
877 if (!xprt_bound(xprt)) { in xprt_connect()
881 if (!xprt_lock_write(xprt, task)) in xprt_connect()
884 if (!xprt_connected(xprt) && !test_bit(XPRT_CLOSE_WAIT, &xprt->state)) { in xprt_connect()
885 task->tk_rqstp->rq_connect_cookie = xprt->connect_cookie; in xprt_connect()
886 rpc_sleep_on_timeout(&xprt->pending, task, NULL, in xprt_connect()
889 if (test_bit(XPRT_CLOSING, &xprt->state)) in xprt_connect()
891 if (xprt_test_and_set_connecting(xprt)) in xprt_connect()
894 if (!xprt_connected(xprt)) { in xprt_connect()
895 xprt->stat.connect_start = jiffies; in xprt_connect()
896 xprt->ops->connect(xprt, task); in xprt_connect()
898 xprt_clear_connecting(xprt); in xprt_connect()
900 rpc_wake_up_queued_task(&xprt->pending, task); in xprt_connect()
903 xprt_release_write(xprt, task); in xprt_connect()
911 unsigned long xprt_reconnect_delay(const struct rpc_xprt *xprt) in xprt_reconnect_delay() argument
915 start = xprt->stat.connect_start + xprt->reestablish_timeout; in xprt_reconnect_delay()
928 void xprt_reconnect_backoff(struct rpc_xprt *xprt, unsigned long init_to) in xprt_reconnect_backoff() argument
930 xprt->reestablish_timeout <<= 1; in xprt_reconnect_backoff()
931 if (xprt->reestablish_timeout > xprt->max_reconnect_timeout) in xprt_reconnect_backoff()
932 xprt->reestablish_timeout = xprt->max_reconnect_timeout; in xprt_reconnect_backoff()
933 if (xprt->reestablish_timeout < init_to) in xprt_reconnect_backoff()
934 xprt->reestablish_timeout = init_to; in xprt_reconnect_backoff()
954 xprt_request_rb_find(struct rpc_xprt *xprt, __be32 xid) in xprt_request_rb_find() argument
956 struct rb_node *n = xprt->recv_queue.rb_node; in xprt_request_rb_find()
976 xprt_request_rb_insert(struct rpc_xprt *xprt, struct rpc_rqst *new) in xprt_request_rb_insert() argument
978 struct rb_node **p = &xprt->recv_queue.rb_node; in xprt_request_rb_insert()
998 rb_insert_color(&new->rq_recv, &xprt->recv_queue); in xprt_request_rb_insert()
1002 xprt_request_rb_remove(struct rpc_xprt *xprt, struct rpc_rqst *req) in xprt_request_rb_remove() argument
1004 rb_erase(&req->rq_recv, &xprt->recv_queue); in xprt_request_rb_remove()
1014 struct rpc_rqst *xprt_lookup_rqst(struct rpc_xprt *xprt, __be32 xid) in xprt_lookup_rqst() argument
1018 entry = xprt_request_rb_find(xprt, xid); in xprt_lookup_rqst()
1020 trace_xprt_lookup_rqst(xprt, xid, 0); in xprt_lookup_rqst()
1027 trace_xprt_lookup_rqst(xprt, xid, -ENOENT); in xprt_lookup_rqst()
1028 xprt->stat.bad_xids++; in xprt_lookup_rqst()
1097 struct rpc_xprt *xprt = req->rq_xprt; in xprt_request_enqueue_receive() local
1103 spin_lock(&xprt->queue_lock); in xprt_request_enqueue_receive()
1110 xprt_request_rb_insert(xprt, req); in xprt_request_enqueue_receive()
1112 spin_unlock(&xprt->queue_lock); in xprt_request_enqueue_receive()
1115 del_singleshot_timer_sync(&xprt->timer); in xprt_request_enqueue_receive()
1164 struct rpc_xprt *xprt = req->rq_xprt; in xprt_complete_rqst() local
1166 xprt->stat.recvs++; in xprt_complete_rqst()
1174 rpc_wake_up_queued_task(&xprt->pending, task); in xprt_complete_rqst()
1181 struct rpc_xprt *xprt = req->rq_xprt; in xprt_timer() local
1186 trace_xprt_timer(xprt, req->rq_xid, task->tk_status); in xprt_timer()
1188 if (xprt->ops->timer) in xprt_timer()
1189 xprt->ops->timer(xprt, task); in xprt_timer()
1245 struct rpc_xprt *xprt = req->rq_xprt; in xprt_request_wait_receive() local
1254 spin_lock(&xprt->queue_lock); in xprt_request_wait_receive()
1256 xprt->ops->wait_for_reply_request(task); in xprt_request_wait_receive()
1263 rpc_wake_up_queued_task_set_status(&xprt->pending, in xprt_request_wait_receive()
1266 spin_unlock(&xprt->queue_lock); in xprt_request_wait_receive()
1285 struct rpc_xprt *xprt = req->rq_xprt; in xprt_request_enqueue_transmit() local
1289 spin_lock(&xprt->queue_lock); in xprt_request_enqueue_transmit()
1295 xprt_clear_congestion_window_wait(xprt); in xprt_request_enqueue_transmit()
1296 list_for_each_entry(pos, &xprt->xmit_queue, rq_xmit) { in xprt_request_enqueue_transmit()
1305 list_for_each_entry(pos, &xprt->xmit_queue, rq_xmit) { in xprt_request_enqueue_transmit()
1316 list_for_each_entry(pos, &xprt->xmit_queue, rq_xmit) { in xprt_request_enqueue_transmit()
1324 list_add_tail(&req->rq_xmit, &xprt->xmit_queue); in xprt_request_enqueue_transmit()
1328 spin_unlock(&xprt->queue_lock); in xprt_request_enqueue_transmit()
1368 struct rpc_xprt *xprt = req->rq_xprt; in xprt_request_dequeue_transmit() local
1370 spin_lock(&xprt->queue_lock); in xprt_request_dequeue_transmit()
1372 spin_unlock(&xprt->queue_lock); in xprt_request_dequeue_transmit()
1386 struct rpc_xprt *xprt = req->rq_xprt; in xprt_request_dequeue_xprt() local
1391 spin_lock(&xprt->queue_lock); in xprt_request_dequeue_xprt()
1396 spin_unlock(&xprt->queue_lock); in xprt_request_dequeue_xprt()
1398 spin_lock(&xprt->queue_lock); in xprt_request_dequeue_xprt()
1401 spin_unlock(&xprt->queue_lock); in xprt_request_dequeue_xprt()
1415 struct rpc_xprt *xprt = req->rq_xprt; in xprt_request_prepare() local
1417 if (xprt->ops->prepare_request) in xprt_request_prepare()
1418 xprt->ops->prepare_request(req); in xprt_request_prepare()
1441 struct rpc_xprt *xprt = req->rq_xprt; in xprt_prepare_transmit() local
1443 if (!xprt_lock_write(xprt, task)) { in xprt_prepare_transmit()
1446 rpc_wake_up_queued_task_set_status(&xprt->sending, in xprt_prepare_transmit()
1456 struct rpc_xprt *xprt = task->tk_rqstp->rq_xprt; in xprt_end_transmit() local
1458 xprt_inject_disconnect(xprt); in xprt_end_transmit()
1459 xprt_release_write(xprt, task); in xprt_end_transmit()
1475 struct rpc_xprt *xprt = req->rq_xprt; in xprt_request_transmit() local
1505 connect_cookie = xprt->connect_cookie; in xprt_request_transmit()
1506 status = xprt->ops->send_request(req); in xprt_request_transmit()
1516 xprt_inject_disconnect(xprt); in xprt_request_transmit()
1519 spin_lock(&xprt->transport_lock); in xprt_request_transmit()
1521 xprt->stat.sends++; in xprt_request_transmit()
1522 xprt->stat.req_u += xprt->stat.sends - xprt->stat.recvs; in xprt_request_transmit()
1523 xprt->stat.bklog_u += xprt->backlog.qlen; in xprt_request_transmit()
1524 xprt->stat.sending_u += xprt->sending.qlen; in xprt_request_transmit()
1525 xprt->stat.pending_u += xprt->pending.qlen; in xprt_request_transmit()
1526 spin_unlock(&xprt->transport_lock); in xprt_request_transmit()
1532 rpc_wake_up_queued_task_set_status(&xprt->sending, task, status); in xprt_request_transmit()
1549 struct rpc_xprt *xprt = req->rq_xprt; in xprt_transmit() local
1552 spin_lock(&xprt->queue_lock); in xprt_transmit()
1554 next = list_first_entry_or_null(&xprt->xmit_queue, in xprt_transmit()
1559 spin_unlock(&xprt->queue_lock); in xprt_transmit()
1563 spin_lock(&xprt->queue_lock); in xprt_transmit()
1574 cond_resched_lock(&xprt->queue_lock); in xprt_transmit()
1576 spin_unlock(&xprt->queue_lock); in xprt_transmit()
1585 void xprt_add_backlog(struct rpc_xprt *xprt, struct rpc_task *task) in xprt_add_backlog() argument
1587 set_bit(XPRT_CONGESTED, &xprt->state); in xprt_add_backlog()
1588 rpc_sleep_on(&xprt->backlog, task, xprt_complete_request_init); in xprt_add_backlog()
1604 bool xprt_wake_up_backlog(struct rpc_xprt *xprt, struct rpc_rqst *req) in xprt_wake_up_backlog() argument
1606 if (rpc_wake_up_first(&xprt->backlog, __xprt_set_rq, req) == NULL) { in xprt_wake_up_backlog()
1607 clear_bit(XPRT_CONGESTED, &xprt->state); in xprt_wake_up_backlog()
1614 static bool xprt_throttle_congested(struct rpc_xprt *xprt, struct rpc_task *task) in xprt_throttle_congested() argument
1618 if (!test_bit(XPRT_CONGESTED, &xprt->state)) in xprt_throttle_congested()
1620 spin_lock(&xprt->reserve_lock); in xprt_throttle_congested()
1621 if (test_bit(XPRT_CONGESTED, &xprt->state)) { in xprt_throttle_congested()
1622 xprt_add_backlog(xprt, task); in xprt_throttle_congested()
1625 spin_unlock(&xprt->reserve_lock); in xprt_throttle_congested()
1630 static struct rpc_rqst *xprt_dynamic_alloc_slot(struct rpc_xprt *xprt) in xprt_dynamic_alloc_slot() argument
1634 if (xprt->num_reqs >= xprt->max_reqs) in xprt_dynamic_alloc_slot()
1636 ++xprt->num_reqs; in xprt_dynamic_alloc_slot()
1637 spin_unlock(&xprt->reserve_lock); in xprt_dynamic_alloc_slot()
1639 spin_lock(&xprt->reserve_lock); in xprt_dynamic_alloc_slot()
1642 --xprt->num_reqs; in xprt_dynamic_alloc_slot()
1648 static bool xprt_dynamic_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req) in xprt_dynamic_free_slot() argument
1650 if (xprt->num_reqs > xprt->min_reqs) { in xprt_dynamic_free_slot()
1651 --xprt->num_reqs; in xprt_dynamic_free_slot()
1658 void xprt_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task) in xprt_alloc_slot() argument
1662 spin_lock(&xprt->reserve_lock); in xprt_alloc_slot()
1663 if (!list_empty(&xprt->free)) { in xprt_alloc_slot()
1664 req = list_entry(xprt->free.next, struct rpc_rqst, rq_list); in xprt_alloc_slot()
1668 req = xprt_dynamic_alloc_slot(xprt); in xprt_alloc_slot()
1678 xprt_add_backlog(xprt, task); in xprt_alloc_slot()
1684 spin_unlock(&xprt->reserve_lock); in xprt_alloc_slot()
1687 xprt->stat.max_slots = max_t(unsigned int, xprt->stat.max_slots, in xprt_alloc_slot()
1688 xprt->num_reqs); in xprt_alloc_slot()
1689 spin_unlock(&xprt->reserve_lock); in xprt_alloc_slot()
1696 void xprt_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req) in xprt_free_slot() argument
1698 spin_lock(&xprt->reserve_lock); in xprt_free_slot()
1699 if (!xprt_wake_up_backlog(xprt, req) && in xprt_free_slot()
1700 !xprt_dynamic_free_slot(xprt, req)) { in xprt_free_slot()
1702 list_add(&req->rq_list, &xprt->free); in xprt_free_slot()
1704 spin_unlock(&xprt->reserve_lock); in xprt_free_slot()
1708 static void xprt_free_all_slots(struct rpc_xprt *xprt) in xprt_free_all_slots() argument
1711 while (!list_empty(&xprt->free)) { in xprt_free_all_slots()
1712 req = list_first_entry(&xprt->free, struct rpc_rqst, rq_list); in xprt_free_all_slots()
1722 struct rpc_xprt *xprt; in xprt_alloc() local
1726 xprt = kzalloc(size, GFP_KERNEL); in xprt_alloc()
1727 if (xprt == NULL) in xprt_alloc()
1730 xprt_init(xprt, net); in xprt_alloc()
1736 list_add(&req->rq_list, &xprt->free); in xprt_alloc()
1739 xprt->max_reqs = max_alloc; in xprt_alloc()
1741 xprt->max_reqs = num_prealloc; in xprt_alloc()
1742 xprt->min_reqs = num_prealloc; in xprt_alloc()
1743 xprt->num_reqs = num_prealloc; in xprt_alloc()
1745 return xprt; in xprt_alloc()
1748 xprt_free(xprt); in xprt_alloc()
1754 void xprt_free(struct rpc_xprt *xprt) in xprt_free() argument
1756 put_net(xprt->xprt_net); in xprt_free()
1757 xprt_free_all_slots(xprt); in xprt_free()
1758 kfree_rcu(xprt, rcu); in xprt_free()
1763 xprt_init_connect_cookie(struct rpc_rqst *req, struct rpc_xprt *xprt) in xprt_init_connect_cookie() argument
1765 req->rq_connect_cookie = xprt_connect_cookie(xprt) - 1; in xprt_init_connect_cookie()
1769 xprt_alloc_xid(struct rpc_xprt *xprt) in xprt_alloc_xid() argument
1773 spin_lock(&xprt->reserve_lock); in xprt_alloc_xid()
1774 xid = (__force __be32)xprt->xid++; in xprt_alloc_xid()
1775 spin_unlock(&xprt->reserve_lock); in xprt_alloc_xid()
1780 xprt_init_xid(struct rpc_xprt *xprt) in xprt_init_xid() argument
1782 xprt->xid = prandom_u32(); in xprt_init_xid()
1788 struct rpc_xprt *xprt = task->tk_xprt; in xprt_request_init() local
1792 req->rq_xprt = xprt; in xprt_request_init()
1794 req->rq_xid = xprt_alloc_xid(xprt); in xprt_request_init()
1795 xprt_init_connect_cookie(req, xprt); in xprt_request_init()
1809 xprt_do_reserve(struct rpc_xprt *xprt, struct rpc_task *task) in xprt_do_reserve() argument
1811 xprt->ops->alloc_slot(xprt, task); in xprt_do_reserve()
1826 struct rpc_xprt *xprt = task->tk_xprt; in xprt_reserve() local
1833 if (!xprt_throttle_congested(xprt, task)) in xprt_reserve()
1834 xprt_do_reserve(xprt, task); in xprt_reserve()
1848 struct rpc_xprt *xprt = task->tk_xprt; in xprt_retry_reserve() local
1855 xprt_do_reserve(xprt, task); in xprt_retry_reserve()
1865 struct rpc_xprt *xprt; in xprt_release() local
1870 xprt = task->tk_xprt; in xprt_release()
1871 xprt_release_write(xprt, task); in xprt_release()
1876 xprt = req->rq_xprt; in xprt_release()
1878 spin_lock(&xprt->transport_lock); in xprt_release()
1879 xprt->ops->release_xprt(xprt, task); in xprt_release()
1880 if (xprt->ops->release_request) in xprt_release()
1881 xprt->ops->release_request(task); in xprt_release()
1882 xprt_schedule_autodisconnect(xprt); in xprt_release()
1883 spin_unlock(&xprt->transport_lock); in xprt_release()
1885 xprt->ops->buf_free(task); in xprt_release()
1895 xprt->ops->free_slot(xprt, req); in xprt_release()
1918 static void xprt_init(struct rpc_xprt *xprt, struct net *net) in xprt_init() argument
1920 kref_init(&xprt->kref); in xprt_init()
1922 spin_lock_init(&xprt->transport_lock); in xprt_init()
1923 spin_lock_init(&xprt->reserve_lock); in xprt_init()
1924 spin_lock_init(&xprt->queue_lock); in xprt_init()
1926 INIT_LIST_HEAD(&xprt->free); in xprt_init()
1927 xprt->recv_queue = RB_ROOT; in xprt_init()
1928 INIT_LIST_HEAD(&xprt->xmit_queue); in xprt_init()
1930 spin_lock_init(&xprt->bc_pa_lock); in xprt_init()
1931 INIT_LIST_HEAD(&xprt->bc_pa_list); in xprt_init()
1933 INIT_LIST_HEAD(&xprt->xprt_switch); in xprt_init()
1935 xprt->last_used = jiffies; in xprt_init()
1936 xprt->cwnd = RPC_INITCWND; in xprt_init()
1937 xprt->bind_index = 0; in xprt_init()
1939 rpc_init_wait_queue(&xprt->binding, "xprt_binding"); in xprt_init()
1940 rpc_init_wait_queue(&xprt->pending, "xprt_pending"); in xprt_init()
1941 rpc_init_wait_queue(&xprt->sending, "xprt_sending"); in xprt_init()
1942 rpc_init_priority_wait_queue(&xprt->backlog, "xprt_backlog"); in xprt_init()
1944 xprt_init_xid(xprt); in xprt_init()
1946 xprt->xprt_net = get_net(net); in xprt_init()
1956 struct rpc_xprt *xprt; in xprt_create_transport() local
1971 xprt = t->setup(args); in xprt_create_transport()
1972 if (IS_ERR(xprt)) in xprt_create_transport()
1975 xprt->idle_timeout = 0; in xprt_create_transport()
1976 INIT_WORK(&xprt->task_cleanup, xprt_autoclose); in xprt_create_transport()
1977 if (xprt_has_timer(xprt)) in xprt_create_transport()
1978 timer_setup(&xprt->timer, xprt_init_autodisconnect, 0); in xprt_create_transport()
1980 timer_setup(&xprt->timer, NULL, 0); in xprt_create_transport()
1983 xprt_destroy(xprt); in xprt_create_transport()
1986 xprt->servername = kstrdup(args->servername, GFP_KERNEL); in xprt_create_transport()
1987 if (xprt->servername == NULL) { in xprt_create_transport()
1988 xprt_destroy(xprt); in xprt_create_transport()
1992 rpc_xprt_debugfs_register(xprt); in xprt_create_transport()
1994 trace_xprt_create(xprt); in xprt_create_transport()
1996 return xprt; in xprt_create_transport()
2001 struct rpc_xprt *xprt = in xprt_destroy_cb() local
2004 trace_xprt_destroy(xprt); in xprt_destroy_cb()
2006 rpc_xprt_debugfs_unregister(xprt); in xprt_destroy_cb()
2007 rpc_destroy_wait_queue(&xprt->binding); in xprt_destroy_cb()
2008 rpc_destroy_wait_queue(&xprt->pending); in xprt_destroy_cb()
2009 rpc_destroy_wait_queue(&xprt->sending); in xprt_destroy_cb()
2010 rpc_destroy_wait_queue(&xprt->backlog); in xprt_destroy_cb()
2011 kfree(xprt->servername); in xprt_destroy_cb()
2015 xprt_destroy_backchannel(xprt, UINT_MAX); in xprt_destroy_cb()
2020 xprt->ops->destroy(xprt); in xprt_destroy_cb()
2028 static void xprt_destroy(struct rpc_xprt *xprt) in xprt_destroy() argument
2033 wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_UNINTERRUPTIBLE); in xprt_destroy()
2035 del_timer_sync(&xprt->timer); in xprt_destroy()
2041 INIT_WORK(&xprt->task_cleanup, xprt_destroy_cb); in xprt_destroy()
2042 schedule_work(&xprt->task_cleanup); in xprt_destroy()
2055 struct rpc_xprt *xprt_get(struct rpc_xprt *xprt) in xprt_get() argument
2057 if (xprt != NULL && kref_get_unless_zero(&xprt->kref)) in xprt_get()
2058 return xprt; in xprt_get()
2068 void xprt_put(struct rpc_xprt *xprt) in xprt_put() argument
2070 if (xprt != NULL) in xprt_put()
2071 kref_put(&xprt->kref, xprt_destroy_kref); in xprt_put()