Lines Matching refs:rm
66 struct rds_message *rm, *tmp; in rds_send_reset() local
70 rm = conn->c_xmit_rm; in rds_send_reset()
76 rds_message_unmapped(rm); in rds_send_reset()
77 rds_message_put(rm); in rds_send_reset()
94 list_for_each_entry_safe(rm, tmp, &conn->c_retrans, m_conn_item) { in rds_send_reset()
95 set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags); in rds_send_reset()
96 set_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags); in rds_send_reset()
137 struct rds_message *rm; in rds_send_xmit() local
178 rm = conn->c_xmit_rm; in rds_send_xmit()
184 if (!rm && test_and_clear_bit(0, &conn->c_map_queued)) { in rds_send_xmit()
185 rm = rds_cong_update_alloc(conn); in rds_send_xmit()
186 if (IS_ERR(rm)) { in rds_send_xmit()
187 ret = PTR_ERR(rm); in rds_send_xmit()
190 rm->data.op_active = 1; in rds_send_xmit()
192 conn->c_xmit_rm = rm; in rds_send_xmit()
202 if (!rm) { in rds_send_xmit()
208 rm = list_entry(conn->c_send_queue.next, in rds_send_xmit()
211 rds_message_addref(rm); in rds_send_xmit()
217 list_move_tail(&rm->m_conn_item, &conn->c_retrans); in rds_send_xmit()
222 if (!rm) in rds_send_xmit()
232 if (rm->rdma.op_active && in rds_send_xmit()
233 test_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags)) { in rds_send_xmit()
235 if (test_and_clear_bit(RDS_MSG_ON_CONN, &rm->m_flags)) in rds_send_xmit()
236 list_move(&rm->m_conn_item, &to_be_dropped); in rds_send_xmit()
242 len = ntohl(rm->m_inc.i_hdr.h_len); in rds_send_xmit()
245 __set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags); in rds_send_xmit()
255 conn->c_xmit_rm = rm; in rds_send_xmit()
259 if (rm->rdma.op_active && !conn->c_xmit_rdma_sent) { in rds_send_xmit()
260 rm->m_final_op = &rm->rdma; in rds_send_xmit()
261 ret = conn->c_trans->xmit_rdma(conn, &rm->rdma); in rds_send_xmit()
268 set_bit(RDS_MSG_MAPPED, &rm->m_flags); in rds_send_xmit()
271 if (rm->atomic.op_active && !conn->c_xmit_atomic_sent) { in rds_send_xmit()
272 rm->m_final_op = &rm->atomic; in rds_send_xmit()
273 ret = conn->c_trans->xmit_atomic(conn, &rm->atomic); in rds_send_xmit()
280 set_bit(RDS_MSG_MAPPED, &rm->m_flags); in rds_send_xmit()
290 if (rm->data.op_nents == 0) { in rds_send_xmit()
294 ops_present = (rm->atomic.op_active || rm->rdma.op_active); in rds_send_xmit()
295 if (rm->atomic.op_active && !rm->atomic.op_silent) in rds_send_xmit()
297 if (rm->rdma.op_active && !rm->rdma.op_silent) in rds_send_xmit()
301 && !rm->m_rdma_cookie) in rds_send_xmit()
302 rm->data.op_active = 0; in rds_send_xmit()
305 if (rm->data.op_active && !conn->c_xmit_data_sent) { in rds_send_xmit()
306 rm->m_final_op = &rm->data; in rds_send_xmit()
307 ret = conn->c_trans->xmit(conn, rm, in rds_send_xmit()
322 sg = &rm->data.op_sg[conn->c_xmit_sg]; in rds_send_xmit()
333 conn->c_xmit_sg == rm->data.op_nents); in rds_send_xmit()
338 (conn->c_xmit_sg == rm->data.op_nents)) in rds_send_xmit()
347 if (!rm->data.op_active || conn->c_xmit_data_sent) { in rds_send_xmit()
356 rds_message_put(rm); in rds_send_xmit()
368 list_for_each_entry(rm, &to_be_dropped, m_conn_item) in rds_send_xmit()
369 rds_message_put(rm); in rds_send_xmit()
395 static void rds_send_sndbuf_remove(struct rds_sock *rs, struct rds_message *rm) in rds_send_sndbuf_remove() argument
397 u32 len = be32_to_cpu(rm->m_inc.i_hdr.h_len); in rds_send_sndbuf_remove()
408 static inline int rds_send_is_acked(struct rds_message *rm, u64 ack, in rds_send_is_acked() argument
412 return is_acked(rm, ack); in rds_send_is_acked()
413 return be64_to_cpu(rm->m_inc.i_hdr.h_sequence) <= ack; in rds_send_is_acked()
422 void rds_rdma_send_complete(struct rds_message *rm, int status) in rds_rdma_send_complete() argument
430 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_rdma_send_complete()
432 notify = rm->rdma.op_notify | rm->data.op_notify; in rds_rdma_send_complete()
433 ro = &rm->rdma; in rds_rdma_send_complete()
434 if (test_bit(RDS_MSG_ON_SOCK, &rm->m_flags) && in rds_rdma_send_complete()
437 rs = rm->m_rs; in rds_rdma_send_complete()
448 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_rdma_send_complete()
460 void rds_atomic_send_complete(struct rds_message *rm, int status) in rds_atomic_send_complete() argument
467 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_atomic_send_complete()
469 ao = &rm->atomic; in rds_atomic_send_complete()
470 if (test_bit(RDS_MSG_ON_SOCK, &rm->m_flags) in rds_atomic_send_complete()
473 rs = rm->m_rs; in rds_atomic_send_complete()
484 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_atomic_send_complete()
499 __rds_send_complete(struct rds_sock *rs, struct rds_message *rm, int status) in __rds_send_complete() argument
504 ro = &rm->rdma; in __rds_send_complete()
511 ao = &rm->atomic; in __rds_send_complete()
529 struct rds_message *rm, *tmp, *found = NULL; in rds_send_get_message() local
534 list_for_each_entry_safe(rm, tmp, &conn->c_retrans, m_conn_item) { in rds_send_get_message()
535 if (&rm->rdma == op) { in rds_send_get_message()
536 atomic_inc(&rm->m_refcount); in rds_send_get_message()
537 found = rm; in rds_send_get_message()
542 list_for_each_entry_safe(rm, tmp, &conn->c_send_queue, m_conn_item) { in rds_send_get_message()
543 if (&rm->rdma == op) { in rds_send_get_message()
544 atomic_inc(&rm->m_refcount); in rds_send_get_message()
545 found = rm; in rds_send_get_message()
569 struct rds_message *rm; in rds_send_remove_from_sock() local
574 rm = list_entry(messages->next, struct rds_message, in rds_send_remove_from_sock()
576 list_del_init(&rm->m_conn_item); in rds_send_remove_from_sock()
588 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_send_remove_from_sock()
589 if (!test_bit(RDS_MSG_ON_SOCK, &rm->m_flags)) in rds_send_remove_from_sock()
592 if (rs != rm->m_rs) { in rds_send_remove_from_sock()
597 rs = rm->m_rs; in rds_send_remove_from_sock()
605 if (test_and_clear_bit(RDS_MSG_ON_SOCK, &rm->m_flags)) { in rds_send_remove_from_sock()
606 struct rm_rdma_op *ro = &rm->rdma; in rds_send_remove_from_sock()
609 list_del_init(&rm->m_sock_item); in rds_send_remove_from_sock()
610 rds_send_sndbuf_remove(rs, rm); in rds_send_remove_from_sock()
619 rm->rdma.op_notifier = NULL; in rds_send_remove_from_sock()
622 rm->m_rs = NULL; in rds_send_remove_from_sock()
627 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_send_remove_from_sock()
628 rds_message_put(rm); in rds_send_remove_from_sock()
630 rds_message_put(rm); in rds_send_remove_from_sock()
650 struct rds_message *rm, *tmp; in rds_send_drop_acked() local
656 list_for_each_entry_safe(rm, tmp, &conn->c_retrans, m_conn_item) { in rds_send_drop_acked()
657 if (!rds_send_is_acked(rm, ack, is_acked)) in rds_send_drop_acked()
660 list_move(&rm->m_conn_item, &list); in rds_send_drop_acked()
661 clear_bit(RDS_MSG_ON_CONN, &rm->m_flags); in rds_send_drop_acked()
677 struct rds_message *rm, *tmp; in rds_send_drop_to() local
685 list_for_each_entry_safe(rm, tmp, &rs->rs_send_queue, m_sock_item) { in rds_send_drop_to()
686 if (dest && (dest->sin_addr.s_addr != rm->m_daddr || in rds_send_drop_to()
687 dest->sin_port != rm->m_inc.i_hdr.h_dport)) in rds_send_drop_to()
690 list_move(&rm->m_sock_item, &list); in rds_send_drop_to()
691 rds_send_sndbuf_remove(rs, rm); in rds_send_drop_to()
692 clear_bit(RDS_MSG_ON_SOCK, &rm->m_flags); in rds_send_drop_to()
704 list_for_each_entry(rm, &list, m_sock_item) { in rds_send_drop_to()
706 conn = rm->m_inc.i_conn; in rds_send_drop_to()
714 if (!test_and_clear_bit(RDS_MSG_ON_CONN, &rm->m_flags)) { in rds_send_drop_to()
716 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_send_drop_to()
717 rm->m_rs = NULL; in rds_send_drop_to()
718 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_send_drop_to()
721 list_del_init(&rm->m_conn_item); in rds_send_drop_to()
728 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_send_drop_to()
731 __rds_send_complete(rs, rm, RDS_RDMA_CANCELED); in rds_send_drop_to()
734 rm->m_rs = NULL; in rds_send_drop_to()
735 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_send_drop_to()
737 rds_message_put(rm); in rds_send_drop_to()
743 rm = list_entry(list.next, struct rds_message, m_sock_item); in rds_send_drop_to()
744 list_del_init(&rm->m_sock_item); in rds_send_drop_to()
746 rds_message_wait(rm); in rds_send_drop_to()
747 rds_message_put(rm); in rds_send_drop_to()
757 struct rds_message *rm, __be16 sport, in rds_send_queue_rm() argument
766 len = be32_to_cpu(rm->m_inc.i_hdr.h_len); in rds_send_queue_rm()
789 __set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags); in rds_send_queue_rm()
791 list_add_tail(&rm->m_sock_item, &rs->rs_send_queue); in rds_send_queue_rm()
792 set_bit(RDS_MSG_ON_SOCK, &rm->m_flags); in rds_send_queue_rm()
793 rds_message_addref(rm); in rds_send_queue_rm()
794 rm->m_rs = rs; in rds_send_queue_rm()
798 rds_message_populate_header(&rm->m_inc.i_hdr, sport, dport, 0); in rds_send_queue_rm()
799 rm->m_inc.i_conn = conn; in rds_send_queue_rm()
800 rds_message_addref(rm); in rds_send_queue_rm()
803 rm->m_inc.i_hdr.h_sequence = cpu_to_be64(conn->c_next_tx_seq++); in rds_send_queue_rm()
804 list_add_tail(&rm->m_conn_item, &conn->c_send_queue); in rds_send_queue_rm()
805 set_bit(RDS_MSG_ON_CONN, &rm->m_flags); in rds_send_queue_rm()
809 rm, len, rs, rs->rs_snd_bytes, in rds_send_queue_rm()
810 (unsigned long long)be64_to_cpu(rm->m_inc.i_hdr.h_sequence)); in rds_send_queue_rm()
877 static int rds_cmsg_send(struct rds_sock *rs, struct rds_message *rm, in rds_cmsg_send() argument
895 ret = rds_cmsg_rdma_args(rs, rm, cmsg); in rds_cmsg_send()
899 ret = rds_cmsg_rdma_dest(rs, rm, cmsg); in rds_cmsg_send()
903 ret = rds_cmsg_rdma_map(rs, rm, cmsg); in rds_cmsg_send()
916 ret = rds_cmsg_atomic(rs, rm, cmsg); in rds_cmsg_send()
938 struct rds_message *rm = NULL; in rds_sendmsg() local
981 rm = rds_message_alloc(ret, GFP_KERNEL); in rds_sendmsg()
982 if (!rm) { in rds_sendmsg()
989 rm->data.op_sg = rds_message_alloc_sgs(rm, ceil(payload_len, PAGE_SIZE)); in rds_sendmsg()
990 if (!rm->data.op_sg) { in rds_sendmsg()
994 ret = rds_message_copy_from_user(rm, msg->msg_iov, payload_len); in rds_sendmsg()
998 rm->data.op_active = 1; in rds_sendmsg()
1000 rm->m_daddr = daddr; in rds_sendmsg()
1018 ret = rds_cmsg_send(rs, rm, msg, &allocated_mr); in rds_sendmsg()
1026 if (rm->rdma.op_active && !conn->c_trans->xmit_rdma) { in rds_sendmsg()
1028 &rm->rdma, conn->c_trans->xmit_rdma); in rds_sendmsg()
1033 if (rm->atomic.op_active && !conn->c_trans->xmit_atomic) { in rds_sendmsg()
1035 &rm->atomic, conn->c_trans->xmit_atomic); in rds_sendmsg()
1048 while (!rds_send_queue_rm(rs, conn, rm, rs->rs_bound_port, in rds_sendmsg()
1062 rds_send_queue_rm(rs, conn, rm, in rds_sendmsg()
1086 rds_message_put(rm); in rds_sendmsg()
1094 rds_rdma_unuse(rs, rds_rdma_cookie_key(rm->m_rdma_cookie), 1); in rds_sendmsg()
1096 if (rm) in rds_sendmsg()
1097 rds_message_put(rm); in rds_sendmsg()
1107 struct rds_message *rm; in rds_send_pong() local
1111 rm = rds_message_alloc(0, GFP_ATOMIC); in rds_send_pong()
1112 if (!rm) { in rds_send_pong()
1117 rm->m_daddr = conn->c_faddr; in rds_send_pong()
1118 rm->data.op_active = 1; in rds_send_pong()
1127 list_add_tail(&rm->m_conn_item, &conn->c_send_queue); in rds_send_pong()
1128 set_bit(RDS_MSG_ON_CONN, &rm->m_flags); in rds_send_pong()
1129 rds_message_addref(rm); in rds_send_pong()
1130 rm->m_inc.i_conn = conn; in rds_send_pong()
1132 rds_message_populate_header(&rm->m_inc.i_hdr, 0, dport, in rds_send_pong()
1143 rds_message_put(rm); in rds_send_pong()
1147 if (rm) in rds_send_pong()
1148 rds_message_put(rm); in rds_send_pong()