Lines Matching refs:ic
66 static void rds_iw_send_unmap_rdma(struct rds_iw_connection *ic, in rds_iw_send_unmap_rdma() argument
70 ib_dma_unmap_sg(ic->i_cm_id->device, in rds_iw_send_unmap_rdma()
77 static void rds_iw_send_unmap_rm(struct rds_iw_connection *ic, in rds_iw_send_unmap_rm() argument
83 rdsdebug("ic %p send %p rm %p\n", ic, send, rm); in rds_iw_send_unmap_rm()
85 ib_dma_unmap_sg(ic->i_cm_id->device, in rds_iw_send_unmap_rm()
90 rds_iw_send_unmap_rdma(ic, &rm->rdma); in rds_iw_send_unmap_rm()
128 void rds_iw_send_init_ring(struct rds_iw_connection *ic) in rds_iw_send_init_ring() argument
133 for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) { in rds_iw_send_init_ring()
148 sge = rds_iw_data_sge(ic, send->s_sge); in rds_iw_send_init_ring()
151 sge = rds_iw_header_sge(ic, send->s_sge); in rds_iw_send_init_ring()
152 sge->addr = ic->i_send_hdrs_dma + (i * sizeof(struct rds_header)); in rds_iw_send_init_ring()
156 send->s_mr = ib_alloc_fast_reg_mr(ic->i_pd, fastreg_message_size); in rds_iw_send_init_ring()
163 ic->i_cm_id->device, fastreg_message_size); in rds_iw_send_init_ring()
171 void rds_iw_send_clear_ring(struct rds_iw_connection *ic) in rds_iw_send_clear_ring() argument
176 for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) { in rds_iw_send_clear_ring()
184 rds_iw_send_unmap_rm(ic, send, IB_WC_WR_FLUSH_ERR); in rds_iw_send_clear_ring()
186 rds_iw_send_unmap_rdma(ic, send->s_op); in rds_iw_send_clear_ring()
199 struct rds_iw_connection *ic = conn->c_transport_data; in rds_iw_send_cq_comp_handler() local
225 ic->i_fastreg_posted = 0; in rds_iw_send_cq_comp_handler()
230 ic->i_fastreg_posted = 1; in rds_iw_send_cq_comp_handler()
235 if (time_after(jiffies, ic->i_ack_queued + HZ/2)) in rds_iw_send_cq_comp_handler()
237 rds_iw_ack_send_complete(ic); in rds_iw_send_cq_comp_handler()
241 oldest = rds_iw_ring_oldest(&ic->i_send_ring); in rds_iw_send_cq_comp_handler()
243 completed = rds_iw_ring_completed(&ic->i_send_ring, wc.wr_id, oldest); in rds_iw_send_cq_comp_handler()
246 send = &ic->i_sends[oldest]; in rds_iw_send_cq_comp_handler()
252 rds_iw_send_unmap_rm(ic, send, wc.status); in rds_iw_send_cq_comp_handler()
285 oldest = (oldest + 1) % ic->i_send_ring.w_nr; in rds_iw_send_cq_comp_handler()
288 rds_iw_ring_free(&ic->i_send_ring, completed); in rds_iw_send_cq_comp_handler()
348 int rds_iw_send_grab_credits(struct rds_iw_connection *ic, in rds_iw_send_grab_credits() argument
355 if (!ic->i_flowctl) in rds_iw_send_grab_credits()
360 oldval = newval = atomic_read(&ic->i_credits); in rds_iw_send_grab_credits()
372 struct rds_connection *conn = ic->i_cm_id->context; in rds_iw_send_grab_credits()
394 if (atomic_cmpxchg(&ic->i_credits, oldval, newval) != oldval) in rds_iw_send_grab_credits()
403 struct rds_iw_connection *ic = conn->c_transport_data; in rds_iw_send_add_credits() local
410 IB_GET_SEND_CREDITS(atomic_read(&ic->i_credits)), in rds_iw_send_add_credits()
413 atomic_add(IB_SET_SEND_CREDITS(credits), &ic->i_credits); in rds_iw_send_add_credits()
424 struct rds_iw_connection *ic = conn->c_transport_data; in rds_iw_advertise_credits() local
429 atomic_add(IB_SET_POST_CREDITS(posted), &ic->i_credits); in rds_iw_advertise_credits()
443 if (IB_GET_POST_CREDITS(atomic_read(&ic->i_credits)) >= 16) in rds_iw_advertise_credits()
444 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_iw_advertise_credits()
448 rds_iw_xmit_populate_wr(struct rds_iw_connection *ic, in rds_iw_xmit_populate_wr() argument
455 WARN_ON(pos != send - ic->i_sends); in rds_iw_xmit_populate_wr()
465 sge = rds_iw_data_sge(ic, send->s_sge); in rds_iw_xmit_populate_wr()
468 sge->lkey = rds_iw_local_dma_lkey(ic); in rds_iw_xmit_populate_wr()
470 sge = rds_iw_header_sge(ic, send->s_sge); in rds_iw_xmit_populate_wr()
478 sge->addr = ic->i_send_hdrs_dma + (pos * sizeof(struct rds_header)); in rds_iw_xmit_populate_wr()
480 sge->lkey = rds_iw_local_dma_lkey(ic); in rds_iw_xmit_populate_wr()
499 struct rds_iw_connection *ic = conn->c_transport_data; in rds_iw_xmit() local
500 struct ib_device *dev = ic->i_cm_id->device; in rds_iw_xmit()
521 if (rds_rdma_cookie_key(rm->m_rdma_cookie) && !ic->i_fastreg_posted) { in rds_iw_xmit()
532 work_alloc = rds_iw_ring_alloc(&ic->i_send_ring, i, &pos); in rds_iw_xmit()
541 if (ic->i_flowctl) { in rds_iw_xmit()
542 credit_alloc = rds_iw_send_grab_credits(ic, work_alloc, &posted, 0, RDS_MAX_ADV_CREDIT); in rds_iw_xmit()
545 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc - credit_alloc); in rds_iw_xmit()
558 if (!ic->i_rm) { in rds_iw_xmit()
570 rdsdebug("ic %p mapping rm %p: %d\n", ic, rm, rm->data.op_count); in rds_iw_xmit()
573 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc); in rds_iw_xmit()
581 ic->i_unsignaled_wrs = rds_iw_sysctl_max_unsig_wrs; in rds_iw_xmit()
582 ic->i_unsignaled_bytes = rds_iw_sysctl_max_unsig_bytes; in rds_iw_xmit()
584 ic->i_rm = rm; in rds_iw_xmit()
611 rm->m_inc.i_hdr.h_ack = cpu_to_be64(rds_iw_piggyb_ack(ic)); in rds_iw_xmit()
617 rds_iw_send_grab_credits(ic, 0, &posted, 1, RDS_MAX_ADV_CREDIT - adv_credits); in rds_iw_xmit()
622 send = &ic->i_sends[pos]; in rds_iw_xmit()
649 rds_iw_xmit_populate_wr(ic, send, pos, 0, 0, send_flags); in rds_iw_xmit()
657 send = &ic->i_sends[pos]; in rds_iw_xmit()
660 rds_iw_xmit_populate_wr(ic, send, pos, in rds_iw_xmit()
669 if (ic->i_unsignaled_wrs-- == 0) { in rds_iw_xmit()
670 ic->i_unsignaled_wrs = rds_iw_sysctl_max_unsig_wrs; in rds_iw_xmit()
674 ic->i_unsignaled_bytes -= len; in rds_iw_xmit()
675 if (ic->i_unsignaled_bytes <= 0) { in rds_iw_xmit()
676 ic->i_unsignaled_bytes = rds_iw_sysctl_max_unsig_bytes; in rds_iw_xmit()
699 memcpy(&ic->i_send_hdrs[pos], &rm->m_inc.i_hdr, sizeof(struct rds_header)); in rds_iw_xmit()
702 struct rds_header *hdr = &ic->i_send_hdrs[pos]; in rds_iw_xmit()
710 struct rds_header *hdr = &ic->i_send_hdrs[pos]; in rds_iw_xmit()
723 pos = (pos + 1) % ic->i_send_ring.w_nr; in rds_iw_xmit()
733 prev->s_rm = ic->i_rm; in rds_iw_xmit()
735 ic->i_rm = NULL; in rds_iw_xmit()
739 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc - i); in rds_iw_xmit()
742 if (ic->i_flowctl && i < credit_alloc) in rds_iw_xmit()
747 ret = ib_post_send(ic->i_cm_id->qp, &first->s_wr, &failed_wr); in rds_iw_xmit()
748 rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic, in rds_iw_xmit()
754 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc); in rds_iw_xmit()
756 ic->i_rm = prev->s_rm; in rds_iw_xmit()
768 static void rds_iw_build_send_fastreg(struct rds_iw_device *rds_iwdev, struct rds_iw_connection *ic… in rds_iw_build_send_fastreg() argument
790 struct rds_iw_connection *ic = conn->c_transport_data; in rds_iw_xmit_rdma() local
807 rds_iwdev = ib_get_client_data(ic->i_cm_id->device, &rds_iw_client); in rds_iw_xmit_rdma()
811 op->op_count = ib_dma_map_sg(ic->i_cm_id->device, in rds_iw_xmit_rdma()
814 rdsdebug("ic %p mapping op %p: %d\n", ic, op, op->op_count); in rds_iw_xmit_rdma()
826 work_alloc = rds_iw_ring_alloc(&ic->i_send_ring, 1, &fr_pos); in rds_iw_xmit_rdma()
828 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc); in rds_iw_xmit_rdma()
841 work_alloc = rds_iw_ring_alloc(&ic->i_send_ring, i, &pos); in rds_iw_xmit_rdma()
843 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc); in rds_iw_xmit_rdma()
849 send = &ic->i_sends[pos]; in rds_iw_xmit_rdma()
851 first = prev = &ic->i_sends[fr_pos]; in rds_iw_xmit_rdma()
868 if (ic->i_unsignaled_wrs-- == 0) { in rds_iw_xmit_rdma()
869 ic->i_unsignaled_wrs = rds_iw_sysctl_max_unsig_wrs; in rds_iw_xmit_rdma()
898 len = ib_sg_dma_len(ic->i_cm_id->device, scat); in rds_iw_xmit_rdma()
901 send->s_page_list->page_list[j] = ib_sg_dma_address(ic->i_cm_id->device, scat); in rds_iw_xmit_rdma()
903 send->s_sge[j].addr = ib_sg_dma_address(ic->i_cm_id->device, scat); in rds_iw_xmit_rdma()
905 send->s_sge[j].lkey = rds_iw_local_dma_lkey(ic); in rds_iw_xmit_rdma()
909 rdsdebug("ic %p sent %d remote_addr %llu\n", ic, sent, remote_addr); in rds_iw_xmit_rdma()
919 send->s_sge[0].lkey = ic->i_sends[fr_pos].s_mr->lkey; in rds_iw_xmit_rdma()
926 if (++send == &ic->i_sends[ic->i_send_ring.w_nr]) in rds_iw_xmit_rdma()
927 send = ic->i_sends; in rds_iw_xmit_rdma()
935 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc - i); in rds_iw_xmit_rdma()
946 rds_iw_build_send_fastreg(rds_iwdev, ic, &ic->i_sends[fr_pos], in rds_iw_xmit_rdma()
952 ret = ib_post_send(ic->i_cm_id->qp, &first->s_wr, &failed_wr); in rds_iw_xmit_rdma()
953 rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic, in rds_iw_xmit_rdma()
959 rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc); in rds_iw_xmit_rdma()
969 struct rds_iw_connection *ic = conn->c_transport_data; in rds_iw_xmit_complete() local
973 rds_iw_attempt_ack(ic); in rds_iw_xmit_complete()