Lines Matching refs:ic
69 static void rds_iw_recv_unmap_page(struct rds_iw_connection *ic, in rds_iw_recv_unmap_page() argument
76 ib_dma_unmap_page(ic->i_cm_id->device, in rds_iw_recv_unmap_page()
82 void rds_iw_recv_init_ring(struct rds_iw_connection *ic) in rds_iw_recv_init_ring() argument
87 for (i = 0, recv = ic->i_recvs; i < ic->i_recv_ring.w_nr; i++, recv++) { in rds_iw_recv_init_ring()
98 sge = rds_iw_data_sge(ic, recv->r_sge); in rds_iw_recv_init_ring()
103 sge = rds_iw_header_sge(ic, recv->r_sge); in rds_iw_recv_init_ring()
104 sge->addr = ic->i_recv_hdrs_dma + (i * sizeof(struct rds_header)); in rds_iw_recv_init_ring()
110 static void rds_iw_recv_clear_one(struct rds_iw_connection *ic, in rds_iw_recv_clear_one() argument
118 rds_iw_recv_unmap_page(ic, recv); in rds_iw_recv_clear_one()
126 void rds_iw_recv_clear_ring(struct rds_iw_connection *ic) in rds_iw_recv_clear_ring() argument
130 for (i = 0; i < ic->i_recv_ring.w_nr; i++) in rds_iw_recv_clear_ring()
131 rds_iw_recv_clear_one(ic, &ic->i_recvs[i]); in rds_iw_recv_clear_ring()
133 if (ic->i_frag.f_page) in rds_iw_recv_clear_ring()
134 rds_iw_frag_drop_page(&ic->i_frag); in rds_iw_recv_clear_ring()
141 struct rds_iw_connection *ic = conn->c_transport_data; in rds_iw_recv_refill_one() local
169 if (!ic->i_frag.f_page) { in rds_iw_recv_refill_one()
170 ic->i_frag.f_page = alloc_page(page_gfp); in rds_iw_recv_refill_one()
171 if (!ic->i_frag.f_page) in rds_iw_recv_refill_one()
173 ic->i_frag.f_offset = 0; in rds_iw_recv_refill_one()
176 dma_addr = ib_dma_map_page(ic->i_cm_id->device, in rds_iw_recv_refill_one()
177 ic->i_frag.f_page, in rds_iw_recv_refill_one()
178 ic->i_frag.f_offset, in rds_iw_recv_refill_one()
181 if (ib_dma_mapping_error(ic->i_cm_id->device, dma_addr)) in rds_iw_recv_refill_one()
189 recv->r_frag->f_page = ic->i_frag.f_page; in rds_iw_recv_refill_one()
190 recv->r_frag->f_offset = ic->i_frag.f_offset; in rds_iw_recv_refill_one()
193 sge = rds_iw_data_sge(ic, recv->r_sge); in rds_iw_recv_refill_one()
197 sge = rds_iw_header_sge(ic, recv->r_sge); in rds_iw_recv_refill_one()
198 sge->addr = ic->i_recv_hdrs_dma + (recv - ic->i_recvs) * sizeof(struct rds_header); in rds_iw_recv_refill_one()
203 if (ic->i_frag.f_offset < RDS_PAGE_LAST_OFF) { in rds_iw_recv_refill_one()
204 ic->i_frag.f_offset += RDS_FRAG_SIZE; in rds_iw_recv_refill_one()
206 put_page(ic->i_frag.f_page); in rds_iw_recv_refill_one()
207 ic->i_frag.f_page = NULL; in rds_iw_recv_refill_one()
208 ic->i_frag.f_offset = 0; in rds_iw_recv_refill_one()
227 struct rds_iw_connection *ic = conn->c_transport_data; in rds_iw_recv_refill() local
235 rds_iw_ring_alloc(&ic->i_recv_ring, 1, &pos)) { in rds_iw_recv_refill()
236 if (pos >= ic->i_recv_ring.w_nr) { in rds_iw_recv_refill()
243 recv = &ic->i_recvs[pos]; in rds_iw_recv_refill()
251 ret = ib_post_recv(ic->i_cm_id->qp, &recv->r_wr, &failed_wr); in rds_iw_recv_refill()
268 if (ic->i_flowctl && posted) in rds_iw_recv_refill()
272 rds_iw_ring_unalloc(&ic->i_recv_ring, 1); in rds_iw_recv_refill()
362 void rds_iw_recv_init_ack(struct rds_iw_connection *ic) in rds_iw_recv_init_ack() argument
364 struct ib_send_wr *wr = &ic->i_ack_wr; in rds_iw_recv_init_ack()
365 struct ib_sge *sge = &ic->i_ack_sge; in rds_iw_recv_init_ack()
367 sge->addr = ic->i_ack_dma; in rds_iw_recv_init_ack()
369 sge->lkey = rds_iw_local_dma_lkey(ic); in rds_iw_recv_init_ack()
401 static void rds_iw_set_ack(struct rds_iw_connection *ic, u64 seq, in rds_iw_set_ack() argument
406 spin_lock_irqsave(&ic->i_ack_lock, flags); in rds_iw_set_ack()
407 ic->i_ack_next = seq; in rds_iw_set_ack()
409 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_iw_set_ack()
410 spin_unlock_irqrestore(&ic->i_ack_lock, flags); in rds_iw_set_ack()
413 static u64 rds_iw_get_ack(struct rds_iw_connection *ic) in rds_iw_get_ack() argument
418 clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_iw_get_ack()
420 spin_lock_irqsave(&ic->i_ack_lock, flags); in rds_iw_get_ack()
421 seq = ic->i_ack_next; in rds_iw_get_ack()
422 spin_unlock_irqrestore(&ic->i_ack_lock, flags); in rds_iw_get_ack()
427 static void rds_iw_set_ack(struct rds_iw_connection *ic, u64 seq, in rds_iw_set_ack() argument
430 atomic64_set(&ic->i_ack_next, seq); in rds_iw_set_ack()
433 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_iw_set_ack()
437 static u64 rds_iw_get_ack(struct rds_iw_connection *ic) in rds_iw_get_ack() argument
439 clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_iw_get_ack()
442 return atomic64_read(&ic->i_ack_next); in rds_iw_get_ack()
447 static void rds_iw_send_ack(struct rds_iw_connection *ic, unsigned int adv_credits) in rds_iw_send_ack() argument
449 struct rds_header *hdr = ic->i_ack; in rds_iw_send_ack()
454 seq = rds_iw_get_ack(ic); in rds_iw_send_ack()
456 rdsdebug("send_ack: ic %p ack %llu\n", ic, (unsigned long long) seq); in rds_iw_send_ack()
461 ic->i_ack_queued = jiffies; in rds_iw_send_ack()
463 ret = ib_post_send(ic->i_cm_id->qp, &ic->i_ack_wr, &failed_wr); in rds_iw_send_ack()
468 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_iw_send_ack()
469 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_iw_send_ack()
473 rds_iw_conn_error(ic->conn, "sending ack failed\n"); in rds_iw_send_ack()
516 void rds_iw_attempt_ack(struct rds_iw_connection *ic) in rds_iw_attempt_ack() argument
520 if (!test_bit(IB_ACK_REQUESTED, &ic->i_ack_flags)) in rds_iw_attempt_ack()
523 if (test_and_set_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags)) { in rds_iw_attempt_ack()
529 if (!rds_iw_send_grab_credits(ic, 1, &adv_credits, 0, RDS_MAX_ADV_CREDIT)) { in rds_iw_attempt_ack()
531 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_iw_attempt_ack()
535 clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_iw_attempt_ack()
536 rds_iw_send_ack(ic, adv_credits); in rds_iw_attempt_ack()
543 void rds_iw_ack_send_complete(struct rds_iw_connection *ic) in rds_iw_ack_send_complete() argument
545 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_iw_ack_send_complete()
546 rds_iw_attempt_ack(ic); in rds_iw_ack_send_complete()
553 u64 rds_iw_piggyb_ack(struct rds_iw_connection *ic) in rds_iw_piggyb_ack() argument
555 if (test_and_clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags)) in rds_iw_piggyb_ack()
557 return rds_iw_get_ack(ic); in rds_iw_piggyb_ack()
653 struct rds_iw_connection *ic = conn->c_transport_data; in rds_iw_process_recv() local
654 struct rds_iw_incoming *iwinc = ic->i_iwinc; in rds_iw_process_recv()
659 rdsdebug("ic %p iwinc %p recv %p byte len %u\n", ic, iwinc, recv, in rds_iw_process_recv()
672 ihdr = &ic->i_recv_hdrs[recv - ic->i_recvs]; in rds_iw_process_recv()
722 ic->i_iwinc = iwinc; in rds_iw_process_recv()
726 ic->i_recv_data_rem = be32_to_cpu(hdr->h_len); in rds_iw_process_recv()
728 rdsdebug("ic %p iwinc %p rem %u flag 0x%x\n", ic, iwinc, in rds_iw_process_recv()
729 ic->i_recv_data_rem, hdr->h_flags); in rds_iw_process_recv()
747 if (ic->i_recv_data_rem > RDS_FRAG_SIZE) in rds_iw_process_recv()
748 ic->i_recv_data_rem -= RDS_FRAG_SIZE; in rds_iw_process_recv()
750 ic->i_recv_data_rem = 0; in rds_iw_process_recv()
751 ic->i_iwinc = NULL; in rds_iw_process_recv()
786 struct rds_iw_connection *ic = conn->c_transport_data; in rds_iw_recv_cq_comp_handler() local
792 tasklet_schedule(&ic->i_recv_tasklet); in rds_iw_recv_cq_comp_handler()
795 static inline void rds_poll_cq(struct rds_iw_connection *ic, in rds_poll_cq() argument
798 struct rds_connection *conn = ic->conn; in rds_poll_cq()
802 while (ib_poll_cq(ic->i_recv_cq, 1, &wc) > 0) { in rds_poll_cq()
808 recv = &ic->i_recvs[rds_iw_ring_oldest(&ic->i_recv_ring)]; in rds_poll_cq()
810 rds_iw_recv_unmap_page(ic, recv); in rds_poll_cq()
829 rds_iw_ring_free(&ic->i_recv_ring, 1); in rds_poll_cq()
835 struct rds_iw_connection *ic = (struct rds_iw_connection *) data; in rds_iw_recv_tasklet_fn() local
836 struct rds_connection *conn = ic->conn; in rds_iw_recv_tasklet_fn()
839 rds_poll_cq(ic, &state); in rds_iw_recv_tasklet_fn()
840 ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED); in rds_iw_recv_tasklet_fn()
841 rds_poll_cq(ic, &state); in rds_iw_recv_tasklet_fn()
844 rds_iw_set_ack(ic, state.ack_next, state.ack_required); in rds_iw_recv_tasklet_fn()
845 if (state.ack_recv_valid && state.ack_recv > ic->i_ack_recv) { in rds_iw_recv_tasklet_fn()
847 ic->i_ack_recv = state.ack_recv; in rds_iw_recv_tasklet_fn()
850 rds_iw_attempt_ack(ic); in rds_iw_recv_tasklet_fn()
855 if (rds_iw_ring_empty(&ic->i_recv_ring)) in rds_iw_recv_tasklet_fn()
861 if (rds_iw_ring_low(&ic->i_recv_ring)) in rds_iw_recv_tasklet_fn()
867 struct rds_iw_connection *ic = conn->c_transport_data; in rds_iw_recv() local
876 mutex_lock(&ic->i_recv_mutex); in rds_iw_recv()
881 mutex_unlock(&ic->i_recv_mutex); in rds_iw_recv()
884 rds_iw_attempt_ack(ic); in rds_iw_recv()