Lines Matching full:ic
47 void rds_ib_recv_init_ring(struct rds_ib_connection *ic) in rds_ib_recv_init_ring() argument
52 for (i = 0, recv = ic->i_recvs; i < ic->i_recv_ring.w_nr; i++, recv++) { in rds_ib_recv_init_ring()
64 sge->addr = ic->i_recv_hdrs_dma + (i * sizeof(struct rds_header)); in rds_ib_recv_init_ring()
66 sge->lkey = ic->i_pd->local_dma_lkey; in rds_ib_recv_init_ring()
71 sge->lkey = ic->i_pd->local_dma_lkey; in rds_ib_recv_init_ring()
121 int rds_ib_recv_alloc_caches(struct rds_ib_connection *ic, gfp_t gfp) in rds_ib_recv_alloc_caches() argument
125 ret = rds_ib_recv_alloc_cache(&ic->i_cache_incs, gfp); in rds_ib_recv_alloc_caches()
127 ret = rds_ib_recv_alloc_cache(&ic->i_cache_frags, gfp); in rds_ib_recv_alloc_caches()
129 free_percpu(ic->i_cache_incs.percpu); in rds_ib_recv_alloc_caches()
155 void rds_ib_recv_free_caches(struct rds_ib_connection *ic) in rds_ib_recv_free_caches() argument
163 rds_ib_cache_xfer_to_ready(&ic->i_cache_incs); in rds_ib_recv_free_caches()
164 rds_ib_cache_splice_all_lists(&ic->i_cache_incs, &list); in rds_ib_recv_free_caches()
165 free_percpu(ic->i_cache_incs.percpu); in rds_ib_recv_free_caches()
173 rds_ib_cache_xfer_to_ready(&ic->i_cache_frags); in rds_ib_recv_free_caches()
174 rds_ib_cache_splice_all_lists(&ic->i_cache_frags, &list); in rds_ib_recv_free_caches()
175 free_percpu(ic->i_cache_frags.percpu); in rds_ib_recv_free_caches()
191 static void rds_ib_frag_free(struct rds_ib_connection *ic, in rds_ib_frag_free() argument
196 rds_ib_recv_cache_put(&frag->f_cache_entry, &ic->i_cache_frags); in rds_ib_frag_free()
197 atomic_add(RDS_FRAG_SIZE / SZ_1K, &ic->i_cache_allocs); in rds_ib_frag_free()
207 struct rds_ib_connection *ic = inc->i_conn->c_transport_data; in rds_ib_inc_free() local
214 rds_ib_frag_free(ic, frag); in rds_ib_inc_free()
219 rds_ib_recv_cache_put(&ibinc->ii_cache_entry, &ic->i_cache_incs); in rds_ib_inc_free()
222 static void rds_ib_recv_clear_one(struct rds_ib_connection *ic, in rds_ib_recv_clear_one() argument
230 ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1, DMA_FROM_DEVICE); in rds_ib_recv_clear_one()
231 rds_ib_frag_free(ic, recv->r_frag); in rds_ib_recv_clear_one()
236 void rds_ib_recv_clear_ring(struct rds_ib_connection *ic) in rds_ib_recv_clear_ring() argument
240 for (i = 0; i < ic->i_recv_ring.w_nr; i++) in rds_ib_recv_clear_ring()
241 rds_ib_recv_clear_one(ic, &ic->i_recvs[i]); in rds_ib_recv_clear_ring()
244 static struct rds_ib_incoming *rds_ib_refill_one_inc(struct rds_ib_connection *ic, in rds_ib_refill_one_inc() argument
251 cache_item = rds_ib_recv_cache_get(&ic->i_cache_incs); in rds_ib_refill_one_inc()
269 rds_inc_init(&ibinc->ii_inc, ic->conn, &ic->conn->c_faddr); in rds_ib_refill_one_inc()
274 static struct rds_page_frag *rds_ib_refill_one_frag(struct rds_ib_connection *ic, in rds_ib_refill_one_frag() argument
281 cache_item = rds_ib_recv_cache_get(&ic->i_cache_frags); in rds_ib_refill_one_frag()
284 atomic_sub(RDS_FRAG_SIZE / SZ_1K, &ic->i_cache_allocs); in rds_ib_refill_one_frag()
309 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_recv_refill_one() local
320 if (!ic->i_cache_incs.ready) in rds_ib_recv_refill_one()
321 rds_ib_cache_xfer_to_ready(&ic->i_cache_incs); in rds_ib_recv_refill_one()
322 if (!ic->i_cache_frags.ready) in rds_ib_recv_refill_one()
323 rds_ib_cache_xfer_to_ready(&ic->i_cache_frags); in rds_ib_recv_refill_one()
330 recv->r_ibinc = rds_ib_refill_one_inc(ic, slab_mask); in rds_ib_recv_refill_one()
336 recv->r_frag = rds_ib_refill_one_frag(ic, slab_mask, page_mask); in rds_ib_recv_refill_one()
340 ret = ib_dma_map_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, in rds_ib_recv_refill_one()
345 sge->addr = ic->i_recv_hdrs_dma + (recv - ic->i_recvs) * sizeof(struct rds_header); in rds_ib_recv_refill_one()
349 sge->addr = ib_sg_dma_address(ic->i_cm_id->device, &recv->r_frag->f_sg); in rds_ib_recv_refill_one()
350 sge->length = ib_sg_dma_len(ic->i_cm_id->device, &recv->r_frag->f_sg); in rds_ib_recv_refill_one()
382 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_recv_refill() local
397 rds_ib_ring_alloc(&ic->i_recv_ring, 1, &pos)) { in rds_ib_recv_refill()
398 if (pos >= ic->i_recv_ring.w_nr) { in rds_ib_recv_refill()
404 recv = &ic->i_recvs[pos]; in rds_ib_recv_refill()
413 ic->i_cm_id->device, in rds_ib_recv_refill()
417 ret = ib_post_recv(ic->i_cm_id->qp, &recv->r_wr, NULL); in rds_ib_recv_refill()
430 if (ic->i_flowctl && posted) in rds_ib_recv_refill()
434 rds_ib_ring_unalloc(&ic->i_recv_ring, 1); in rds_ib_recv_refill()
449 ((can_wait && rds_ib_ring_low(&ic->i_recv_ring)) || in rds_ib_recv_refill()
450 rds_ib_ring_empty(&ic->i_recv_ring))) { in rds_ib_recv_refill()
563 /* ic starts out kzalloc()ed */
564 void rds_ib_recv_init_ack(struct rds_ib_connection *ic) in rds_ib_recv_init_ack() argument
566 struct ib_send_wr *wr = &ic->i_ack_wr; in rds_ib_recv_init_ack()
567 struct ib_sge *sge = &ic->i_ack_sge; in rds_ib_recv_init_ack()
569 sge->addr = ic->i_ack_dma; in rds_ib_recv_init_ack()
571 sge->lkey = ic->i_pd->local_dma_lkey; in rds_ib_recv_init_ack()
603 void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq, int ack_required) in rds_ib_set_ack() argument
607 spin_lock_irqsave(&ic->i_ack_lock, flags); in rds_ib_set_ack()
608 ic->i_ack_next = seq; in rds_ib_set_ack()
610 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_set_ack()
611 spin_unlock_irqrestore(&ic->i_ack_lock, flags); in rds_ib_set_ack()
614 static u64 rds_ib_get_ack(struct rds_ib_connection *ic) in rds_ib_get_ack() argument
619 clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_get_ack()
621 spin_lock_irqsave(&ic->i_ack_lock, flags); in rds_ib_get_ack()
622 seq = ic->i_ack_next; in rds_ib_get_ack()
623 spin_unlock_irqrestore(&ic->i_ack_lock, flags); in rds_ib_get_ack()
628 void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq, int ack_required) in rds_ib_set_ack() argument
630 atomic64_set(&ic->i_ack_next, seq); in rds_ib_set_ack()
633 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_set_ack()
637 static u64 rds_ib_get_ack(struct rds_ib_connection *ic) in rds_ib_get_ack() argument
639 clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_get_ack()
642 return atomic64_read(&ic->i_ack_next); in rds_ib_get_ack()
647 static void rds_ib_send_ack(struct rds_ib_connection *ic, unsigned int adv_credits) in rds_ib_send_ack() argument
649 struct rds_header *hdr = ic->i_ack; in rds_ib_send_ack()
653 seq = rds_ib_get_ack(ic); in rds_ib_send_ack()
655 rdsdebug("send_ack: ic %p ack %llu\n", ic, (unsigned long long) seq); in rds_ib_send_ack()
660 ic->i_ack_queued = jiffies; in rds_ib_send_ack()
662 ret = ib_post_send(ic->i_cm_id->qp, &ic->i_ack_wr, NULL); in rds_ib_send_ack()
667 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_ib_send_ack()
668 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_send_ack()
672 rds_ib_conn_error(ic->conn, "sending ack failed\n"); in rds_ib_send_ack()
715 void rds_ib_attempt_ack(struct rds_ib_connection *ic) in rds_ib_attempt_ack() argument
719 if (!test_bit(IB_ACK_REQUESTED, &ic->i_ack_flags)) in rds_ib_attempt_ack()
722 if (test_and_set_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags)) { in rds_ib_attempt_ack()
728 if (!rds_ib_send_grab_credits(ic, 1, &adv_credits, 0, RDS_MAX_ADV_CREDIT)) { in rds_ib_attempt_ack()
730 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_ib_attempt_ack()
734 clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_attempt_ack()
735 rds_ib_send_ack(ic, adv_credits); in rds_ib_attempt_ack()
742 void rds_ib_ack_send_complete(struct rds_ib_connection *ic) in rds_ib_ack_send_complete() argument
744 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_ib_ack_send_complete()
745 rds_ib_attempt_ack(ic); in rds_ib_ack_send_complete()
752 u64 rds_ib_piggyb_ack(struct rds_ib_connection *ic) in rds_ib_piggyb_ack() argument
754 if (test_and_clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags)) in rds_ib_piggyb_ack()
756 return rds_ib_get_ack(ic); in rds_ib_piggyb_ack()
838 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_process_recv() local
839 struct rds_ib_incoming *ibinc = ic->i_ibinc; in rds_ib_process_recv()
844 rdsdebug("ic %p ibinc %p recv %p byte len %u\n", ic, ibinc, recv, in rds_ib_process_recv()
857 ihdr = &ic->i_recv_hdrs[recv - ic->i_recvs]; in rds_ib_process_recv()
893 rds_ib_frag_free(ic, recv->r_frag); in rds_ib_process_recv()
907 ic->i_ibinc = ibinc; in rds_ib_process_recv()
913 ic->i_recv_data_rem = be32_to_cpu(hdr->h_len); in rds_ib_process_recv()
917 rdsdebug("ic %p ibinc %p rem %u flag 0x%x\n", ic, ibinc, in rds_ib_process_recv()
918 ic->i_recv_data_rem, hdr->h_flags); in rds_ib_process_recv()
936 if (ic->i_recv_data_rem > RDS_FRAG_SIZE) in rds_ib_process_recv()
937 ic->i_recv_data_rem -= RDS_FRAG_SIZE; in rds_ib_process_recv()
939 ic->i_recv_data_rem = 0; in rds_ib_process_recv()
940 ic->i_ibinc = NULL; in rds_ib_process_recv()
963 void rds_ib_recv_cqe_handler(struct rds_ib_connection *ic, in rds_ib_recv_cqe_handler() argument
967 struct rds_connection *conn = ic->conn; in rds_ib_recv_cqe_handler()
976 recv = &ic->i_recvs[rds_ib_ring_oldest(&ic->i_recv_ring)]; in rds_ib_recv_cqe_handler()
977 ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1, in rds_ib_recv_cqe_handler()
1003 rds_ib_frag_free(ic, recv->r_frag); in rds_ib_recv_cqe_handler()
1006 rds_ib_ring_free(&ic->i_recv_ring, 1); in rds_ib_recv_cqe_handler()
1011 if (rds_ib_ring_empty(&ic->i_recv_ring)) in rds_ib_recv_cqe_handler()
1014 if (rds_ib_ring_low(&ic->i_recv_ring)) { in rds_ib_recv_cqe_handler()
1023 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_recv_path() local
1027 rds_ib_attempt_ack(ic); in rds_ib_recv_path()