Lines Matching refs:ic
47 void rds_ib_recv_init_ring(struct rds_ib_connection *ic) in rds_ib_recv_init_ring() argument
52 for (i = 0, recv = ic->i_recvs; i < ic->i_recv_ring.w_nr; i++, recv++) { in rds_ib_recv_init_ring()
64 sge->addr = ic->i_recv_hdrs_dma[i]; in rds_ib_recv_init_ring()
66 sge->lkey = ic->i_pd->local_dma_lkey; in rds_ib_recv_init_ring()
71 sge->lkey = ic->i_pd->local_dma_lkey; in rds_ib_recv_init_ring()
121 int rds_ib_recv_alloc_caches(struct rds_ib_connection *ic, gfp_t gfp) in rds_ib_recv_alloc_caches() argument
125 ret = rds_ib_recv_alloc_cache(&ic->i_cache_incs, gfp); in rds_ib_recv_alloc_caches()
127 ret = rds_ib_recv_alloc_cache(&ic->i_cache_frags, gfp); in rds_ib_recv_alloc_caches()
129 free_percpu(ic->i_cache_incs.percpu); in rds_ib_recv_alloc_caches()
155 void rds_ib_recv_free_caches(struct rds_ib_connection *ic) in rds_ib_recv_free_caches() argument
163 rds_ib_cache_xfer_to_ready(&ic->i_cache_incs); in rds_ib_recv_free_caches()
164 rds_ib_cache_splice_all_lists(&ic->i_cache_incs, &list); in rds_ib_recv_free_caches()
165 free_percpu(ic->i_cache_incs.percpu); in rds_ib_recv_free_caches()
174 rds_ib_cache_xfer_to_ready(&ic->i_cache_frags); in rds_ib_recv_free_caches()
175 rds_ib_cache_splice_all_lists(&ic->i_cache_frags, &list); in rds_ib_recv_free_caches()
176 free_percpu(ic->i_cache_frags.percpu); in rds_ib_recv_free_caches()
192 static void rds_ib_frag_free(struct rds_ib_connection *ic, in rds_ib_frag_free() argument
197 rds_ib_recv_cache_put(&frag->f_cache_entry, &ic->i_cache_frags); in rds_ib_frag_free()
198 atomic_add(RDS_FRAG_SIZE / SZ_1K, &ic->i_cache_allocs); in rds_ib_frag_free()
208 struct rds_ib_connection *ic = inc->i_conn->c_transport_data; in rds_ib_inc_free() local
215 rds_ib_frag_free(ic, frag); in rds_ib_inc_free()
220 rds_ib_recv_cache_put(&ibinc->ii_cache_entry, &ic->i_cache_incs); in rds_ib_inc_free()
223 static void rds_ib_recv_clear_one(struct rds_ib_connection *ic, in rds_ib_recv_clear_one() argument
231 ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1, DMA_FROM_DEVICE); in rds_ib_recv_clear_one()
232 rds_ib_frag_free(ic, recv->r_frag); in rds_ib_recv_clear_one()
237 void rds_ib_recv_clear_ring(struct rds_ib_connection *ic) in rds_ib_recv_clear_ring() argument
241 for (i = 0; i < ic->i_recv_ring.w_nr; i++) in rds_ib_recv_clear_ring()
242 rds_ib_recv_clear_one(ic, &ic->i_recvs[i]); in rds_ib_recv_clear_ring()
245 static struct rds_ib_incoming *rds_ib_refill_one_inc(struct rds_ib_connection *ic, in rds_ib_refill_one_inc() argument
252 cache_item = rds_ib_recv_cache_get(&ic->i_cache_incs); in rds_ib_refill_one_inc()
270 rds_inc_init(&ibinc->ii_inc, ic->conn, &ic->conn->c_faddr); in rds_ib_refill_one_inc()
275 static struct rds_page_frag *rds_ib_refill_one_frag(struct rds_ib_connection *ic, in rds_ib_refill_one_frag() argument
282 cache_item = rds_ib_recv_cache_get(&ic->i_cache_frags); in rds_ib_refill_one_frag()
285 atomic_sub(RDS_FRAG_SIZE / SZ_1K, &ic->i_cache_allocs); in rds_ib_refill_one_frag()
310 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_recv_refill_one() local
321 if (!ic->i_cache_incs.ready) in rds_ib_recv_refill_one()
322 rds_ib_cache_xfer_to_ready(&ic->i_cache_incs); in rds_ib_recv_refill_one()
323 if (!ic->i_cache_frags.ready) in rds_ib_recv_refill_one()
324 rds_ib_cache_xfer_to_ready(&ic->i_cache_frags); in rds_ib_recv_refill_one()
331 recv->r_ibinc = rds_ib_refill_one_inc(ic, slab_mask); in rds_ib_recv_refill_one()
337 recv->r_frag = rds_ib_refill_one_frag(ic, slab_mask, page_mask); in rds_ib_recv_refill_one()
341 ret = ib_dma_map_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, in rds_ib_recv_refill_one()
346 sge->addr = ic->i_recv_hdrs_dma[recv - ic->i_recvs]; in rds_ib_recv_refill_one()
384 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_recv_refill() local
400 rds_ib_ring_alloc(&ic->i_recv_ring, 1, &pos)) { in rds_ib_recv_refill()
401 if (pos >= ic->i_recv_ring.w_nr) { in rds_ib_recv_refill()
407 recv = &ic->i_recvs[pos]; in rds_ib_recv_refill()
419 ret = ib_post_recv(ic->i_cm_id->qp, &recv->r_wr, NULL); in rds_ib_recv_refill()
437 if (ic->i_flowctl && posted) in rds_ib_recv_refill()
441 rds_ib_ring_unalloc(&ic->i_recv_ring, 1); in rds_ib_recv_refill()
457 (can_wait && rds_ib_ring_low(&ic->i_recv_ring)) || in rds_ib_recv_refill()
458 rds_ib_ring_empty(&ic->i_recv_ring))) { in rds_ib_recv_refill()
574 void rds_ib_recv_init_ack(struct rds_ib_connection *ic) in rds_ib_recv_init_ack() argument
576 struct ib_send_wr *wr = &ic->i_ack_wr; in rds_ib_recv_init_ack()
577 struct ib_sge *sge = &ic->i_ack_sge; in rds_ib_recv_init_ack()
579 sge->addr = ic->i_ack_dma; in rds_ib_recv_init_ack()
581 sge->lkey = ic->i_pd->local_dma_lkey; in rds_ib_recv_init_ack()
613 void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq, int ack_required) in rds_ib_set_ack() argument
617 spin_lock_irqsave(&ic->i_ack_lock, flags); in rds_ib_set_ack()
618 ic->i_ack_next = seq; in rds_ib_set_ack()
620 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_set_ack()
621 spin_unlock_irqrestore(&ic->i_ack_lock, flags); in rds_ib_set_ack()
624 static u64 rds_ib_get_ack(struct rds_ib_connection *ic) in rds_ib_get_ack() argument
629 clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_get_ack()
631 spin_lock_irqsave(&ic->i_ack_lock, flags); in rds_ib_get_ack()
632 seq = ic->i_ack_next; in rds_ib_get_ack()
633 spin_unlock_irqrestore(&ic->i_ack_lock, flags); in rds_ib_get_ack()
638 void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq, int ack_required) in rds_ib_set_ack() argument
640 atomic64_set(&ic->i_ack_next, seq); in rds_ib_set_ack()
643 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_set_ack()
647 static u64 rds_ib_get_ack(struct rds_ib_connection *ic) in rds_ib_get_ack() argument
649 clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_get_ack()
652 return atomic64_read(&ic->i_ack_next); in rds_ib_get_ack()
657 static void rds_ib_send_ack(struct rds_ib_connection *ic, unsigned int adv_credits) in rds_ib_send_ack() argument
659 struct rds_header *hdr = ic->i_ack; in rds_ib_send_ack()
663 seq = rds_ib_get_ack(ic); in rds_ib_send_ack()
665 rdsdebug("send_ack: ic %p ack %llu\n", ic, (unsigned long long) seq); in rds_ib_send_ack()
667 ib_dma_sync_single_for_cpu(ic->rds_ibdev->dev, ic->i_ack_dma, in rds_ib_send_ack()
673 ib_dma_sync_single_for_device(ic->rds_ibdev->dev, ic->i_ack_dma, in rds_ib_send_ack()
676 ic->i_ack_queued = jiffies; in rds_ib_send_ack()
678 ret = ib_post_send(ic->i_cm_id->qp, &ic->i_ack_wr, NULL); in rds_ib_send_ack()
683 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_ib_send_ack()
684 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_send_ack()
688 rds_ib_conn_error(ic->conn, "sending ack failed\n"); in rds_ib_send_ack()
731 void rds_ib_attempt_ack(struct rds_ib_connection *ic) in rds_ib_attempt_ack() argument
735 if (!test_bit(IB_ACK_REQUESTED, &ic->i_ack_flags)) in rds_ib_attempt_ack()
738 if (test_and_set_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags)) { in rds_ib_attempt_ack()
744 if (!rds_ib_send_grab_credits(ic, 1, &adv_credits, 0, RDS_MAX_ADV_CREDIT)) { in rds_ib_attempt_ack()
746 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_ib_attempt_ack()
750 clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_attempt_ack()
751 rds_ib_send_ack(ic, adv_credits); in rds_ib_attempt_ack()
758 void rds_ib_ack_send_complete(struct rds_ib_connection *ic) in rds_ib_ack_send_complete() argument
760 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_ib_ack_send_complete()
761 rds_ib_attempt_ack(ic); in rds_ib_ack_send_complete()
768 u64 rds_ib_piggyb_ack(struct rds_ib_connection *ic) in rds_ib_piggyb_ack() argument
770 if (test_and_clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags)) in rds_ib_piggyb_ack()
772 return rds_ib_get_ack(ic); in rds_ib_piggyb_ack()
852 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_process_recv() local
853 struct rds_ib_incoming *ibinc = ic->i_ibinc; in rds_ib_process_recv()
855 dma_addr_t dma_addr = ic->i_recv_hdrs_dma[recv - ic->i_recvs]; in rds_ib_process_recv()
859 rdsdebug("ic %p ibinc %p recv %p byte len %u\n", ic, ibinc, recv, in rds_ib_process_recv()
872 ihdr = ic->i_recv_hdrs[recv - ic->i_recvs]; in rds_ib_process_recv()
874 ib_dma_sync_single_for_cpu(ic->rds_ibdev->dev, dma_addr, in rds_ib_process_recv()
910 rds_ib_frag_free(ic, recv->r_frag); in rds_ib_process_recv()
924 ic->i_ibinc = ibinc; in rds_ib_process_recv()
930 ic->i_recv_data_rem = be32_to_cpu(hdr->h_len); in rds_ib_process_recv()
934 rdsdebug("ic %p ibinc %p rem %u flag 0x%x\n", ic, ibinc, in rds_ib_process_recv()
935 ic->i_recv_data_rem, hdr->h_flags); in rds_ib_process_recv()
953 if (ic->i_recv_data_rem > RDS_FRAG_SIZE) in rds_ib_process_recv()
954 ic->i_recv_data_rem -= RDS_FRAG_SIZE; in rds_ib_process_recv()
956 ic->i_recv_data_rem = 0; in rds_ib_process_recv()
957 ic->i_ibinc = NULL; in rds_ib_process_recv()
979 ib_dma_sync_single_for_device(ic->rds_ibdev->dev, dma_addr, in rds_ib_process_recv()
983 void rds_ib_recv_cqe_handler(struct rds_ib_connection *ic, in rds_ib_recv_cqe_handler() argument
987 struct rds_connection *conn = ic->conn; in rds_ib_recv_cqe_handler()
996 recv = &ic->i_recvs[rds_ib_ring_oldest(&ic->i_recv_ring)]; in rds_ib_recv_cqe_handler()
997 ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1, in rds_ib_recv_cqe_handler()
1024 rds_ib_frag_free(ic, recv->r_frag); in rds_ib_recv_cqe_handler()
1027 rds_ib_ring_free(&ic->i_recv_ring, 1); in rds_ib_recv_cqe_handler()
1032 if (rds_ib_ring_empty(&ic->i_recv_ring)) in rds_ib_recv_cqe_handler()
1035 if (rds_ib_ring_low(&ic->i_recv_ring)) { in rds_ib_recv_cqe_handler()
1044 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_recv_path() local
1048 rds_ib_attempt_ack(ic); in rds_ib_recv_path()