Lines Matching refs:ic
48 void rds_ib_recv_init_ring(struct rds_ib_connection *ic) in rds_ib_recv_init_ring() argument
53 for (i = 0, recv = ic->i_recvs; i < ic->i_recv_ring.w_nr; i++, recv++) { in rds_ib_recv_init_ring()
65 sge->addr = ic->i_recv_hdrs_dma[i]; in rds_ib_recv_init_ring()
67 sge->lkey = ic->i_pd->local_dma_lkey; in rds_ib_recv_init_ring()
72 sge->lkey = ic->i_pd->local_dma_lkey; in rds_ib_recv_init_ring()
122 int rds_ib_recv_alloc_caches(struct rds_ib_connection *ic, gfp_t gfp) in rds_ib_recv_alloc_caches() argument
126 ret = rds_ib_recv_alloc_cache(&ic->i_cache_incs, gfp); in rds_ib_recv_alloc_caches()
128 ret = rds_ib_recv_alloc_cache(&ic->i_cache_frags, gfp); in rds_ib_recv_alloc_caches()
130 free_percpu(ic->i_cache_incs.percpu); in rds_ib_recv_alloc_caches()
156 void rds_ib_recv_free_caches(struct rds_ib_connection *ic) in rds_ib_recv_free_caches() argument
164 rds_ib_cache_xfer_to_ready(&ic->i_cache_incs); in rds_ib_recv_free_caches()
165 rds_ib_cache_splice_all_lists(&ic->i_cache_incs, &list); in rds_ib_recv_free_caches()
166 free_percpu(ic->i_cache_incs.percpu); in rds_ib_recv_free_caches()
175 rds_ib_cache_xfer_to_ready(&ic->i_cache_frags); in rds_ib_recv_free_caches()
176 rds_ib_cache_splice_all_lists(&ic->i_cache_frags, &list); in rds_ib_recv_free_caches()
177 free_percpu(ic->i_cache_frags.percpu); in rds_ib_recv_free_caches()
193 static void rds_ib_frag_free(struct rds_ib_connection *ic, in rds_ib_frag_free() argument
198 rds_ib_recv_cache_put(&frag->f_cache_entry, &ic->i_cache_frags); in rds_ib_frag_free()
199 atomic_add(RDS_FRAG_SIZE / SZ_1K, &ic->i_cache_allocs); in rds_ib_frag_free()
209 struct rds_ib_connection *ic = inc->i_conn->c_transport_data; in rds_ib_inc_free() local
216 rds_ib_frag_free(ic, frag); in rds_ib_inc_free()
221 rds_ib_recv_cache_put(&ibinc->ii_cache_entry, &ic->i_cache_incs); in rds_ib_inc_free()
224 static void rds_ib_recv_clear_one(struct rds_ib_connection *ic, in rds_ib_recv_clear_one() argument
232 ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1, DMA_FROM_DEVICE); in rds_ib_recv_clear_one()
233 rds_ib_frag_free(ic, recv->r_frag); in rds_ib_recv_clear_one()
238 void rds_ib_recv_clear_ring(struct rds_ib_connection *ic) in rds_ib_recv_clear_ring() argument
242 for (i = 0; i < ic->i_recv_ring.w_nr; i++) in rds_ib_recv_clear_ring()
243 rds_ib_recv_clear_one(ic, &ic->i_recvs[i]); in rds_ib_recv_clear_ring()
246 static struct rds_ib_incoming *rds_ib_refill_one_inc(struct rds_ib_connection *ic, in rds_ib_refill_one_inc() argument
253 cache_item = rds_ib_recv_cache_get(&ic->i_cache_incs); in rds_ib_refill_one_inc()
271 rds_inc_init(&ibinc->ii_inc, ic->conn, &ic->conn->c_faddr); in rds_ib_refill_one_inc()
276 static struct rds_page_frag *rds_ib_refill_one_frag(struct rds_ib_connection *ic, in rds_ib_refill_one_frag() argument
283 cache_item = rds_ib_recv_cache_get(&ic->i_cache_frags); in rds_ib_refill_one_frag()
286 atomic_sub(RDS_FRAG_SIZE / SZ_1K, &ic->i_cache_allocs); in rds_ib_refill_one_frag()
311 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_recv_refill_one() local
322 if (!ic->i_cache_incs.ready) in rds_ib_recv_refill_one()
323 rds_ib_cache_xfer_to_ready(&ic->i_cache_incs); in rds_ib_recv_refill_one()
324 if (!ic->i_cache_frags.ready) in rds_ib_recv_refill_one()
325 rds_ib_cache_xfer_to_ready(&ic->i_cache_frags); in rds_ib_recv_refill_one()
332 recv->r_ibinc = rds_ib_refill_one_inc(ic, slab_mask); in rds_ib_recv_refill_one()
338 recv->r_frag = rds_ib_refill_one_frag(ic, slab_mask, page_mask); in rds_ib_recv_refill_one()
342 ret = ib_dma_map_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, in rds_ib_recv_refill_one()
347 sge->addr = ic->i_recv_hdrs_dma[recv - ic->i_recvs]; in rds_ib_recv_refill_one()
385 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_recv_refill() local
401 rds_ib_ring_alloc(&ic->i_recv_ring, 1, &pos)) { in rds_ib_recv_refill()
402 if (pos >= ic->i_recv_ring.w_nr) { in rds_ib_recv_refill()
408 recv = &ic->i_recvs[pos]; in rds_ib_recv_refill()
420 ret = ib_post_recv(ic->i_cm_id->qp, &recv->r_wr, NULL); in rds_ib_recv_refill()
438 if (ic->i_flowctl && posted) in rds_ib_recv_refill()
442 rds_ib_ring_unalloc(&ic->i_recv_ring, 1); in rds_ib_recv_refill()
458 (can_wait && rds_ib_ring_low(&ic->i_recv_ring)) || in rds_ib_recv_refill()
459 rds_ib_ring_empty(&ic->i_recv_ring))) { in rds_ib_recv_refill()
575 void rds_ib_recv_init_ack(struct rds_ib_connection *ic) in rds_ib_recv_init_ack() argument
577 struct ib_send_wr *wr = &ic->i_ack_wr; in rds_ib_recv_init_ack()
578 struct ib_sge *sge = &ic->i_ack_sge; in rds_ib_recv_init_ack()
580 sge->addr = ic->i_ack_dma; in rds_ib_recv_init_ack()
582 sge->lkey = ic->i_pd->local_dma_lkey; in rds_ib_recv_init_ack()
614 void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq, int ack_required) in rds_ib_set_ack() argument
618 spin_lock_irqsave(&ic->i_ack_lock, flags); in rds_ib_set_ack()
619 ic->i_ack_next = seq; in rds_ib_set_ack()
621 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_set_ack()
622 spin_unlock_irqrestore(&ic->i_ack_lock, flags); in rds_ib_set_ack()
625 static u64 rds_ib_get_ack(struct rds_ib_connection *ic) in rds_ib_get_ack() argument
630 clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_get_ack()
632 spin_lock_irqsave(&ic->i_ack_lock, flags); in rds_ib_get_ack()
633 seq = ic->i_ack_next; in rds_ib_get_ack()
634 spin_unlock_irqrestore(&ic->i_ack_lock, flags); in rds_ib_get_ack()
639 void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq, int ack_required) in rds_ib_set_ack() argument
641 atomic64_set(&ic->i_ack_next, seq); in rds_ib_set_ack()
644 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_set_ack()
648 static u64 rds_ib_get_ack(struct rds_ib_connection *ic) in rds_ib_get_ack() argument
650 clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_get_ack()
653 return atomic64_read(&ic->i_ack_next); in rds_ib_get_ack()
658 static void rds_ib_send_ack(struct rds_ib_connection *ic, unsigned int adv_credits) in rds_ib_send_ack() argument
660 struct rds_header *hdr = ic->i_ack; in rds_ib_send_ack()
664 seq = rds_ib_get_ack(ic); in rds_ib_send_ack()
666 rdsdebug("send_ack: ic %p ack %llu\n", ic, (unsigned long long) seq); in rds_ib_send_ack()
668 ib_dma_sync_single_for_cpu(ic->rds_ibdev->dev, ic->i_ack_dma, in rds_ib_send_ack()
674 ib_dma_sync_single_for_device(ic->rds_ibdev->dev, ic->i_ack_dma, in rds_ib_send_ack()
677 ic->i_ack_queued = jiffies; in rds_ib_send_ack()
679 ret = ib_post_send(ic->i_cm_id->qp, &ic->i_ack_wr, NULL); in rds_ib_send_ack()
684 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_ib_send_ack()
685 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_send_ack()
689 rds_ib_conn_error(ic->conn, "sending ack failed\n"); in rds_ib_send_ack()
732 void rds_ib_attempt_ack(struct rds_ib_connection *ic) in rds_ib_attempt_ack() argument
736 if (!test_bit(IB_ACK_REQUESTED, &ic->i_ack_flags)) in rds_ib_attempt_ack()
739 if (test_and_set_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags)) { in rds_ib_attempt_ack()
745 if (!rds_ib_send_grab_credits(ic, 1, &adv_credits, 0, RDS_MAX_ADV_CREDIT)) { in rds_ib_attempt_ack()
747 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_ib_attempt_ack()
751 clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_attempt_ack()
752 rds_ib_send_ack(ic, adv_credits); in rds_ib_attempt_ack()
759 void rds_ib_ack_send_complete(struct rds_ib_connection *ic) in rds_ib_ack_send_complete() argument
761 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_ib_ack_send_complete()
762 rds_ib_attempt_ack(ic); in rds_ib_ack_send_complete()
769 u64 rds_ib_piggyb_ack(struct rds_ib_connection *ic) in rds_ib_piggyb_ack() argument
771 if (test_and_clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags)) in rds_ib_piggyb_ack()
773 return rds_ib_get_ack(ic); in rds_ib_piggyb_ack()
853 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_process_recv() local
854 struct rds_ib_incoming *ibinc = ic->i_ibinc; in rds_ib_process_recv()
856 dma_addr_t dma_addr = ic->i_recv_hdrs_dma[recv - ic->i_recvs]; in rds_ib_process_recv()
860 rdsdebug("ic %p ibinc %p recv %p byte len %u\n", ic, ibinc, recv, in rds_ib_process_recv()
873 ihdr = ic->i_recv_hdrs[recv - ic->i_recvs]; in rds_ib_process_recv()
875 ib_dma_sync_single_for_cpu(ic->rds_ibdev->dev, dma_addr, in rds_ib_process_recv()
911 rds_ib_frag_free(ic, recv->r_frag); in rds_ib_process_recv()
925 ic->i_ibinc = ibinc; in rds_ib_process_recv()
931 ic->i_recv_data_rem = be32_to_cpu(hdr->h_len); in rds_ib_process_recv()
935 rdsdebug("ic %p ibinc %p rem %u flag 0x%x\n", ic, ibinc, in rds_ib_process_recv()
936 ic->i_recv_data_rem, hdr->h_flags); in rds_ib_process_recv()
954 if (ic->i_recv_data_rem > RDS_FRAG_SIZE) in rds_ib_process_recv()
955 ic->i_recv_data_rem -= RDS_FRAG_SIZE; in rds_ib_process_recv()
957 ic->i_recv_data_rem = 0; in rds_ib_process_recv()
958 ic->i_ibinc = NULL; in rds_ib_process_recv()
980 ib_dma_sync_single_for_device(ic->rds_ibdev->dev, dma_addr, in rds_ib_process_recv()
984 void rds_ib_recv_cqe_handler(struct rds_ib_connection *ic, in rds_ib_recv_cqe_handler() argument
988 struct rds_connection *conn = ic->conn; in rds_ib_recv_cqe_handler()
997 recv = &ic->i_recvs[rds_ib_ring_oldest(&ic->i_recv_ring)]; in rds_ib_recv_cqe_handler()
998 ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1, in rds_ib_recv_cqe_handler()
1025 rds_ib_frag_free(ic, recv->r_frag); in rds_ib_recv_cqe_handler()
1028 rds_ib_ring_free(&ic->i_recv_ring, 1); in rds_ib_recv_cqe_handler()
1033 if (rds_ib_ring_empty(&ic->i_recv_ring)) in rds_ib_recv_cqe_handler()
1036 if (rds_ib_ring_low(&ic->i_recv_ring)) { in rds_ib_recv_cqe_handler()
1045 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_recv_path() local
1049 rds_ib_attempt_ack(ic); in rds_ib_recv_path()