203 static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf,  in get_egrbuf()  argument
208 	*update |= !(idx & (rcd->egrbufs.threshold - 1)) && !offset;  in get_egrbuf()
209 	return (void *)(((u64)(rcd->egrbufs.rcvtids[idx].addr)) +  in get_egrbuf()
213 static inline void *hfi1_get_header(struct hfi1_ctxtdata *rcd,  in hfi1_get_header()  argument
218 	return (void *)(rhf_addr - rcd->rhf_offset + offset);  in hfi1_get_header()
221 static inline struct ib_header *hfi1_get_msgheader(struct hfi1_ctxtdata *rcd,  in hfi1_get_msgheader()  argument
224 	return (struct ib_header *)hfi1_get_header(rcd, rhf_addr);  in hfi1_get_msgheader()
228 		*hfi1_get_16B_header(struct hfi1_ctxtdata *rcd,  in hfi1_get_16B_header()  argument
231 	return (struct hfi1_16b_header *)hfi1_get_header(rcd, rhf_addr);  in hfi1_get_16B_header()
254 static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,  in rcv_hdrerr()  argument
260 	struct hfi1_ibport *ibp = rcd_to_iport(rcd);  in rcv_hdrerr()
337 				hfi1_rc_hdrerr(rcd, packet, qp);  in rcv_hdrerr()
415 static inline void init_packet(struct hfi1_ctxtdata *rcd,  in init_packet()  argument
418 	packet->rsize = get_hdrqentsize(rcd); /* words */  in init_packet()
419 	packet->maxcnt = get_hdrq_cnt(rcd) * packet->rsize; /* words */  in init_packet()
420 	packet->rcd = rcd;  in init_packet()
423 	packet->rhf_addr = get_rhf_addr(rcd);  in init_packet()
425 	packet->rhqoff = hfi1_rcd_head(rcd);  in init_packet()
540 	struct hfi1_ctxtdata *rcd;  member
551 	struct hfi1_ctxtdata *rcd = packet->rcd;  in init_ps_mdata()  local
553 	mdata->rcd = rcd;  in init_ps_mdata()
558 	if (get_dma_rtail_setting(rcd)) {  in init_ps_mdata()
559 		mdata->ps_tail = get_rcvhdrtail(rcd);  in init_ps_mdata()
560 		if (rcd->ctxt == HFI1_CTRL_CTXT)  in init_ps_mdata()
561 			mdata->ps_seq = hfi1_seq_cnt(rcd);  in init_ps_mdata()
566 		mdata->ps_seq = hfi1_seq_cnt(rcd);  in init_ps_mdata()
571 			  struct hfi1_ctxtdata *rcd)  in ps_done()  argument
573 	if (get_dma_rtail_setting(rcd))  in ps_done()
579 			  struct hfi1_ctxtdata *rcd)  in ps_skip()  argument
585 	if ((rcd->ctxt == HFI1_CTRL_CTXT) && (mdata->ps_head != mdata->ps_tail))  in ps_skip()
592 				   struct hfi1_ctxtdata *rcd)  in update_ps_mdata()  argument
599 	if (!get_dma_rtail_setting(rcd) ||  in update_ps_mdata()
600 	    rcd->ctxt == HFI1_CTRL_CTXT)  in update_ps_mdata()
612 #define prescan_rxq(rcd, packet) \  argument
614 		if (rcd->ppd->cc_prescan) \
619 	struct hfi1_ctxtdata *rcd = packet->rcd;  in __prescan_rxq()  local
625 		struct hfi1_ibport *ibp = rcd_to_iport(rcd);  in __prescan_rxq()
626 		__le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +  in __prescan_rxq()
627 					 packet->rcd->rhf_offset;  in __prescan_rxq()
630 		struct rvt_dev_info *rdi = &rcd->dd->verbs_dev.rdi;  in __prescan_rxq()
635 		if (ps_done(&mdata, rhf, rcd))  in __prescan_rxq()
638 		if (ps_skip(&mdata, rhf, rcd))  in __prescan_rxq()
644 		packet->hdr = hfi1_get_msgheader(packet->rcd, rhf_addr);  in __prescan_rxq()
678 		update_ps_mdata(&mdata, rcd);  in __prescan_rxq()
685 	struct hfi1_ctxtdata *rcd = packet->rcd;  in process_rcv_qp_work()  local
691 	list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) {  in process_rcv_qp_work()
721 		this_cpu_inc(*packet->rcd->dd->rcv_limit);  in max_packet_exceeded()
739 	packet->rcd->dd->ctx0_seq_drop++;  in skip_rcv_packet()
748 	packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +  in skip_rcv_packet()
749 				     packet->rcd->rhf_offset;  in skip_rcv_packet()
763 	packet->ebuf = get_egrbuf(packet->rcd, packet->rhf,  in process_rcv_packet_napi()
771 		       packet->tlen - ((packet->rcd->rcvhdrqentsize -  in process_rcv_packet_napi()
775 	packet->rcd->rhf_rcv_function_map[packet->etype](packet);  in process_rcv_packet_napi()
783 	packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +  in process_rcv_packet_napi()
784 				      packet->rcd->rhf_offset;  in process_rcv_packet_napi()
800 		packet->ebuf = get_egrbuf(packet->rcd, packet->rhf,  in process_rcv_packet()
808 			       packet->tlen - ((get_hdrqentsize(packet->rcd) -  in process_rcv_packet()
821 	packet->rcd->rhf_rcv_function_map[packet->etype](packet);  in process_rcv_packet()
831 	packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +  in process_rcv_packet()
832 				      packet->rcd->rhf_offset;  in process_rcv_packet()
847 		update_usrhead(packet->rcd, packet->rhqoff, packet->updegr,  in process_rcv_update()
862 	update_usrhead(packet->rcd, hfi1_rcd_head(packet->rcd), packet->updegr,  in finish_packet()
875 int handle_receive_interrupt_napi_fp(struct hfi1_ctxtdata *rcd, int budget)  in handle_receive_interrupt_napi_fp()  argument
879 	init_packet(rcd, &packet);  in handle_receive_interrupt_napi_fp()
880 	if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))  in handle_receive_interrupt_napi_fp()
885 		if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))  in handle_receive_interrupt_napi_fp()
890 	hfi1_set_rcd_head(rcd, packet.rhqoff);  in handle_receive_interrupt_napi_fp()
899 int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread)  in handle_receive_interrupt_nodma_rtail()  argument
904 	init_packet(rcd, &packet);  in handle_receive_interrupt_nodma_rtail()
905 	if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) {  in handle_receive_interrupt_nodma_rtail()
910 	prescan_rxq(rcd, &packet);  in handle_receive_interrupt_nodma_rtail()
914 		if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))  in handle_receive_interrupt_nodma_rtail()
919 	hfi1_set_rcd_head(rcd, packet.rhqoff);  in handle_receive_interrupt_nodma_rtail()
925 int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread)  in handle_receive_interrupt_dma_rtail()  argument
931 	init_packet(rcd, &packet);  in handle_receive_interrupt_dma_rtail()
932 	hdrqtail = get_rcvhdrtail(rcd);  in handle_receive_interrupt_dma_rtail()
939 	prescan_rxq(rcd, &packet);  in handle_receive_interrupt_dma_rtail()
948 	hfi1_set_rcd_head(rcd, packet.rhqoff);  in handle_receive_interrupt_dma_rtail()
954 static void set_all_fastpath(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)  in set_all_fastpath()  argument
963 	if (rcd->ctxt >= dd->first_dyn_alloc_ctxt && !rcd->is_vnic) {  in set_all_fastpath()
964 		hfi1_rcd_get(rcd);  in set_all_fastpath()
965 		hfi1_set_fast(rcd);  in set_all_fastpath()
966 		hfi1_rcd_put(rcd);  in set_all_fastpath()
971 		rcd = hfi1_rcd_get_by_index(dd, i);  in set_all_fastpath()
972 		if (rcd && (i < dd->first_dyn_alloc_ctxt || rcd->is_vnic))  in set_all_fastpath()
973 			hfi1_set_fast(rcd);  in set_all_fastpath()
974 		hfi1_rcd_put(rcd);  in set_all_fastpath()
980 	struct hfi1_ctxtdata *rcd;  in set_all_slowpath()  local
985 		rcd = hfi1_rcd_get_by_index(dd, i);  in set_all_slowpath()
986 		if (!rcd)  in set_all_slowpath()
988 		if (i < dd->first_dyn_alloc_ctxt || rcd->is_vnic)  in set_all_slowpath()
989 			rcd->do_interrupt = rcd->slow_handler;  in set_all_slowpath()
991 		hfi1_rcd_put(rcd);  in set_all_slowpath()
1001 		struct ib_header *hdr = hfi1_get_msgheader(packet->rcd,  in __set_armed_to_active()
1006 						packet->rcd,  in __set_armed_to_active()
1011 		int hwstate = driver_lstate(packet->rcd->ppd);  in __set_armed_to_active()
1013 				&packet->rcd->ppd->linkstate_active_work;  in __set_armed_to_active()
1016 			dd_dev_info(packet->rcd->dd,  in __set_armed_to_active()
1022 		queue_work(packet->rcd->ppd->link_wq, lsaw);  in __set_armed_to_active()
1036 	if (likely(packet->rcd->ppd->host_link_state != HLS_UP_ARMED))  in set_armed_to_active()
1048 int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)  in handle_receive_interrupt()  argument
1050 	struct hfi1_devdata *dd = rcd->dd;  in handle_receive_interrupt()
1056 	if (!rcd->rcvhdrq)  in handle_receive_interrupt()
1059 	needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1;  in handle_receive_interrupt()
1061 	init_packet(rcd, &packet);  in handle_receive_interrupt()
1063 	if (!get_dma_rtail_setting(rcd)) {  in handle_receive_interrupt()
1064 		if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) {  in handle_receive_interrupt()
1070 		hdrqtail = get_rcvhdrtail(rcd);  in handle_receive_interrupt()
1081 		if (rcd->ctxt == HFI1_CTRL_CTXT)  in handle_receive_interrupt()
1082 			if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))  in handle_receive_interrupt()
1086 	prescan_rxq(rcd, &packet);  in handle_receive_interrupt()
1092 			packet.rhf_addr = (__le32 *)rcd->rcvhdrq +  in handle_receive_interrupt()
1094 					  rcd->rhf_offset;  in handle_receive_interrupt()
1106 		if (!get_dma_rtail_setting(rcd)) {  in handle_receive_interrupt()
1107 			if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))  in handle_receive_interrupt()
1116 			if (rcd->ctxt == HFI1_CTRL_CTXT) {  in handle_receive_interrupt()
1119 				lseq = hfi1_seq_incr(rcd,  in handle_receive_interrupt()
1128 			set_all_fastpath(dd, rcd);  in handle_receive_interrupt()
1134 	hfi1_set_rcd_head(rcd, packet.rhqoff);  in handle_receive_interrupt()
1154 int handle_receive_interrupt_napi_sp(struct hfi1_ctxtdata *rcd, int budget)  in handle_receive_interrupt_napi_sp()  argument
1156 	struct hfi1_devdata *dd = rcd->dd;  in handle_receive_interrupt_napi_sp()
1161 	init_packet(rcd, &packet);  in handle_receive_interrupt_napi_sp()
1162 	if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))  in handle_receive_interrupt_napi_sp()
1169 			packet.rhf_addr = (__le32 *)rcd->rcvhdrq +  in handle_receive_interrupt_napi_sp()
1171 					  rcd->rhf_offset;  in handle_receive_interrupt_napi_sp()
1180 		if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))  in handle_receive_interrupt_napi_sp()
1185 			set_all_fastpath(dd, rcd);  in handle_receive_interrupt_napi_sp()
1191 	hfi1_set_rcd_head(rcd, packet.rhqoff);  in handle_receive_interrupt_napi_sp()
1224 	struct hfi1_ctxtdata *rcd;  in receive_interrupt_work()  local
1236 		rcd = hfi1_rcd_get_by_index(dd, i);  in receive_interrupt_work()
1237 		if (rcd)  in receive_interrupt_work()
1238 			force_recv_intr(rcd);  in receive_interrupt_work()
1239 		hfi1_rcd_put(rcd);  in receive_interrupt_work()
1451 	if (dd->rcd)  in hfi1_reset_device()
1486 			hfi1_get_msgheader(packet->rcd,  in hfi1_setup_ib_header()
1493 	struct hfi1_pportdata *ppd = packet->rcd->ppd;  in hfi1_bypass_ingress_pkt_check()
1522 	struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);  in hfi1_setup_9B_packet()
1581 	struct hfi1_ctxtdata *rcd = packet->rcd;  in hfi1_setup_bypass_packet()  local
1582 	struct hfi1_pportdata *ppd = rcd->ppd;  in hfi1_setup_bypass_packet()
1587 			hfi1_get_16B_header(packet->rcd,  in hfi1_setup_bypass_packet()
1654 	struct hfi1_ctxtdata *rcd = packet->rcd;  in show_eflags_errs()  local
1657 	dd_dev_err(rcd->dd,  in show_eflags_errs()
1659 		   rcd->ctxt, packet->rhf,  in show_eflags_errs()
1672 	struct hfi1_ctxtdata *rcd = packet->rcd;  in handle_eflags()  local
1674 	rcv_hdrerr(rcd, rcd->ppd, packet);  in handle_eflags()
1683 	struct hfi1_ctxtdata *rcd = packet->rcd;  in hfi1_ipoib_ib_rcv()  local
1684 	struct napi_struct *napi = rcd->napi;  in hfi1_ipoib_ib_rcv()
1706 	netdev = hfi1_netdev_get_data(rcd->dd, qpnum);  in hfi1_ipoib_ib_rcv()
1710 	trace_input_ibhdr(rcd->dd, packet, !!(rhf_dc_info(packet->rhf)));  in hfi1_ipoib_ib_rcv()
1711 	trace_ctxt_rsm_hist(rcd->ctxt);  in hfi1_ipoib_ib_rcv()
1751 	ibp = rcd_to_iport(packet->rcd);  in hfi1_ipoib_ib_rcv()
1779 	struct hfi1_devdata *dd = packet->rcd->dd;  in process_receive_bypass()
1817 		 hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) &&  in process_receive_error()
1826 		dd_dev_err(packet->rcd->dd,  in process_receive_error()
1837 		struct hfi1_ctxtdata *rcd = packet->rcd;  in kdeth_process_expected()  local
1839 		if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet))  in kdeth_process_expected()
1854 		struct hfi1_ctxtdata *rcd = packet->rcd;  in kdeth_process_eager()  local
1857 		if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet))  in kdeth_process_eager()
1866 	dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n",  in process_receive_invalid()
1872 void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd)  in seqfile_dump_rcd()  argument
1879 		   rcd->ctxt, get_hdrq_cnt(rcd), get_hdrqentsize(rcd),  in seqfile_dump_rcd()
1880 		   get_dma_rtail_setting(rcd) ?  in seqfile_dump_rcd()
1882 		   read_kctxt_csr(rcd->dd, rcd->ctxt, RCV_CTXT_CTRL),  in seqfile_dump_rcd()
1883 		   read_kctxt_csr(rcd->dd, rcd->ctxt, RCV_CTXT_STATUS),  in seqfile_dump_rcd()
1884 		   read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) &  in seqfile_dump_rcd()
1886 		   read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL),  in seqfile_dump_rcd()
1887 		   rcd->head);  in seqfile_dump_rcd()
1889 	init_packet(rcd, &packet);  in seqfile_dump_rcd()
1893 		__le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +  in seqfile_dump_rcd()
1894 					 rcd->rhf_offset;  in seqfile_dump_rcd()
1902 		if (ps_done(&mdata, rhf, rcd))  in seqfile_dump_rcd()
1905 		if (ps_skip(&mdata, rhf, rcd))  in seqfile_dump_rcd()
1911 		packet.hdr = hfi1_get_msgheader(rcd, rhf_addr);  in seqfile_dump_rcd()
1930 		update_ps_mdata(&mdata, rcd);  in seqfile_dump_rcd()