275 	wqe->sqe.flags = 0;  in siw_qp_mpa_rts()
276 	wqe->sqe.num_sge = 1;  in siw_qp_mpa_rts()
277 	wqe->sqe.sge[0].length = 0;  in siw_qp_mpa_rts()
278 	wqe->sqe.sge[0].laddr = 0;  in siw_qp_mpa_rts()
279 	wqe->sqe.sge[0].lkey = 0;  in siw_qp_mpa_rts()
284 	wqe->sqe.rkey = 1;  in siw_qp_mpa_rts()
285 	wqe->sqe.raddr = 0;  in siw_qp_mpa_rts()
289 		wqe->sqe.opcode = SIW_OP_WRITE;  in siw_qp_mpa_rts()
293 		wqe->sqe.opcode = SIW_OP_READ;  in siw_qp_mpa_rts()
299 			siw_read_to_orq(rreq, &wqe->sqe);  in siw_qp_mpa_rts()
479 				rreq->ddp_msn = htonl(wqe->sqe.sge[0].length);  in siw_send_terminate()
482 				rreq->sink_stag = htonl(wqe->sqe.rkey);  in siw_send_terminate()
483 				rreq->sink_to = cpu_to_be64(wqe->sqe.raddr);  in siw_send_terminate()
484 				rreq->read_size = htonl(wqe->sqe.sge[0].length);  in siw_send_terminate()
485 				rreq->source_stag = htonl(wqe->sqe.sge[0].lkey);  in siw_send_terminate()
487 					cpu_to_be64(wqe->sqe.sge[0].laddr);  in siw_send_terminate()
868 void siw_read_to_orq(struct siw_sqe *rreq, struct siw_sqe *sqe)  in siw_read_to_orq()  argument
870 	rreq->id = sqe->id;  in siw_read_to_orq()
871 	rreq->opcode = sqe->opcode;  in siw_read_to_orq()
872 	rreq->sge[0].laddr = sqe->sge[0].laddr;  in siw_read_to_orq()
873 	rreq->sge[0].length = sqe->sge[0].length;  in siw_read_to_orq()
874 	rreq->sge[0].lkey = sqe->sge[0].lkey;  in siw_read_to_orq()
875 	rreq->sge[1].lkey = sqe->sge[1].lkey;  in siw_read_to_orq()
876 	rreq->flags = sqe->flags | SIW_WQE_VALID;  in siw_read_to_orq()
888 	struct siw_sqe *irqe, *sqe;  in siw_activate_tx()  local
895 		sqe = sq_get_next(qp);  in siw_activate_tx()
901 		if (sqe && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) {  in siw_activate_tx()
909 		wqe->sqe.opcode = SIW_OP_READ_RESPONSE;  in siw_activate_tx()
910 		wqe->sqe.flags = 0;  in siw_activate_tx()
912 			wqe->sqe.num_sge = 1;  in siw_activate_tx()
913 			wqe->sqe.sge[0].length = irqe->sge[0].length;  in siw_activate_tx()
914 			wqe->sqe.sge[0].laddr = irqe->sge[0].laddr;  in siw_activate_tx()
915 			wqe->sqe.sge[0].lkey = irqe->sge[0].lkey;  in siw_activate_tx()
917 			wqe->sqe.num_sge = 0;  in siw_activate_tx()
923 		wqe->sqe.sge[1].length = irqe->sge[1].length;  in siw_activate_tx()
925 		wqe->sqe.rkey = irqe->rkey;  in siw_activate_tx()
926 		wqe->sqe.raddr = irqe->raddr;  in siw_activate_tx()
936 	sqe = sq_get_next(qp);  in siw_activate_tx()
937 	if (sqe) {  in siw_activate_tx()
943 		memcpy(&wqe->sqe, sqe, sizeof(*sqe));  in siw_activate_tx()
945 		if (wqe->sqe.opcode >= SIW_NUM_OPCODES) {  in siw_activate_tx()
949 		if (wqe->sqe.flags & SIW_WQE_INLINE) {  in siw_activate_tx()
950 			if (wqe->sqe.opcode != SIW_OP_SEND &&  in siw_activate_tx()
951 			    wqe->sqe.opcode != SIW_OP_WRITE) {  in siw_activate_tx()
955 			if (wqe->sqe.sge[0].length > SIW_MAX_INLINE) {  in siw_activate_tx()
959 			wqe->sqe.sge[0].laddr = (uintptr_t)&wqe->sqe.sge[1];  in siw_activate_tx()
960 			wqe->sqe.sge[0].lkey = 0;  in siw_activate_tx()
961 			wqe->sqe.num_sge = 1;  in siw_activate_tx()
963 		if (wqe->sqe.flags & SIW_WQE_READ_FENCE) {  in siw_activate_tx()
965 			if (unlikely(wqe->sqe.opcode == SIW_OP_READ ||  in siw_activate_tx()
966 				     wqe->sqe.opcode ==  in siw_activate_tx()
980 		} else if (wqe->sqe.opcode == SIW_OP_READ ||  in siw_activate_tx()
981 			   wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV) {  in siw_activate_tx()
984 			wqe->sqe.num_sge = 1;  in siw_activate_tx()
994 				siw_read_to_orq(rreq, &wqe->sqe);  in siw_activate_tx()
1004 		smp_store_mb(sqe->flags, 0);  in siw_activate_tx()
1047 int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes,  in siw_sqe_complete()  argument
1054 		u32 sqe_flags = sqe->flags;  in siw_sqe_complete()
1067 			cqe->id = sqe->id;  in siw_sqe_complete()
1068 			cqe->opcode = sqe->opcode;  in siw_sqe_complete()
1081 			smp_store_mb(sqe->flags, 0);  in siw_sqe_complete()
1100 		smp_store_mb(sqe->flags, 0);  in siw_sqe_complete()
1177 	struct siw_sqe *sqe;  in siw_sq_flush()  local
1185 		sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size];  in siw_sq_flush()
1186 		if (!READ_ONCE(sqe->flags))  in siw_sq_flush()
1189 		if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)  in siw_sq_flush()
1192 		WRITE_ONCE(sqe->flags, 0);  in siw_sq_flush()
1212 			siw_sqe_complete(qp, &wqe->sqe, wqe->bytes,  in siw_sq_flush()
1221 		sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size];  in siw_sq_flush()
1222 		if (!READ_ONCE(sqe->flags))  in siw_sq_flush()
1226 		if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)  in siw_sq_flush()
1233 		WRITE_ONCE(sqe->flags, 0);  in siw_sq_flush()
1270 			siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR);  in siw_rq_flush()