73 static dma_addr_t read_txq_cur_addr(void __iomem *mbase, u8 epnum)  in read_txq_cur_addr()  argument
78 	txcpr = mtu3_readl(mbase, USB_QMU_TQCPR(epnum));  in read_txq_cur_addr()
79 	txhiar = mtu3_readl(mbase, USB_QMU_TQHIAR(epnum));  in read_txq_cur_addr()
84 static dma_addr_t read_rxq_cur_addr(void __iomem *mbase, u8 epnum)  in read_rxq_cur_addr()  argument
89 	rxcpr = mtu3_readl(mbase, USB_QMU_RQCPR(epnum));  in read_rxq_cur_addr()
90 	rxhiar = mtu3_readl(mbase, USB_QMU_RQHIAR(epnum));  in read_rxq_cur_addr()
95 static void write_txq_start_addr(void __iomem *mbase, u8 epnum, dma_addr_t dma)  in write_txq_start_addr()  argument
99 	mtu3_writel(mbase, USB_QMU_TQSAR(epnum),  in write_txq_start_addr()
101 	tqhiar = mtu3_readl(mbase, USB_QMU_TQHIAR(epnum));  in write_txq_start_addr()
104 	mtu3_writel(mbase, USB_QMU_TQHIAR(epnum), tqhiar);  in write_txq_start_addr()
107 static void write_rxq_start_addr(void __iomem *mbase, u8 epnum, dma_addr_t dma)  in write_rxq_start_addr()  argument
111 	mtu3_writel(mbase, USB_QMU_RQSAR(epnum),  in write_rxq_start_addr()
113 	rqhiar = mtu3_readl(mbase, USB_QMU_RQHIAR(epnum));  in write_rxq_start_addr()
116 	mtu3_writel(mbase, USB_QMU_RQHIAR(epnum), rqhiar);  in write_rxq_start_addr()
193 	int epnum = mep->epnum;  in mtu3_qmu_resume()  local
196 	offset = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);  in mtu3_qmu_resume()
263 		mep->epnum, gpd, enq, &enq_dma);  in mtu3_prepare_tx_gpd()
306 		mep->epnum, gpd, enq, &enq_dma);  in mtu3_prepare_rx_gpd()
336 	u8 epnum = mep->epnum;  in mtu3_qmu_start()  local
340 		write_txq_start_addr(mbase, epnum, ring->dma);  in mtu3_qmu_start()
341 		mtu3_setbits(mbase, MU3D_EP_TXCR0(epnum), TX_DMAREQEN);  in mtu3_qmu_start()
343 		mtu3_setbits(mbase, U3D_QCR1, QMU_TX_ZLP(epnum));  in mtu3_qmu_start()
345 				QMU_TX_LEN_ERR(epnum) | QMU_TX_CS_ERR(epnum));  in mtu3_qmu_start()
347 		if (mtu3_readl(mbase, USB_QMU_TQCSR(epnum)) & QMU_Q_ACTIVE) {  in mtu3_qmu_start()
348 			dev_warn(mtu->dev, "Tx %d Active Now!\n", epnum);  in mtu3_qmu_start()
351 		mtu3_writel(mbase, USB_QMU_TQCSR(epnum), QMU_Q_START);  in mtu3_qmu_start()
354 		write_rxq_start_addr(mbase, epnum, ring->dma);  in mtu3_qmu_start()
355 		mtu3_setbits(mbase, MU3D_EP_RXCR0(epnum), RX_DMAREQEN);  in mtu3_qmu_start()
357 		mtu3_clrbits(mbase, U3D_QCR3, QMU_RX_ZLP(epnum));  in mtu3_qmu_start()
359 		mtu3_setbits(mbase, U3D_QCR3, QMU_RX_COZ(epnum));  in mtu3_qmu_start()
361 				QMU_RX_LEN_ERR(epnum) | QMU_RX_CS_ERR(epnum));  in mtu3_qmu_start()
362 		mtu3_writel(mbase, U3D_RQERRIESR1, QMU_RX_ZLP_ERR(epnum));  in mtu3_qmu_start()
364 		if (mtu3_readl(mbase, USB_QMU_RQCSR(epnum)) & QMU_Q_ACTIVE) {  in mtu3_qmu_start()
365 			dev_warn(mtu->dev, "Rx %d Active Now!\n", epnum);  in mtu3_qmu_start()
368 		mtu3_writel(mbase, USB_QMU_RQCSR(epnum), QMU_Q_START);  in mtu3_qmu_start()
379 	int epnum = mep->epnum;  in mtu3_qmu_stop()  local
384 	qcsr = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);  in mtu3_qmu_stop()
418 static void qmu_tx_zlp_error_handler(struct mtu3 *mtu, u8 epnum)  in qmu_tx_zlp_error_handler()  argument
420 	struct mtu3_ep *mep = mtu->in_eps + epnum;  in qmu_tx_zlp_error_handler()
433 	cur_gpd_dma = read_txq_cur_addr(mbase, epnum);  in qmu_tx_zlp_error_handler()
437 		dev_err(mtu->dev, "TX EP%d buffer length error(!=0)\n", epnum);  in qmu_tx_zlp_error_handler()
444 	mtu3_clrbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);  in qmu_tx_zlp_error_handler()
446 	ret = readl_poll_timeout_atomic(mbase + MU3D_EP_TXCR0(mep->epnum),  in qmu_tx_zlp_error_handler()
452 	mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_TXPKTRDY);  in qmu_tx_zlp_error_handler()
459 	mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);  in qmu_tx_zlp_error_handler()
470 static void qmu_done_tx(struct mtu3 *mtu, u8 epnum)  in qmu_done_tx()  argument
472 	struct mtu3_ep *mep = mtu->in_eps + epnum;  in qmu_done_tx()
482 	cur_gpd_dma = read_txq_cur_addr(mbase, epnum);  in qmu_done_tx()
486 		__func__, epnum, gpd, gpd_current, ring->enqueue);  in qmu_done_tx()
506 		__func__, epnum, ring->dequeue, ring->enqueue);  in qmu_done_tx()
510 static void qmu_done_rx(struct mtu3 *mtu, u8 epnum)  in qmu_done_rx()  argument
512 	struct mtu3_ep *mep = mtu->out_eps + epnum;  in qmu_done_rx()
521 	cur_gpd_dma = read_rxq_cur_addr(mbase, epnum);  in qmu_done_rx()
525 		__func__, epnum, gpd, gpd_current, ring->enqueue);  in qmu_done_rx()
545 		__func__, epnum, ring->dequeue, ring->enqueue);  in qmu_done_rx()