244 	struct spi_controller *ctlr = spi->controller;  in rockchip_spi_set_cs()  local
245 	struct rockchip_spi *rs = spi_controller_get_devdata(ctlr);  in rockchip_spi_set_cs()
273 static void rockchip_spi_handle_err(struct spi_controller *ctlr,  in rockchip_spi_handle_err()  argument
276 	struct rockchip_spi *rs = spi_controller_get_devdata(ctlr);  in rockchip_spi_handle_err()
287 		dmaengine_terminate_async(ctlr->dma_tx);  in rockchip_spi_handle_err()
290 		dmaengine_terminate_async(ctlr->dma_rx);  in rockchip_spi_handle_err()
348 	struct spi_controller *ctlr = dev_id;  in rockchip_spi_isr()  local
349 	struct rockchip_spi *rs = spi_controller_get_devdata(ctlr);  in rockchip_spi_isr()
353 		ctlr->slave_abort(ctlr);  in rockchip_spi_isr()
368 		spi_finalize_current_transfer(ctlr);  in rockchip_spi_isr()
375 				    struct spi_controller *ctlr,  in rockchip_spi_prepare_irq()  argument
398 	struct spi_controller *ctlr = data;  in rockchip_spi_dma_rxcb()  local
399 	struct rockchip_spi *rs = spi_controller_get_devdata(ctlr);  in rockchip_spi_dma_rxcb()
409 	spi_finalize_current_transfer(ctlr);  in rockchip_spi_dma_rxcb()
414 	struct spi_controller *ctlr = data;  in rockchip_spi_dma_txcb()  local
415 	struct rockchip_spi *rs = spi_controller_get_devdata(ctlr);  in rockchip_spi_dma_txcb()
422 	wait_for_tx_idle(rs, ctlr->slave);  in rockchip_spi_dma_txcb()
425 	spi_finalize_current_transfer(ctlr);  in rockchip_spi_dma_txcb()
442 		struct spi_controller *ctlr, struct spi_transfer *xfer)  in rockchip_spi_prepare_dma()  argument
460 		dmaengine_slave_config(ctlr->dma_rx, &rxconf);  in rockchip_spi_prepare_dma()
463 				ctlr->dma_rx,  in rockchip_spi_prepare_dma()
470 		rxdesc->callback_param = ctlr;  in rockchip_spi_prepare_dma()
482 		dmaengine_slave_config(ctlr->dma_tx, &txconf);  in rockchip_spi_prepare_dma()
485 				ctlr->dma_tx,  in rockchip_spi_prepare_dma()
490 				dmaengine_terminate_sync(ctlr->dma_rx);  in rockchip_spi_prepare_dma()
495 		txdesc->callback_param = ctlr;  in rockchip_spi_prepare_dma()
501 		ctlr->dma_rx->cookie = dmaengine_submit(rxdesc);  in rockchip_spi_prepare_dma()
502 		dma_async_issue_pending(ctlr->dma_rx);  in rockchip_spi_prepare_dma()
513 		dma_async_issue_pending(ctlr->dma_tx);  in rockchip_spi_prepare_dma()
611 static int rockchip_spi_slave_abort(struct spi_controller *ctlr)  in rockchip_spi_slave_abort()  argument
613 	struct rockchip_spi *rs = spi_controller_get_devdata(ctlr);  in rockchip_spi_slave_abort()
620 		dmaengine_pause(ctlr->dma_rx);  in rockchip_spi_slave_abort()
621 		status = dmaengine_tx_status(ctlr->dma_rx, ctlr->dma_rx->cookie, &state);  in rockchip_spi_slave_abort()
651 		dmaengine_terminate_sync(ctlr->dma_rx);  in rockchip_spi_slave_abort()
653 		dmaengine_terminate_sync(ctlr->dma_tx);  in rockchip_spi_slave_abort()
657 	spi_finalize_current_transfer(ctlr);  in rockchip_spi_slave_abort()
663 		struct spi_controller *ctlr,  in rockchip_spi_transfer_one()  argument
667 	struct rockchip_spi *rs = spi_controller_get_devdata(ctlr);  in rockchip_spi_transfer_one()
673 		spi_finalize_current_transfer(ctlr);  in rockchip_spi_transfer_one()
692 	use_dma = ctlr->can_dma ? ctlr->can_dma(ctlr, spi, xfer) : false;  in rockchip_spi_transfer_one()
694 	ret = rockchip_spi_config(rs, spi, xfer, use_dma, ctlr->slave);  in rockchip_spi_transfer_one()
699 		return rockchip_spi_prepare_dma(rs, ctlr, xfer);  in rockchip_spi_transfer_one()
701 	return rockchip_spi_prepare_irq(rs, ctlr, xfer);  in rockchip_spi_transfer_one()
704 static bool rockchip_spi_can_dma(struct spi_controller *ctlr,  in rockchip_spi_can_dma()  argument
708 	struct rockchip_spi *rs = spi_controller_get_devdata(ctlr);  in rockchip_spi_can_dma()
750 	struct spi_controller *ctlr;  in rockchip_spi_probe()  local
759 		ctlr = spi_alloc_slave(&pdev->dev,  in rockchip_spi_probe()
762 		ctlr = spi_alloc_master(&pdev->dev,  in rockchip_spi_probe()
765 	if (!ctlr)  in rockchip_spi_probe()
768 	platform_set_drvdata(pdev, ctlr);  in rockchip_spi_probe()
770 	rs = spi_controller_get_devdata(ctlr);  in rockchip_spi_probe()
771 	ctlr->slave = slave_mode;  in rockchip_spi_probe()
814 			IRQF_ONESHOT, dev_name(&pdev->dev), ctlr);  in rockchip_spi_probe()
850 	ctlr->auto_runtime_pm = true;  in rockchip_spi_probe()
851 	ctlr->bus_num = pdev->id;  in rockchip_spi_probe()
852 	ctlr->mode_bits = SPI_CPOL | SPI_CPHA | SPI_LOOP | SPI_LSB_FIRST;  in rockchip_spi_probe()
854 		ctlr->mode_bits |= SPI_NO_CS;  in rockchip_spi_probe()
855 		ctlr->slave_abort = rockchip_spi_slave_abort;  in rockchip_spi_probe()
857 		ctlr->flags = SPI_MASTER_GPIO_SS;  in rockchip_spi_probe()
858 		ctlr->max_native_cs = ROCKCHIP_SPI_MAX_CS_NUM;  in rockchip_spi_probe()
865 		ctlr->num_chipselect = num_cs;  in rockchip_spi_probe()
866 		ctlr->use_gpio_descriptors = true;  in rockchip_spi_probe()
868 	ctlr->dev.of_node = pdev->dev.of_node;  in rockchip_spi_probe()
869 	ctlr->bits_per_word_mask = SPI_BPW_MASK(16) | SPI_BPW_MASK(8) | SPI_BPW_MASK(4);  in rockchip_spi_probe()
870 	ctlr->min_speed_hz = rs->freq / BAUDR_SCKDV_MAX;  in rockchip_spi_probe()
871 	ctlr->max_speed_hz = min(rs->freq / BAUDR_SCKDV_MIN, MAX_SCLK_OUT);  in rockchip_spi_probe()
873 	ctlr->setup = rockchip_spi_setup;  in rockchip_spi_probe()
874 	ctlr->set_cs = rockchip_spi_set_cs;  in rockchip_spi_probe()
875 	ctlr->transfer_one = rockchip_spi_transfer_one;  in rockchip_spi_probe()
876 	ctlr->max_transfer_size = rockchip_spi_max_transfer_size;  in rockchip_spi_probe()
877 	ctlr->handle_err = rockchip_spi_handle_err;  in rockchip_spi_probe()
879 	ctlr->dma_tx = dma_request_chan(rs->dev, "tx");  in rockchip_spi_probe()
880 	if (IS_ERR(ctlr->dma_tx)) {  in rockchip_spi_probe()
882 		if (PTR_ERR(ctlr->dma_tx) == -EPROBE_DEFER) {  in rockchip_spi_probe()
887 		ctlr->dma_tx = NULL;  in rockchip_spi_probe()
890 	ctlr->dma_rx = dma_request_chan(rs->dev, "rx");  in rockchip_spi_probe()
891 	if (IS_ERR(ctlr->dma_rx)) {  in rockchip_spi_probe()
892 		if (PTR_ERR(ctlr->dma_rx) == -EPROBE_DEFER) {  in rockchip_spi_probe()
897 		ctlr->dma_rx = NULL;  in rockchip_spi_probe()
900 	if (ctlr->dma_tx && ctlr->dma_rx) {  in rockchip_spi_probe()
903 		ctlr->can_dma = rockchip_spi_can_dma;  in rockchip_spi_probe()
909 		ctlr->mode_bits |= SPI_CS_HIGH;  in rockchip_spi_probe()
910 		if (ctlr->can_dma && slave_mode)  in rockchip_spi_probe()
920 	ret = devm_spi_register_controller(&pdev->dev, ctlr);  in rockchip_spi_probe()
929 	if (ctlr->dma_rx)  in rockchip_spi_probe()
930 		dma_release_channel(ctlr->dma_rx);  in rockchip_spi_probe()
932 	if (ctlr->dma_tx)  in rockchip_spi_probe()
933 		dma_release_channel(ctlr->dma_tx);  in rockchip_spi_probe()
941 	spi_controller_put(ctlr);  in rockchip_spi_probe()
948 	struct spi_controller *ctlr = spi_controller_get(platform_get_drvdata(pdev));  in rockchip_spi_remove()  local
949 	struct rockchip_spi *rs = spi_controller_get_devdata(ctlr);  in rockchip_spi_remove()
960 	if (ctlr->dma_tx)  in rockchip_spi_remove()
961 		dma_release_channel(ctlr->dma_tx);  in rockchip_spi_remove()
962 	if (ctlr->dma_rx)  in rockchip_spi_remove()
963 		dma_release_channel(ctlr->dma_rx);  in rockchip_spi_remove()
965 	spi_controller_put(ctlr);  in rockchip_spi_remove()
974 	struct spi_controller *ctlr = dev_get_drvdata(dev);  in rockchip_spi_suspend()  local
976 	ret = spi_controller_suspend(ctlr);  in rockchip_spi_suspend()
992 	struct spi_controller *ctlr = dev_get_drvdata(dev);  in rockchip_spi_resume()  local
993 	struct rockchip_spi *rs = spi_controller_get_devdata(ctlr);  in rockchip_spi_resume()
1001 	ret = spi_controller_resume(ctlr);  in rockchip_spi_resume()
1014 	struct spi_controller *ctlr = dev_get_drvdata(dev);  in rockchip_spi_runtime_suspend()  local
1015 	struct rockchip_spi *rs = spi_controller_get_devdata(ctlr);  in rockchip_spi_runtime_suspend()
1026 	struct spi_controller *ctlr = dev_get_drvdata(dev);  in rockchip_spi_runtime_resume()  local
1027 	struct rockchip_spi *rs = spi_controller_get_devdata(ctlr);  in rockchip_spi_runtime_resume()