1 // SPDX-License-Identifier: GPL-2.0-or-later
3  *  linux/drivers/mmc/host/sdhci.c - Secure Digital Host Controller Interface driver
5  *  Copyright (C) 2005-2008 Pierre Ossman, All Rights Reserved.
9  *     - JMicron (hardware and technical support)
19 #include <linux/dma-mapping.h>
34 #include <linux/mmc/slot-gpio.h>
41 	pr_debug("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
44 	pr_err("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
53 static bool sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd);
74 	SDHCI_DUMP("Wake-up:   0x%08x | Clock:    0x%08x\n",  in sdhci_dumpregs()
89 	SDHCI_DUMP("Cmd:       0x%08x | Max curr: 0x%08x\n",  in sdhci_dumpregs()
101 	if (host->flags & SDHCI_USE_ADMA) {  in sdhci_dumpregs()
102 		if (host->flags & SDHCI_USE_64_BIT_DMA) {  in sdhci_dumpregs()
114 	if (host->ops->dump_vendor_regs)  in sdhci_dumpregs()
115 		host->ops->dump_vendor_regs(host);  in sdhci_dumpregs()
145 	host->v4_mode = true;  in sdhci_enable_v4_mode()
150 static inline bool sdhci_data_line_cmd(struct mmc_command *cmd)  in sdhci_data_line_cmd()  argument
152 	return cmd->data || cmd->flags & MMC_RSP_BUSY;  in sdhci_data_line_cmd()
159 	if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) ||  in sdhci_set_card_detection()
160 	    !mmc_card_is_removable(host->mmc) || mmc_can_gpio_cd(host->mmc))  in sdhci_set_card_detection()
167 		host->ier |= present ? SDHCI_INT_CARD_REMOVE :  in sdhci_set_card_detection()
170 		host->ier &= ~(SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT);  in sdhci_set_card_detection()
173 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);  in sdhci_set_card_detection()
174 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);  in sdhci_set_card_detection()
189 	if (host->bus_on)  in sdhci_runtime_pm_bus_on()
191 	host->bus_on = true;  in sdhci_runtime_pm_bus_on()
192 	pm_runtime_get_noresume(host->mmc->parent);  in sdhci_runtime_pm_bus_on()
197 	if (!host->bus_on)  in sdhci_runtime_pm_bus_off()
199 	host->bus_on = false;  in sdhci_runtime_pm_bus_off()
200 	pm_runtime_put_noidle(host->mmc->parent);  in sdhci_runtime_pm_bus_off()
210 		host->clock = 0;  in sdhci_reset()
211 		/* Reset-all turns off SD Bus Power */  in sdhci_reset()
212 		if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)  in sdhci_reset()
227 				mmc_hostname(host->mmc), (int)mask);  in sdhci_reset()
238 	if (host->quirks & SDHCI_QUIRK_NO_CARD_NO_RESET) {  in sdhci_do_reset()
239 		struct mmc_host *mmc = host->mmc;  in sdhci_do_reset()
241 		if (!mmc->ops->get_cd(mmc))  in sdhci_do_reset()
245 	host->ops->reset(host, mask);  in sdhci_do_reset()
248 		if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {  in sdhci_do_reset()
249 			if (host->ops->enable_dma)  in sdhci_do_reset()
250 				host->ops->enable_dma(host);  in sdhci_do_reset()
254 		host->preset_enabled = false;  in sdhci_do_reset()
260 	host->ier = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT |  in sdhci_set_default_irqs()
266 	if (host->tuning_mode == SDHCI_TUNING_MODE_2 ||  in sdhci_set_default_irqs()
267 	    host->tuning_mode == SDHCI_TUNING_MODE_3)  in sdhci_set_default_irqs()
268 		host->ier |= SDHCI_INT_RETUNE;  in sdhci_set_default_irqs()
270 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);  in sdhci_set_default_irqs()
271 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);  in sdhci_set_default_irqs()
279 	if (host->version < SDHCI_SPEC_200)  in sdhci_config_dma()
290 	if (!(host->flags & SDHCI_REQ_USE_DMA))  in sdhci_config_dma()
294 	if (host->flags & SDHCI_USE_ADMA)  in sdhci_config_dma()
297 	if (host->flags & SDHCI_USE_64_BIT_DMA) {  in sdhci_config_dma()
299 		 * If v4 mode, all supported DMA can be 64-bit addressing if  in sdhci_config_dma()
300 		 * controller supports 64-bit system address, otherwise only  in sdhci_config_dma()
301 		 * ADMA can support 64-bit addressing.  in sdhci_config_dma()
303 		if (host->v4_mode) {  in sdhci_config_dma()
307 		} else if (host->flags & SDHCI_USE_ADMA) {  in sdhci_config_dma()
322 	struct mmc_host *mmc = host->mmc;  in sdhci_init()
330 	if (host->v4_mode)  in sdhci_init()
333 	spin_lock_irqsave(&host->lock, flags);  in sdhci_init()
335 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_init()
337 	host->cqe_on = false;  in sdhci_init()
341 		host->clock = 0;  in sdhci_init()
342 		mmc->ops->set_ios(mmc, &mmc->ios);  in sdhci_init()
348 	u32 cd = host->ier & (SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT);  in sdhci_reinit()
359 	if (cd != (host->ier & (SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT)))  in sdhci_reinit()
360 		mmc_detect_change(host->mmc, msecs_to_jiffies(200));  in sdhci_reinit()
367 	if (host->quirks & SDHCI_QUIRK_NO_LED)  in __sdhci_led_activate()
379 	if (host->quirks & SDHCI_QUIRK_NO_LED)  in __sdhci_led_deactivate()
394 	spin_lock_irqsave(&host->lock, flags);  in sdhci_led_control()
396 	if (host->runtime_suspended)  in sdhci_led_control()
404 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_led_control()
409 	struct mmc_host *mmc = host->mmc;  in sdhci_led_register()
411 	if (host->quirks & SDHCI_QUIRK_NO_LED)  in sdhci_led_register()
414 	snprintf(host->led_name, sizeof(host->led_name),  in sdhci_led_register()
417 	host->led.name = host->led_name;  in sdhci_led_register()
418 	host->led.brightness = LED_OFF;  in sdhci_led_register()
419 	host->led.default_trigger = mmc_hostname(mmc);  in sdhci_led_register()
420 	host->led.brightness_set = sdhci_led_control;  in sdhci_led_register()
422 	return led_classdev_register(mmc_dev(mmc), &host->led);  in sdhci_led_register()
427 	if (host->quirks & SDHCI_QUIRK_NO_LED)  in sdhci_led_unregister()
430 	led_classdev_unregister(&host->led);  in sdhci_led_unregister()
467 	if (sdhci_data_line_cmd(mrq->cmd))  in sdhci_mod_timer()
468 		mod_timer(&host->data_timer, timeout);  in sdhci_mod_timer()
470 		mod_timer(&host->timer, timeout);  in sdhci_mod_timer()
475 	if (sdhci_data_line_cmd(mrq->cmd))  in sdhci_del_timer()
476 		del_timer(&host->data_timer);  in sdhci_del_timer()
478 		del_timer(&host->timer);  in sdhci_del_timer()
483 	return host->cmd || host->data_cmd;  in sdhci_has_requests()
501 	blksize = host->data->blksz;  in sdhci_read_block_pio()
507 		BUG_ON(!sg_miter_next(&host->sg_miter));  in sdhci_read_block_pio()
509 		len = min(host->sg_miter.length, blksize);  in sdhci_read_block_pio()
511 		blksize -= len;  in sdhci_read_block_pio()
512 		host->sg_miter.consumed = len;  in sdhci_read_block_pio()
514 		buf = host->sg_miter.addr;  in sdhci_read_block_pio()
526 			chunk--;  in sdhci_read_block_pio()
527 			len--;  in sdhci_read_block_pio()
531 	sg_miter_stop(&host->sg_miter);  in sdhci_read_block_pio()
545 	blksize = host->data->blksz;  in sdhci_write_block_pio()
552 		BUG_ON(!sg_miter_next(&host->sg_miter));  in sdhci_write_block_pio()
554 		len = min(host->sg_miter.length, blksize);  in sdhci_write_block_pio()
556 		blksize -= len;  in sdhci_write_block_pio()
557 		host->sg_miter.consumed = len;  in sdhci_write_block_pio()
559 		buf = host->sg_miter.addr;  in sdhci_write_block_pio()
566 			len--;  in sdhci_write_block_pio()
576 	sg_miter_stop(&host->sg_miter);  in sdhci_write_block_pio()
585 	if (host->blocks == 0)  in sdhci_transfer_pio()
588 	if (host->data->flags & MMC_DATA_READ)  in sdhci_transfer_pio()
598 	if ((host->quirks & SDHCI_QUIRK_BROKEN_SMALL_PIO) &&  in sdhci_transfer_pio()
599 		(host->data->blocks == 1))  in sdhci_transfer_pio()
603 		if (host->quirks & SDHCI_QUIRK_PIO_NEEDS_DELAY)  in sdhci_transfer_pio()
606 		if (host->data->flags & MMC_DATA_READ)  in sdhci_transfer_pio()
611 		host->blocks--;  in sdhci_transfer_pio()
612 		if (host->blocks == 0)  in sdhci_transfer_pio()
628 	if (data->host_cookie == COOKIE_PRE_MAPPED)  in sdhci_pre_dma_transfer()
629 		return data->sg_count;  in sdhci_pre_dma_transfer()
632 	if (host->bounce_buffer) {  in sdhci_pre_dma_transfer()
633 		unsigned int length = data->blksz * data->blocks;  in sdhci_pre_dma_transfer()
635 		if (length > host->bounce_buffer_size) {  in sdhci_pre_dma_transfer()
637 			       mmc_hostname(host->mmc), length,  in sdhci_pre_dma_transfer()
638 			       host->bounce_buffer_size);  in sdhci_pre_dma_transfer()
639 			return -EIO;  in sdhci_pre_dma_transfer()
643 			if (host->ops->copy_to_bounce_buffer) {  in sdhci_pre_dma_transfer()
644 				host->ops->copy_to_bounce_buffer(host,  in sdhci_pre_dma_transfer()
647 				sg_copy_to_buffer(data->sg, data->sg_len,  in sdhci_pre_dma_transfer()
648 						  host->bounce_buffer, length);  in sdhci_pre_dma_transfer()
652 		dma_sync_single_for_device(host->mmc->parent,  in sdhci_pre_dma_transfer()
653 					   host->bounce_addr,  in sdhci_pre_dma_transfer()
654 					   host->bounce_buffer_size,  in sdhci_pre_dma_transfer()
660 		sg_count = dma_map_sg(mmc_dev(host->mmc),  in sdhci_pre_dma_transfer()
661 				      data->sg, data->sg_len,  in sdhci_pre_dma_transfer()
666 		return -ENOSPC;  in sdhci_pre_dma_transfer()
668 	data->sg_count = sg_count;  in sdhci_pre_dma_transfer()
669 	data->host_cookie = cookie;  in sdhci_pre_dma_transfer()
677 	return kmap_atomic(sg_page(sg)) + sg->offset;  in sdhci_kmap_atomic()
687 			   dma_addr_t addr, int len, unsigned int cmd)  in sdhci_adma_write_desc()  argument
691 	/* 32-bit and 64-bit descriptors have these members in same position */  in sdhci_adma_write_desc()
692 	dma_desc->cmd = cpu_to_le16(cmd);  in sdhci_adma_write_desc()
693 	dma_desc->len = cpu_to_le16(len);  in sdhci_adma_write_desc()
694 	dma_desc->addr_lo = cpu_to_le32(lower_32_bits(addr));  in sdhci_adma_write_desc()
696 	if (host->flags & SDHCI_USE_64_BIT_DMA)  in sdhci_adma_write_desc()
697 		dma_desc->addr_hi = cpu_to_le32(upper_32_bits(addr));  in sdhci_adma_write_desc()
699 	*desc += host->desc_sz;  in sdhci_adma_write_desc()
705 					   int len, unsigned int cmd)  in __sdhci_adma_write_desc()  argument
707 	if (host->ops->adma_write_desc)  in __sdhci_adma_write_desc()
708 		host->ops->adma_write_desc(host, desc, addr, len, cmd);  in __sdhci_adma_write_desc()
710 		sdhci_adma_write_desc(host, desc, addr, len, cmd);  in __sdhci_adma_write_desc()
717 	/* 32-bit and 64-bit descriptors have 'cmd' in same position */  in sdhci_adma_mark_end()
718 	dma_desc->cmd |= cpu_to_le16(ADMA2_END);  in sdhci_adma_mark_end()
736 	host->sg_count = sg_count;  in sdhci_adma_table_pre()
738 	desc = host->adma_table;  in sdhci_adma_table_pre()
739 	align = host->align_buffer;  in sdhci_adma_table_pre()
741 	align_addr = host->align_addr;  in sdhci_adma_table_pre()
743 	for_each_sg(data->sg, sg, host->sg_count, i) {  in sdhci_adma_table_pre()
749 		 * be 32-bit aligned. If they aren't, then we use a bounce  in sdhci_adma_table_pre()
753 		offset = (SDHCI_ADMA2_ALIGN - (addr & SDHCI_ADMA2_MASK)) &  in sdhci_adma_table_pre()
756 			if (data->flags & MMC_DATA_WRITE) {  in sdhci_adma_table_pre()
772 			len -= offset;  in sdhci_adma_table_pre()
781 		while (len > host->max_adma) {  in sdhci_adma_table_pre()
786 			len -= n;  in sdhci_adma_table_pre()
798 		WARN_ON((desc - host->adma_table) >= host->adma_table_sz);  in sdhci_adma_table_pre()
801 	if (host->quirks & SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC) {  in sdhci_adma_table_pre()
803 		if (desc != host->adma_table) {  in sdhci_adma_table_pre()
804 			desc -= host->desc_sz;  in sdhci_adma_table_pre()
808 		/* Add a terminating entry - nop, end, valid */  in sdhci_adma_table_pre()
822 	if (data->flags & MMC_DATA_READ) {  in sdhci_adma_table_post()
826 		for_each_sg(data->sg, sg, host->sg_count, i)  in sdhci_adma_table_post()
833 			dma_sync_sg_for_cpu(mmc_dev(host->mmc), data->sg,  in sdhci_adma_table_post()
834 					    data->sg_len, DMA_FROM_DEVICE);  in sdhci_adma_table_post()
836 			align = host->align_buffer;  in sdhci_adma_table_post()
838 			for_each_sg(data->sg, sg, host->sg_count, i) {  in sdhci_adma_table_post()
840 					size = SDHCI_ADMA2_ALIGN -  in sdhci_adma_table_post()
857 	if (host->flags & SDHCI_USE_64_BIT_DMA)  in sdhci_set_adma_addr()
863 	if (host->bounce_buffer)  in sdhci_sdma_address()
864 		return host->bounce_addr;  in sdhci_sdma_address()
866 		return sg_dma_address(host->data->sg);  in sdhci_sdma_address()
871 	if (host->v4_mode)  in sdhci_set_sdma_addr()
878 					 struct mmc_command *cmd,  in sdhci_target_timeout()  argument
885 		target_timeout = cmd->busy_timeout * 1000;  in sdhci_target_timeout()
887 		target_timeout = DIV_ROUND_UP(data->timeout_ns, 1000);  in sdhci_target_timeout()
888 		if (host->clock && data->timeout_clks) {  in sdhci_target_timeout()
892 			 * data->timeout_clks is in units of clock cycles.  in sdhci_target_timeout()
893 			 * host->clock is in Hz.  target_timeout is in us.  in sdhci_target_timeout()
896 			val = 1000000ULL * data->timeout_clks;  in sdhci_target_timeout()
897 			if (do_div(val, host->clock))  in sdhci_target_timeout()
907 				  struct mmc_command *cmd)  in sdhci_calc_sw_timeout()  argument
909 	struct mmc_data *data = cmd->data;  in sdhci_calc_sw_timeout()
910 	struct mmc_host *mmc = host->mmc;  in sdhci_calc_sw_timeout()
911 	struct mmc_ios *ios = &mmc->ios;  in sdhci_calc_sw_timeout()
912 	unsigned char bus_width = 1 << ios->bus_width;  in sdhci_calc_sw_timeout()
918 	target_timeout = sdhci_target_timeout(host, cmd, data);  in sdhci_calc_sw_timeout()
922 		blksz = data->blksz;  in sdhci_calc_sw_timeout()
923 		freq = host->mmc->actual_clock ? : host->clock;  in sdhci_calc_sw_timeout()
929 		host->data_timeout = data->blocks * target_timeout +  in sdhci_calc_sw_timeout()
932 		host->data_timeout = target_timeout;  in sdhci_calc_sw_timeout()
935 	if (host->data_timeout)  in sdhci_calc_sw_timeout()
936 		host->data_timeout += MMC_CMD_TRANSFER_TIME;  in sdhci_calc_sw_timeout()
939 static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd,  in sdhci_calc_timeout()  argument
951 	 * longer to time out, but that's much better than having a too-short  in sdhci_calc_timeout()
954 	if (host->quirks & SDHCI_QUIRK_BROKEN_TIMEOUT_VAL)  in sdhci_calc_timeout()
958 	if (cmd == NULL)  in sdhci_calc_timeout()
961 	data = cmd->data;  in sdhci_calc_timeout()
963 	if (!data && !cmd->busy_timeout)  in sdhci_calc_timeout()
967 	target_timeout = sdhci_target_timeout(host, cmd, data);  in sdhci_calc_timeout()
975 	 * (2) host->timeout_clk < 2^16  in sdhci_calc_timeout()
980 	current_timeout = (1 << 13) * 1000 / host->timeout_clk;  in sdhci_calc_timeout()
989 		if (!(host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT))  in sdhci_calc_timeout()
990 			DBG("Too large timeout 0x%x requested for CMD%d!\n",  in sdhci_calc_timeout()
991 			    count, cmd->opcode);  in sdhci_calc_timeout()
1005 	if (host->flags & SDHCI_REQ_USE_DMA)  in sdhci_set_transfer_irqs()
1006 		host->ier = (host->ier & ~pio_irqs) | dma_irqs;  in sdhci_set_transfer_irqs()
1008 		host->ier = (host->ier & ~dma_irqs) | pio_irqs;  in sdhci_set_transfer_irqs()
1010 	if (host->flags & (SDHCI_AUTO_CMD23 | SDHCI_AUTO_CMD12))  in sdhci_set_transfer_irqs()
1011 		host->ier |= SDHCI_INT_AUTO_CMD_ERR;  in sdhci_set_transfer_irqs()
1013 		host->ier &= ~SDHCI_INT_AUTO_CMD_ERR;  in sdhci_set_transfer_irqs()
1015 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);  in sdhci_set_transfer_irqs()
1016 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);  in sdhci_set_transfer_irqs()
1022 		host->ier |= SDHCI_INT_DATA_TIMEOUT;  in sdhci_set_data_timeout_irq()
1024 		host->ier &= ~SDHCI_INT_DATA_TIMEOUT;  in sdhci_set_data_timeout_irq()
1025 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);  in sdhci_set_data_timeout_irq()
1026 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);  in sdhci_set_data_timeout_irq()
1030 void __sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)  in __sdhci_set_timeout()  argument
1033 	u8 count = sdhci_calc_timeout(host, cmd, &too_big);  in __sdhci_set_timeout()
1036 	    host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT) {  in __sdhci_set_timeout()
1037 		sdhci_calc_sw_timeout(host, cmd);  in __sdhci_set_timeout()
1039 	} else if (!(host->ier & SDHCI_INT_DATA_TIMEOUT)) {  in __sdhci_set_timeout()
1047 static void sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)  in sdhci_set_timeout()  argument
1049 	if (host->ops->set_timeout)  in sdhci_set_timeout()
1050 		host->ops->set_timeout(host, cmd);  in sdhci_set_timeout()
1052 		__sdhci_set_timeout(host, cmd);  in sdhci_set_timeout()
1058 	WARN_ON(host->data);  in sdhci_initialize_data()
1061 	BUG_ON(data->blksz * data->blocks > 524288);  in sdhci_initialize_data()
1062 	BUG_ON(data->blksz > host->mmc->max_blk_size);  in sdhci_initialize_data()
1063 	BUG_ON(data->blocks > 65535);  in sdhci_initialize_data()
1065 	host->data = data;  in sdhci_initialize_data()
1066 	host->data_early = 0;  in sdhci_initialize_data()
1067 	host->data->bytes_xfered = 0;  in sdhci_initialize_data()
1075 		     SDHCI_MAKE_BLKSZ(host->sdma_boundary, data->blksz),  in sdhci_set_block_info()
1078 	 * For Version 4.10 onwards, if v4 mode is enabled, 32-bit Block Count  in sdhci_set_block_info()
1079 	 * can be supported, in that case 16-bit block count register must be 0.  in sdhci_set_block_info()
1081 	if (host->version >= SDHCI_SPEC_410 && host->v4_mode &&  in sdhci_set_block_info()
1082 	    (host->quirks2 & SDHCI_QUIRK2_USE_32BIT_BLK_CNT)) {  in sdhci_set_block_info()
1085 		sdhci_writew(host, data->blocks, SDHCI_32BIT_BLK_CNT);  in sdhci_set_block_info()
1087 		sdhci_writew(host, data->blocks, SDHCI_BLOCK_COUNT);  in sdhci_set_block_info()
1091 static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd)  in sdhci_prepare_data()  argument
1093 	struct mmc_data *data = cmd->data;  in sdhci_prepare_data()
1097 	if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {  in sdhci_prepare_data()
1102 		host->flags |= SDHCI_REQ_USE_DMA;  in sdhci_prepare_data()
1113 		if (host->flags & SDHCI_USE_ADMA) {  in sdhci_prepare_data()
1114 			if (host->quirks & SDHCI_QUIRK_32BIT_ADMA_SIZE) {  in sdhci_prepare_data()
1124 			if (host->quirks & SDHCI_QUIRK_32BIT_DMA_SIZE)  in sdhci_prepare_data()
1126 			if (host->quirks & SDHCI_QUIRK_32BIT_DMA_ADDR)  in sdhci_prepare_data()
1131 			for_each_sg(data->sg, sg, data->sg_len, i) {  in sdhci_prepare_data()
1132 				if (sg->length & length_mask) {  in sdhci_prepare_data()
1134 					    sg->length);  in sdhci_prepare_data()
1135 					host->flags &= ~SDHCI_REQ_USE_DMA;  in sdhci_prepare_data()
1138 				if (sg->offset & offset_mask) {  in sdhci_prepare_data()
1140 					host->flags &= ~SDHCI_REQ_USE_DMA;  in sdhci_prepare_data()
1147 	if (host->flags & SDHCI_REQ_USE_DMA) {  in sdhci_prepare_data()
1156 			host->flags &= ~SDHCI_REQ_USE_DMA;  in sdhci_prepare_data()
1157 		} else if (host->flags & SDHCI_USE_ADMA) {  in sdhci_prepare_data()
1159 			sdhci_set_adma_addr(host, host->adma_addr);  in sdhci_prepare_data()
1168 	if (!(host->flags & SDHCI_REQ_USE_DMA)) {  in sdhci_prepare_data()
1172 		if (host->data->flags & MMC_DATA_READ)  in sdhci_prepare_data()
1176 		sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);  in sdhci_prepare_data()
1177 		host->blocks = data->blocks;  in sdhci_prepare_data()
1190 	struct mmc_host *mmc = host->mmc;  in sdhci_external_dma_init()
1192 	host->tx_chan = dma_request_chan(mmc->parent, "tx");  in sdhci_external_dma_init()
1193 	if (IS_ERR(host->tx_chan)) {  in sdhci_external_dma_init()
1194 		ret = PTR_ERR(host->tx_chan);  in sdhci_external_dma_init()
1195 		if (ret != -EPROBE_DEFER)  in sdhci_external_dma_init()
1197 		host->tx_chan = NULL;  in sdhci_external_dma_init()
1201 	host->rx_chan = dma_request_chan(mmc->parent, "rx");  in sdhci_external_dma_init()
1202 	if (IS_ERR(host->rx_chan)) {  in sdhci_external_dma_init()
1203 		if (host->tx_chan) {  in sdhci_external_dma_init()
1204 			dma_release_channel(host->tx_chan);  in sdhci_external_dma_init()
1205 			host->tx_chan = NULL;  in sdhci_external_dma_init()
1208 		ret = PTR_ERR(host->rx_chan);  in sdhci_external_dma_init()
1209 		if (ret != -EPROBE_DEFER)  in sdhci_external_dma_init()
1211 		host->rx_chan = NULL;  in sdhci_external_dma_init()
1220 	return data->flags & MMC_DATA_WRITE ? host->tx_chan : host->rx_chan;  in sdhci_external_dma_channel()
1224 				    struct mmc_command *cmd)  in sdhci_external_dma_setup()  argument
1229 	struct mmc_data *data = cmd->data;  in sdhci_external_dma_setup()
1235 	if (!host->mapbase)  in sdhci_external_dma_setup()
1236 		return -EINVAL;  in sdhci_external_dma_setup()
1239 	cfg.src_addr = host->mapbase + SDHCI_BUFFER;  in sdhci_external_dma_setup()
1240 	cfg.dst_addr = host->mapbase + SDHCI_BUFFER;  in sdhci_external_dma_setup()
1243 	cfg.src_maxburst = data->blksz / 4;  in sdhci_external_dma_setup()
1244 	cfg.dst_maxburst = data->blksz / 4;  in sdhci_external_dma_setup()
1247 	for (i = 0; i < data->sg_len; i++) {  in sdhci_external_dma_setup()
1248 		if ((data->sg + i)->length % data->blksz)  in sdhci_external_dma_setup()
1249 			return -EINVAL;  in sdhci_external_dma_setup()
1260 		return -EINVAL;  in sdhci_external_dma_setup()
1262 	dir = data->flags & MMC_DATA_WRITE ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;  in sdhci_external_dma_setup()
1263 	desc = dmaengine_prep_slave_sg(chan, data->sg, data->sg_len, dir,  in sdhci_external_dma_setup()
1266 		return -EINVAL;  in sdhci_external_dma_setup()
1268 	desc->callback = NULL;  in sdhci_external_dma_setup()
1269 	desc->callback_param = NULL;  in sdhci_external_dma_setup()
1280 	if (host->tx_chan) {  in sdhci_external_dma_release()
1281 		dma_release_channel(host->tx_chan);  in sdhci_external_dma_release()
1282 		host->tx_chan = NULL;  in sdhci_external_dma_release()
1285 	if (host->rx_chan) {  in sdhci_external_dma_release()
1286 		dma_release_channel(host->rx_chan);  in sdhci_external_dma_release()
1287 		host->rx_chan = NULL;  in sdhci_external_dma_release()
1294 					      struct mmc_command *cmd)  in __sdhci_external_dma_prepare_data()  argument
1296 	struct mmc_data *data = cmd->data;  in __sdhci_external_dma_prepare_data()
1300 	host->flags |= SDHCI_REQ_USE_DMA;  in __sdhci_external_dma_prepare_data()
1307 					    struct mmc_command *cmd)  in sdhci_external_dma_prepare_data()  argument
1309 	if (!sdhci_external_dma_setup(host, cmd)) {  in sdhci_external_dma_prepare_data()
1310 		__sdhci_external_dma_prepare_data(host, cmd);  in sdhci_external_dma_prepare_data()
1314 		       mmc_hostname(host->mmc));  in sdhci_external_dma_prepare_data()
1315 		sdhci_prepare_data(host, cmd);  in sdhci_external_dma_prepare_data()
1320 					    struct mmc_command *cmd)  in sdhci_external_dma_pre_transfer()  argument
1324 	if (!cmd->data)  in sdhci_external_dma_pre_transfer()
1327 	chan = sdhci_external_dma_channel(host, cmd->data);  in sdhci_external_dma_pre_transfer()
1336 	return -EOPNOTSUPP;  in sdhci_external_dma_init()
1344 						   struct mmc_command *cmd)  in sdhci_external_dma_prepare_data()  argument
1351 						   struct mmc_command *cmd)  in sdhci_external_dma_pre_transfer()  argument
1365 	host->use_external_dma = en;  in sdhci_switch_external_dma()
1372 	return !mrq->sbc && (host->flags & SDHCI_AUTO_CMD12) &&  in sdhci_auto_cmd12()
1373 	       !mrq->cap_cmd_during_tfr;  in sdhci_auto_cmd12()
1379 	return mrq->sbc && (host->flags & SDHCI_AUTO_CMD23);  in sdhci_auto_cmd23()
1385 	return mrq->sbc && !(host->flags & SDHCI_AUTO_CMD23);  in sdhci_manual_cmd23()
1389 					 struct mmc_command *cmd,  in sdhci_auto_cmd_select()  argument
1392 	bool use_cmd12 = sdhci_auto_cmd12(host, cmd->mrq) &&  in sdhci_auto_cmd_select()
1393 			 (cmd->opcode != SD_IO_RW_EXTENDED);  in sdhci_auto_cmd_select()
1394 	bool use_cmd23 = sdhci_auto_cmd23(host, cmd->mrq);  in sdhci_auto_cmd_select()
1398 	 * In case of Version 4.10 or later, use of 'Auto CMD Auto  in sdhci_auto_cmd_select()
1401 	 * here because some controllers (e.g sdhci-of-dwmshc) expect it.  in sdhci_auto_cmd_select()
1403 	if (host->version >= SDHCI_SPEC_410 && host->v4_mode &&  in sdhci_auto_cmd_select()
1419 	 * on successful completion (so no Auto-CMD12).  in sdhci_auto_cmd_select()
1428 	struct mmc_command *cmd)  in sdhci_set_transfer_mode()  argument
1431 	struct mmc_data *data = cmd->data;  in sdhci_set_transfer_mode()
1434 		if (host->quirks2 &  in sdhci_set_transfer_mode()
1437 			if (cmd->opcode != MMC_SEND_TUNING_BLOCK_HS200)  in sdhci_set_transfer_mode()
1440 		/* clear Auto CMD settings for no data CMDs */  in sdhci_set_transfer_mode()
1448 	WARN_ON(!host->data);  in sdhci_set_transfer_mode()
1450 	if (!(host->quirks2 & SDHCI_QUIRK2_SUPPORT_SINGLE))  in sdhci_set_transfer_mode()
1453 	if (mmc_op_multi(cmd->opcode) || data->blocks > 1) {  in sdhci_set_transfer_mode()
1455 		sdhci_auto_cmd_select(host, cmd, &mode);  in sdhci_set_transfer_mode()
1456 		if (sdhci_auto_cmd23(host, cmd->mrq))  in sdhci_set_transfer_mode()
1457 			sdhci_writel(host, cmd->mrq->sbc->arg, SDHCI_ARGUMENT2);  in sdhci_set_transfer_mode()
1460 	if (data->flags & MMC_DATA_READ)  in sdhci_set_transfer_mode()
1462 	if (host->flags & SDHCI_REQ_USE_DMA)  in sdhci_set_transfer_mode()
1470 	return (!(host->flags & SDHCI_DEVICE_DEAD) &&  in sdhci_needs_reset()
1471 		((mrq->cmd && mrq->cmd->error) ||  in sdhci_needs_reset()
1472 		 (mrq->sbc && mrq->sbc->error) ||  in sdhci_needs_reset()
1473 		 (mrq->data && mrq->data->stop && mrq->data->stop->error) ||  in sdhci_needs_reset()
1474 		 (host->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST)));  in sdhci_needs_reset()
1482 		if (host->mrqs_done[i] == mrq) {  in sdhci_set_mrq_done()
1489 		if (!host->mrqs_done[i]) {  in sdhci_set_mrq_done()
1490 			host->mrqs_done[i] = mrq;  in sdhci_set_mrq_done()
1500 	if (host->cmd && host->cmd->mrq == mrq)  in __sdhci_finish_mrq()
1501 		host->cmd = NULL;  in __sdhci_finish_mrq()
1503 	if (host->data_cmd && host->data_cmd->mrq == mrq)  in __sdhci_finish_mrq()
1504 		host->data_cmd = NULL;  in __sdhci_finish_mrq()
1506 	if (host->deferred_cmd && host->deferred_cmd->mrq == mrq)  in __sdhci_finish_mrq()
1507 		host->deferred_cmd = NULL;  in __sdhci_finish_mrq()
1509 	if (host->data && host->data->mrq == mrq)  in __sdhci_finish_mrq()
1510 		host->data = NULL;  in __sdhci_finish_mrq()
1513 		host->pending_reset = true;  in __sdhci_finish_mrq()
1527 	queue_work(host->complete_wq, &host->complete_work);  in sdhci_finish_mrq()
1532 	struct mmc_command *data_cmd = host->data_cmd;  in __sdhci_finish_data()
1533 	struct mmc_data *data = host->data;  in __sdhci_finish_data()
1535 	host->data = NULL;  in __sdhci_finish_data()
1536 	host->data_cmd = NULL;  in __sdhci_finish_data()
1542 	if (data->error) {  in __sdhci_finish_data()
1543 		if (!host->cmd || host->cmd == data_cmd)  in __sdhci_finish_data()
1548 	if ((host->flags & (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA)) ==  in __sdhci_finish_data()
1559 	if (data->error)  in __sdhci_finish_data()
1560 		data->bytes_xfered = 0;  in __sdhci_finish_data()
1562 		data->bytes_xfered = data->blksz * data->blocks;  in __sdhci_finish_data()
1565 	 * Need to send CMD12 if -  in __sdhci_finish_data()
1566 	 * a) open-ended multiblock transfer not using auto CMD12 (no CMD23)  in __sdhci_finish_data()
1569 	if (data->stop &&  in __sdhci_finish_data()
1570 	    ((!data->mrq->sbc && !sdhci_auto_cmd12(host, data->mrq)) ||  in __sdhci_finish_data()
1571 	     data->error)) {  in __sdhci_finish_data()
1577 		if (data->mrq->cap_cmd_during_tfr) {  in __sdhci_finish_data()
1578 			__sdhci_finish_mrq(host, data->mrq);  in __sdhci_finish_data()
1581 			host->cmd = NULL;  in __sdhci_finish_data()
1582 			if (!sdhci_send_command(host, data->stop)) {  in __sdhci_finish_data()
1588 					data->stop->error = -EIO;  in __sdhci_finish_data()
1589 					__sdhci_finish_mrq(host, data->mrq);  in __sdhci_finish_data()
1591 					WARN_ON(host->deferred_cmd);  in __sdhci_finish_data()
1592 					host->deferred_cmd = data->stop;  in __sdhci_finish_data()
1597 		__sdhci_finish_mrq(host, data->mrq);  in __sdhci_finish_data()
1606 static bool sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd)  in sdhci_send_command()  argument
1612 	WARN_ON(host->cmd);  in sdhci_send_command()
1615 	cmd->error = 0;  in sdhci_send_command()
1617 	if ((host->quirks2 & SDHCI_QUIRK2_STOP_WITH_TC) &&  in sdhci_send_command()
1618 	    cmd->opcode == MMC_STOP_TRANSMISSION)  in sdhci_send_command()
1619 		cmd->flags |= MMC_RSP_BUSY;  in sdhci_send_command()
1622 	if (sdhci_data_line_cmd(cmd))  in sdhci_send_command()
1627 	if (cmd->mrq->data && (cmd == cmd->mrq->data->stop))  in sdhci_send_command()
1633 	host->cmd = cmd;  in sdhci_send_command()
1634 	host->data_timeout = 0;  in sdhci_send_command()
1635 	if (sdhci_data_line_cmd(cmd)) {  in sdhci_send_command()
1636 		WARN_ON(host->data_cmd);  in sdhci_send_command()
1637 		host->data_cmd = cmd;  in sdhci_send_command()
1638 		sdhci_set_timeout(host, cmd);  in sdhci_send_command()
1641 	if (cmd->data) {  in sdhci_send_command()
1642 		if (host->use_external_dma)  in sdhci_send_command()
1643 			sdhci_external_dma_prepare_data(host, cmd);  in sdhci_send_command()
1645 			sdhci_prepare_data(host, cmd);  in sdhci_send_command()
1648 	sdhci_writel(host, cmd->arg, SDHCI_ARGUMENT);  in sdhci_send_command()
1650 	sdhci_set_transfer_mode(host, cmd);  in sdhci_send_command()
1652 	if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {  in sdhci_send_command()
1655 		 * This does not happen in practice because 136-bit response  in sdhci_send_command()
1659 		cmd->flags &= ~MMC_RSP_BUSY;  in sdhci_send_command()
1662 	if (!(cmd->flags & MMC_RSP_PRESENT))  in sdhci_send_command()
1664 	else if (cmd->flags & MMC_RSP_136)  in sdhci_send_command()
1666 	else if (cmd->flags & MMC_RSP_BUSY)  in sdhci_send_command()
1671 	if (cmd->flags & MMC_RSP_CRC)  in sdhci_send_command()
1673 	if (cmd->flags & MMC_RSP_OPCODE)  in sdhci_send_command()
1677 	if (cmd->data || cmd->opcode == MMC_SEND_TUNING_BLOCK ||  in sdhci_send_command()
1678 	    cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200)  in sdhci_send_command()
1682 	if (host->data_timeout)  in sdhci_send_command()
1683 		timeout += nsecs_to_jiffies(host->data_timeout);  in sdhci_send_command()
1684 	else if (!cmd->data && cmd->busy_timeout > 9000)  in sdhci_send_command()
1685 		timeout += DIV_ROUND_UP(cmd->busy_timeout, 1000) * HZ + HZ;  in sdhci_send_command()
1688 	sdhci_mod_timer(host, cmd->mrq, timeout);  in sdhci_send_command()
1690 	if (host->use_external_dma)  in sdhci_send_command()
1691 		sdhci_external_dma_pre_transfer(host, cmd);  in sdhci_send_command()
1693 	sdhci_writew(host, SDHCI_MAKE_CMD(cmd->opcode, flags), SDHCI_COMMAND);  in sdhci_send_command()
1699 				struct mmc_command *cmd, bool present)  in sdhci_present_error()  argument
1701 	if (!present || host->flags & SDHCI_DEVICE_DEAD) {  in sdhci_present_error()
1702 		cmd->error = -ENOMEDIUM;  in sdhci_present_error()
1710 				     struct mmc_command *cmd,  in sdhci_send_command_retry()  argument
1712 	__releases(host->lock)  in sdhci_send_command_retry()
1713 	__acquires(host->lock)  in sdhci_send_command_retry()
1715 	struct mmc_command *deferred_cmd = host->deferred_cmd;  in sdhci_send_command_retry()
1719 	while (!sdhci_send_command(host, cmd)) {  in sdhci_send_command_retry()
1720 		if (!timeout--) {  in sdhci_send_command_retry()
1722 			       mmc_hostname(host->mmc));  in sdhci_send_command_retry()
1724 			cmd->error = -EIO;  in sdhci_send_command_retry()
1728 		spin_unlock_irqrestore(&host->lock, flags);  in sdhci_send_command_retry()
1732 		present = host->mmc->ops->get_cd(host->mmc);  in sdhci_send_command_retry()
1734 		spin_lock_irqsave(&host->lock, flags);  in sdhci_send_command_retry()
1737 		if (cmd == deferred_cmd && cmd != host->deferred_cmd)  in sdhci_send_command_retry()
1740 		if (sdhci_present_error(host, cmd, present))  in sdhci_send_command_retry()
1744 	if (cmd == host->deferred_cmd)  in sdhci_send_command_retry()
1745 		host->deferred_cmd = NULL;  in sdhci_send_command_retry()
1750 static void sdhci_read_rsp_136(struct sdhci_host *host, struct mmc_command *cmd)  in sdhci_read_rsp_136()  argument
1755 		reg = SDHCI_RESPONSE + (3 - i) * 4;  in sdhci_read_rsp_136()
1756 		cmd->resp[i] = sdhci_readl(host, reg);  in sdhci_read_rsp_136()
1759 	if (host->quirks2 & SDHCI_QUIRK2_RSP_136_HAS_CRC)  in sdhci_read_rsp_136()
1764 		cmd->resp[i] <<= 8;  in sdhci_read_rsp_136()
1766 			cmd->resp[i] |= cmd->resp[i + 1] >> 24;  in sdhci_read_rsp_136()
1772 	struct mmc_command *cmd = host->cmd;  in sdhci_finish_command()  local
1774 	host->cmd = NULL;  in sdhci_finish_command()
1776 	if (cmd->flags & MMC_RSP_PRESENT) {  in sdhci_finish_command()
1777 		if (cmd->flags & MMC_RSP_136) {  in sdhci_finish_command()
1778 			sdhci_read_rsp_136(host, cmd);  in sdhci_finish_command()
1780 			cmd->resp[0] = sdhci_readl(host, SDHCI_RESPONSE);  in sdhci_finish_command()
1784 	if (cmd->mrq->cap_cmd_during_tfr && cmd == cmd->mrq->cmd)  in sdhci_finish_command()
1785 		mmc_command_done(host->mmc, cmd->mrq);  in sdhci_finish_command()
1797 	if (cmd->flags & MMC_RSP_BUSY) {  in sdhci_finish_command()
1798 		if (cmd->data) {  in sdhci_finish_command()
1800 		} else if (!(host->quirks & SDHCI_QUIRK_NO_BUSY_IRQ) &&  in sdhci_finish_command()
1801 			   cmd == host->data_cmd) {  in sdhci_finish_command()
1808 	if (cmd == cmd->mrq->sbc) {  in sdhci_finish_command()
1809 		if (!sdhci_send_command(host, cmd->mrq->cmd)) {  in sdhci_finish_command()
1810 			WARN_ON(host->deferred_cmd);  in sdhci_finish_command()
1811 			host->deferred_cmd = cmd->mrq->cmd;  in sdhci_finish_command()
1816 		if (host->data && host->data_early)  in sdhci_finish_command()
1819 		if (!cmd->data)  in sdhci_finish_command()
1820 			__sdhci_finish_mrq(host, cmd->mrq);  in sdhci_finish_command()
1828 	switch (host->timing) {  in sdhci_get_preset_value()
1854 		pr_warn("%s: Invalid UHS-I mode selected\n",  in sdhci_get_preset_value()
1855 			mmc_hostname(host->mmc));  in sdhci_get_preset_value()
1870 	if (host->version >= SDHCI_SPEC_300) {  in sdhci_calc_clk()
1871 		if (host->preset_enabled) {  in sdhci_calc_clk()
1877 			if (host->clk_mul &&  in sdhci_calc_clk()
1881 				clk_mul = host->clk_mul;  in sdhci_calc_clk()
1892 		if (host->clk_mul) {  in sdhci_calc_clk()
1894 				if ((host->max_clk * host->clk_mul / div)  in sdhci_calc_clk()
1898 			if ((host->max_clk * host->clk_mul / div) <= clock) {  in sdhci_calc_clk()
1905 				clk_mul = host->clk_mul;  in sdhci_calc_clk()
1906 				div--;  in sdhci_calc_clk()
1916 		if (!host->clk_mul || switch_base_clk) {  in sdhci_calc_clk()
1918 			if (host->max_clk <= clock)  in sdhci_calc_clk()
1923 					if ((host->max_clk / div) <= clock)  in sdhci_calc_clk()
1929 			if ((host->quirks2 & SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN)  in sdhci_calc_clk()
1930 				&& !div && host->max_clk <= 25000000)  in sdhci_calc_clk()
1936 			if ((host->max_clk / div) <= clock)  in sdhci_calc_clk()
1945 		*actual_clock = (host->max_clk * clk_mul) / real_div;  in sdhci_calc_clk()
1971 			       mmc_hostname(host->mmc));  in sdhci_enable_clk()
1978 	if (host->version >= SDHCI_SPEC_410 && host->v4_mode) {  in sdhci_enable_clk()
1993 				       mmc_hostname(host->mmc));  in sdhci_enable_clk()
2010 	host->mmc->actual_clock = 0;  in sdhci_set_clock()
2017 	clk = sdhci_calc_clk(host, clock, &host->mmc->actual_clock);  in sdhci_set_clock()
2025 	struct mmc_host *mmc = host->mmc;  in sdhci_set_power_reg()
2027 	mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);  in sdhci_set_power_reg()
2068 			     mmc_hostname(host->mmc), vdd);  in sdhci_set_power_noreg()
2073 	if (host->pwr == pwr)  in sdhci_set_power_noreg()
2076 	host->pwr = pwr;  in sdhci_set_power_noreg()
2080 		if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)  in sdhci_set_power_noreg()
2087 		if (!(host->quirks & SDHCI_QUIRK_SINGLE_POWER_WRITE))  in sdhci_set_power_noreg()
2095 		if (host->quirks & SDHCI_QUIRK_NO_SIMULT_VDD_AND_POWER)  in sdhci_set_power_noreg()
2102 		if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)  in sdhci_set_power_noreg()
2109 		if (host->quirks & SDHCI_QUIRK_DELAY_AFTER_POWER)  in sdhci_set_power_noreg()
2118 	if (IS_ERR(host->mmc->supply.vmmc))  in sdhci_set_power()
2135 	if (!IS_ERR(host->mmc->supply.vmmc)) {  in sdhci_set_power_and_bus_voltage()
2136 		struct mmc_host *mmc = host->mmc;  in sdhci_set_power_and_bus_voltage()
2138 		mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);  in sdhci_set_power_and_bus_voltage()
2153 	struct mmc_command *cmd;  in sdhci_request()  local
2158 	present = mmc->ops->get_cd(mmc);  in sdhci_request()
2160 	spin_lock_irqsave(&host->lock, flags);  in sdhci_request()
2164 	if (sdhci_present_error(host, mrq->cmd, present))  in sdhci_request()
2167 	cmd = sdhci_manual_cmd23(host, mrq) ? mrq->sbc : mrq->cmd;  in sdhci_request()
2169 	if (!sdhci_send_command_retry(host, cmd, flags))  in sdhci_request()
2172 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_request()
2178 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_request()
2185 	struct mmc_command *cmd;  in sdhci_request_atomic()  local
2189 	spin_lock_irqsave(&host->lock, flags);  in sdhci_request_atomic()
2191 	if (sdhci_present_error(host, mrq->cmd, true)) {  in sdhci_request_atomic()
2196 	cmd = sdhci_manual_cmd23(host, mrq) ? mrq->sbc : mrq->cmd;  in sdhci_request_atomic()
2202 	 * again in non-atomic context. So we should not finish this request  in sdhci_request_atomic()
2205 	if (!sdhci_send_command(host, cmd))  in sdhci_request_atomic()
2206 		ret = -EBUSY;  in sdhci_request_atomic()
2211 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_request_atomic()
2225 		if (host->mmc->caps & MMC_CAP_8_BIT_DATA)  in sdhci_set_bus_width()
2256 		ctrl_2 |= SDHCI_CTRL_HS400; /* Non-standard */  in sdhci_set_uhs_signaling()
2266 	if (ios->power_mode == MMC_POWER_UNDEFINED)  in sdhci_set_ios()
2269 	if (host->flags & SDHCI_DEVICE_DEAD) {  in sdhci_set_ios()
2270 		if (!IS_ERR(mmc->supply.vmmc) &&  in sdhci_set_ios()
2271 		    ios->power_mode == MMC_POWER_OFF)  in sdhci_set_ios()
2272 			mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);  in sdhci_set_ios()
2280 	if (ios->power_mode == MMC_POWER_OFF) {  in sdhci_set_ios()
2285 	if (host->version >= SDHCI_SPEC_300 &&  in sdhci_set_ios()
2286 		(ios->power_mode == MMC_POWER_UP) &&  in sdhci_set_ios()
2287 		!(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN))  in sdhci_set_ios()
2290 	if (!ios->clock || ios->clock != host->clock) {  in sdhci_set_ios()
2291 		host->ops->set_clock(host, ios->clock);  in sdhci_set_ios()
2292 		host->clock = ios->clock;  in sdhci_set_ios()
2294 		if (host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK &&  in sdhci_set_ios()
2295 		    host->clock) {  in sdhci_set_ios()
2296 			host->timeout_clk = host->mmc->actual_clock ?  in sdhci_set_ios()
2297 						host->mmc->actual_clock / 1000 :  in sdhci_set_ios()
2298 						host->clock / 1000;  in sdhci_set_ios()
2299 			host->mmc->max_busy_timeout =  in sdhci_set_ios()
2300 				host->ops->get_max_timeout_count ?  in sdhci_set_ios()
2301 				host->ops->get_max_timeout_count(host) :  in sdhci_set_ios()
2303 			host->mmc->max_busy_timeout /= host->timeout_clk;  in sdhci_set_ios()
2307 	if (host->ops->set_power)  in sdhci_set_ios()
2308 		host->ops->set_power(host, ios->power_mode, ios->vdd);  in sdhci_set_ios()
2310 		sdhci_set_power(host, ios->power_mode, ios->vdd);  in sdhci_set_ios()
2312 	if (host->ops->platform_send_init_74_clocks)  in sdhci_set_ios()
2313 		host->ops->platform_send_init_74_clocks(host, ios->power_mode);  in sdhci_set_ios()
2315 	host->ops->set_bus_width(host, ios->bus_width);  in sdhci_set_ios()
2319 	if (!(host->quirks & SDHCI_QUIRK_NO_HISPD_BIT)) {  in sdhci_set_ios()
2320 		if (ios->timing == MMC_TIMING_SD_HS ||  in sdhci_set_ios()
2321 		     ios->timing == MMC_TIMING_MMC_HS ||  in sdhci_set_ios()
2322 		     ios->timing == MMC_TIMING_MMC_HS400 ||  in sdhci_set_ios()
2323 		     ios->timing == MMC_TIMING_MMC_HS200 ||  in sdhci_set_ios()
2324 		     ios->timing == MMC_TIMING_MMC_DDR52 ||  in sdhci_set_ios()
2325 		     ios->timing == MMC_TIMING_UHS_SDR50 ||  in sdhci_set_ios()
2326 		     ios->timing == MMC_TIMING_UHS_SDR104 ||  in sdhci_set_ios()
2327 		     ios->timing == MMC_TIMING_UHS_DDR50 ||  in sdhci_set_ios()
2328 		     ios->timing == MMC_TIMING_UHS_SDR25)  in sdhci_set_ios()
2334 	if (host->version >= SDHCI_SPEC_300) {  in sdhci_set_ios()
2337 		if (!host->preset_enabled) {  in sdhci_set_ios()
2345 			if (ios->drv_type == MMC_SET_DRIVER_TYPE_A)  in sdhci_set_ios()
2347 			else if (ios->drv_type == MMC_SET_DRIVER_TYPE_B)  in sdhci_set_ios()
2349 			else if (ios->drv_type == MMC_SET_DRIVER_TYPE_C)  in sdhci_set_ios()
2351 			else if (ios->drv_type == MMC_SET_DRIVER_TYPE_D)  in sdhci_set_ios()
2375 			/* Re-enable SD Clock */  in sdhci_set_ios()
2376 			host->ops->set_clock(host, host->clock);  in sdhci_set_ios()
2384 		host->ops->set_uhs_signaling(host, ios->timing);  in sdhci_set_ios()
2385 		host->timing = ios->timing;  in sdhci_set_ios()
2387 		if (!(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN) &&  in sdhci_set_ios()
2388 				((ios->timing == MMC_TIMING_UHS_SDR12) ||  in sdhci_set_ios()
2389 				 (ios->timing == MMC_TIMING_UHS_SDR25) ||  in sdhci_set_ios()
2390 				 (ios->timing == MMC_TIMING_UHS_SDR50) ||  in sdhci_set_ios()
2391 				 (ios->timing == MMC_TIMING_UHS_SDR104) ||  in sdhci_set_ios()
2392 				 (ios->timing == MMC_TIMING_UHS_DDR50) ||  in sdhci_set_ios()
2393 				 (ios->timing == MMC_TIMING_MMC_DDR52))) {  in sdhci_set_ios()
2398 			ios->drv_type = FIELD_GET(SDHCI_PRESET_DRV_MASK,  in sdhci_set_ios()
2402 		/* Re-enable SD Clock */  in sdhci_set_ios()
2403 		host->ops->set_clock(host, host->clock);  in sdhci_set_ios()
2412 	if (host->quirks & SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS)  in sdhci_set_ios()
2422 	if (host->flags & SDHCI_DEVICE_DEAD)  in sdhci_get_cd()
2426 	if (!mmc_card_is_removable(host->mmc))  in sdhci_get_cd()
2437 	if (host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION)  in sdhci_get_cd()
2449 	spin_lock_irqsave(&host->lock, flags);  in sdhci_check_ro()
2451 	if (host->flags & SDHCI_DEVICE_DEAD)  in sdhci_check_ro()
2453 	else if (host->ops->get_ro)  in sdhci_check_ro()
2454 		is_readonly = host->ops->get_ro(host);  in sdhci_check_ro()
2455 	else if (mmc_can_gpio_ro(host->mmc))  in sdhci_check_ro()
2456 		is_readonly = mmc_gpio_get_ro(host->mmc);  in sdhci_check_ro()
2461 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_check_ro()
2463 	/* This quirk needs to be replaced by a callback-function later */  in sdhci_check_ro()
2464 	return host->quirks & SDHCI_QUIRK_INVERTED_WRITE_PROTECT ?  in sdhci_check_ro()
2475 	if (!(host->quirks & SDHCI_QUIRK_UNSTABLE_RO_DETECT))  in sdhci_get_ro()
2493 	if (host->ops && host->ops->hw_reset)  in sdhci_hw_reset()
2494 		host->ops->hw_reset(host);  in sdhci_hw_reset()
2499 	if (!(host->flags & SDHCI_DEVICE_DEAD)) {  in sdhci_enable_sdio_irq_nolock()
2501 			host->ier |= SDHCI_INT_CARD_INT;  in sdhci_enable_sdio_irq_nolock()
2503 			host->ier &= ~SDHCI_INT_CARD_INT;  in sdhci_enable_sdio_irq_nolock()
2505 		sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);  in sdhci_enable_sdio_irq_nolock()
2506 		sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);  in sdhci_enable_sdio_irq_nolock()
2516 		pm_runtime_get_noresume(host->mmc->parent);  in sdhci_enable_sdio_irq()
2518 	spin_lock_irqsave(&host->lock, flags);  in sdhci_enable_sdio_irq()
2520 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_enable_sdio_irq()
2523 		pm_runtime_put_noidle(host->mmc->parent);  in sdhci_enable_sdio_irq()
2532 	spin_lock_irqsave(&host->lock, flags);  in sdhci_ack_sdio_irq()
2534 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_ack_sdio_irq()
2548 	if (host->version < SDHCI_SPEC_300)  in sdhci_start_signal_voltage_switch()
2553 	switch (ios->signal_voltage) {  in sdhci_start_signal_voltage_switch()
2555 		if (!(host->flags & SDHCI_SIGNALING_330))  in sdhci_start_signal_voltage_switch()
2556 			return -EINVAL;  in sdhci_start_signal_voltage_switch()
2561 		if (!IS_ERR(mmc->supply.vqmmc)) {  in sdhci_start_signal_voltage_switch()
2566 				return -EIO;  in sdhci_start_signal_voltage_switch()
2580 		return -EAGAIN;  in sdhci_start_signal_voltage_switch()
2582 		if (!(host->flags & SDHCI_SIGNALING_180))  in sdhci_start_signal_voltage_switch()
2583 			return -EINVAL;  in sdhci_start_signal_voltage_switch()
2584 		if (!IS_ERR(mmc->supply.vqmmc)) {  in sdhci_start_signal_voltage_switch()
2589 				return -EIO;  in sdhci_start_signal_voltage_switch()
2601 		if (host->ops->voltage_switch)  in sdhci_start_signal_voltage_switch()
2602 			host->ops->voltage_switch(host);  in sdhci_start_signal_voltage_switch()
2612 		return -EAGAIN;  in sdhci_start_signal_voltage_switch()
2614 		if (!(host->flags & SDHCI_SIGNALING_120))  in sdhci_start_signal_voltage_switch()
2615 			return -EINVAL;  in sdhci_start_signal_voltage_switch()
2616 		if (!IS_ERR(mmc->supply.vqmmc)) {  in sdhci_start_signal_voltage_switch()
2621 				return -EIO;  in sdhci_start_signal_voltage_switch()
2637 	/* Check whether DAT[0] is 0 */  in sdhci_card_busy()
2648 	spin_lock_irqsave(&host->lock, flags);  in sdhci_prepare_hs400_tuning()
2649 	host->flags |= SDHCI_HS400_TUNING;  in sdhci_prepare_hs400_tuning()
2650 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_prepare_hs400_tuning()
2661 	if (host->quirks2 & SDHCI_QUIRK2_TUNING_WORK_AROUND)  in sdhci_start_tuning()
2682 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);  in sdhci_end_tuning()
2683 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);  in sdhci_end_tuning()
2707 	mmc_abort_tuning(host->mmc, opcode);  in sdhci_abort_tuning()
2714  * automatically) so mmc_send_tuning() will return -EIO. Also the tuning command
2720 	struct mmc_host *mmc = host->mmc;  in sdhci_send_tuning()
2721 	struct mmc_command cmd = {};  in sdhci_send_tuning()  local
2724 	u32 b = host->sdma_boundary;  in sdhci_send_tuning()
2726 	spin_lock_irqsave(&host->lock, flags);  in sdhci_send_tuning()
2728 	cmd.opcode = opcode;  in sdhci_send_tuning()
2729 	cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;  in sdhci_send_tuning()
2730 	cmd.mrq = &mrq;  in sdhci_send_tuning()
2732 	mrq.cmd = &cmd;  in sdhci_send_tuning()
2738 	if (cmd.opcode == MMC_SEND_TUNING_BLOCK_HS200 &&  in sdhci_send_tuning()
2739 	    mmc->ios.bus_width == MMC_BUS_WIDTH_8)  in sdhci_send_tuning()
2752 	if (!sdhci_send_command_retry(host, &cmd, flags)) {  in sdhci_send_tuning()
2753 		spin_unlock_irqrestore(&host->lock, flags);  in sdhci_send_tuning()
2754 		host->tuning_done = 0;  in sdhci_send_tuning()
2758 	host->cmd = NULL;  in sdhci_send_tuning()
2762 	host->tuning_done = 0;  in sdhci_send_tuning()
2764 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_send_tuning()
2767 	wait_event_timeout(host->buf_ready_int, (host->tuning_done == 1),  in sdhci_send_tuning()
2781 	for (i = 0; i < host->tuning_loop_count; i++) {  in __sdhci_execute_tuning()
2786 		if (!host->tuning_done) {  in __sdhci_execute_tuning()
2788 				 mmc_hostname(host->mmc));  in __sdhci_execute_tuning()
2790 			return -ETIMEDOUT;  in __sdhci_execute_tuning()
2794 		if (host->tuning_delay > 0)  in __sdhci_execute_tuning()
2795 			mdelay(host->tuning_delay);  in __sdhci_execute_tuning()
2807 		mmc_hostname(host->mmc));  in __sdhci_execute_tuning()
2809 	return -EAGAIN;  in __sdhci_execute_tuning()
2819 	hs400_tuning = host->flags & SDHCI_HS400_TUNING;  in sdhci_execute_tuning()
2821 	if (host->tuning_mode == SDHCI_TUNING_MODE_1)  in sdhci_execute_tuning()
2822 		tuning_count = host->tuning_count;  in sdhci_execute_tuning()
2831 	switch (host->timing) {  in sdhci_execute_tuning()
2834 		err = -EINVAL;  in sdhci_execute_tuning()
2839 		 * Periodic re-tuning for HS400 is not expected to be needed, so  in sdhci_execute_tuning()
2851 		if (host->flags & SDHCI_SDR50_NEEDS_TUNING)  in sdhci_execute_tuning()
2859 	if (host->ops->platform_execute_tuning) {  in sdhci_execute_tuning()
2860 		err = host->ops->platform_execute_tuning(host, opcode);  in sdhci_execute_tuning()
2864 	host->mmc->retune_period = tuning_count;  in sdhci_execute_tuning()
2866 	if (host->tuning_delay < 0)  in sdhci_execute_tuning()
2867 		host->tuning_delay = opcode == MMC_SEND_TUNING_BLOCK;  in sdhci_execute_tuning()
2871 	host->tuning_err = __sdhci_execute_tuning(host, opcode);  in sdhci_execute_tuning()
2875 	host->flags &= ~SDHCI_HS400_TUNING;  in sdhci_execute_tuning()
2884 	if (host->version < SDHCI_SPEC_300)  in sdhci_enable_preset_value()
2891 	if (host->preset_enabled != enable) {  in sdhci_enable_preset_value()
2902 			host->flags |= SDHCI_PV_ENABLED;  in sdhci_enable_preset_value()
2904 			host->flags &= ~SDHCI_PV_ENABLED;  in sdhci_enable_preset_value()
2906 		host->preset_enabled = enable;  in sdhci_enable_preset_value()
2914 	struct mmc_data *data = mrq->data;  in sdhci_post_req()
2916 	if (data->host_cookie != COOKIE_UNMAPPED)  in sdhci_post_req()
2917 		dma_unmap_sg(mmc_dev(host->mmc), data->sg, data->sg_len,  in sdhci_post_req()
2920 	data->host_cookie = COOKIE_UNMAPPED;  in sdhci_post_req()
2927 	mrq->data->host_cookie = COOKIE_UNMAPPED;  in sdhci_pre_req()
2930 	 * No pre-mapping in the pre hook if we're using the bounce buffer,  in sdhci_pre_req()
2934 	if (host->flags & SDHCI_REQ_USE_DMA && !host->bounce_buffer)  in sdhci_pre_req()
2935 		sdhci_pre_dma_transfer(host, mrq->data, COOKIE_PRE_MAPPED);  in sdhci_pre_req()
2940 	if (host->data_cmd) {  in sdhci_error_out_mrqs()
2941 		host->data_cmd->error = err;  in sdhci_error_out_mrqs()
2942 		sdhci_finish_mrq(host, host->data_cmd->mrq);  in sdhci_error_out_mrqs()
2945 	if (host->cmd) {  in sdhci_error_out_mrqs()
2946 		host->cmd->error = err;  in sdhci_error_out_mrqs()
2947 		sdhci_finish_mrq(host, host->cmd->mrq);  in sdhci_error_out_mrqs()
2958 	if (host->ops->card_event)  in sdhci_card_event()
2959 		host->ops->card_event(host);  in sdhci_card_event()
2961 	present = mmc->ops->get_cd(mmc);  in sdhci_card_event()
2963 	spin_lock_irqsave(&host->lock, flags);  in sdhci_card_event()
2968 			mmc_hostname(host->mmc));  in sdhci_card_event()
2970 			mmc_hostname(host->mmc));  in sdhci_card_event()
2975 		sdhci_error_out_mrqs(host, -ENOMEDIUM);  in sdhci_card_event()
2978 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_card_event()
3010 	spin_lock_irqsave(&host->lock, flags);  in sdhci_request_done()
3013 		mrq = host->mrqs_done[i];  in sdhci_request_done()
3019 		spin_unlock_irqrestore(&host->lock, flags);  in sdhci_request_done()
3031 		 * also be in mrqs_done, otherwise host->cmd and host->data_cmd  in sdhci_request_done()
3034 		if (host->cmd || host->data_cmd) {  in sdhci_request_done()
3035 			spin_unlock_irqrestore(&host->lock, flags);  in sdhci_request_done()
3040 		if (host->quirks & SDHCI_QUIRK_CLOCK_BEFORE_RESET)  in sdhci_request_done()
3042 			host->ops->set_clock(host, host->clock);  in sdhci_request_done()
3051 		host->pending_reset = false;  in sdhci_request_done()
3059 	if (host->flags & SDHCI_REQ_USE_DMA) {  in sdhci_request_done()
3060 		struct mmc_data *data = mrq->data;  in sdhci_request_done()
3062 		if (host->use_external_dma && data &&  in sdhci_request_done()
3063 		    (mrq->cmd->error || data->error)) {  in sdhci_request_done()
3066 			host->mrqs_done[i] = NULL;  in sdhci_request_done()
3067 			spin_unlock_irqrestore(&host->lock, flags);  in sdhci_request_done()
3069 			spin_lock_irqsave(&host->lock, flags);  in sdhci_request_done()
3073 		if (data && data->host_cookie == COOKIE_MAPPED) {  in sdhci_request_done()
3074 			if (host->bounce_buffer) {  in sdhci_request_done()
3080 					unsigned int length = data->bytes_xfered;  in sdhci_request_done()
3082 					if (length > host->bounce_buffer_size) {  in sdhci_request_done()
3084 						       mmc_hostname(host->mmc),  in sdhci_request_done()
3085 						       host->bounce_buffer_size,  in sdhci_request_done()
3086 						       data->bytes_xfered);  in sdhci_request_done()
3088 						length = host->bounce_buffer_size;  in sdhci_request_done()
3091 						host->mmc->parent,  in sdhci_request_done()
3092 						host->bounce_addr,  in sdhci_request_done()
3093 						host->bounce_buffer_size,  in sdhci_request_done()
3095 					sg_copy_from_buffer(data->sg,  in sdhci_request_done()
3096 						data->sg_len,  in sdhci_request_done()
3097 						host->bounce_buffer,  in sdhci_request_done()
3102 						host->mmc->parent,  in sdhci_request_done()
3103 						host->bounce_addr,  in sdhci_request_done()
3104 						host->bounce_buffer_size,  in sdhci_request_done()
3109 				dma_unmap_sg(mmc_dev(host->mmc), data->sg,  in sdhci_request_done()
3110 					     data->sg_len,  in sdhci_request_done()
3113 			data->host_cookie = COOKIE_UNMAPPED;  in sdhci_request_done()
3117 	host->mrqs_done[i] = NULL;  in sdhci_request_done()
3119 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_request_done()
3121 	if (host->ops->request_done)  in sdhci_request_done()
3122 		host->ops->request_done(host, mrq);  in sdhci_request_done()
3124 		mmc_request_done(host->mmc, mrq);  in sdhci_request_done()
3145 	spin_lock_irqsave(&host->lock, flags);  in sdhci_timeout_timer()
3147 	if (host->cmd && !sdhci_data_line_cmd(host->cmd)) {  in sdhci_timeout_timer()
3148 		pr_err("%s: Timeout waiting for hardware cmd interrupt.\n",  in sdhci_timeout_timer()
3149 		       mmc_hostname(host->mmc));  in sdhci_timeout_timer()
3152 		host->cmd->error = -ETIMEDOUT;  in sdhci_timeout_timer()
3153 		sdhci_finish_mrq(host, host->cmd->mrq);  in sdhci_timeout_timer()
3156 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_timeout_timer()
3166 	spin_lock_irqsave(&host->lock, flags);  in sdhci_timeout_data_timer()
3168 	if (host->data || host->data_cmd ||  in sdhci_timeout_data_timer()
3169 	    (host->cmd && sdhci_data_line_cmd(host->cmd))) {  in sdhci_timeout_data_timer()
3171 		       mmc_hostname(host->mmc));  in sdhci_timeout_data_timer()
3174 		if (host->data) {  in sdhci_timeout_data_timer()
3175 			host->data->error = -ETIMEDOUT;  in sdhci_timeout_data_timer()
3177 			queue_work(host->complete_wq, &host->complete_work);  in sdhci_timeout_data_timer()
3178 		} else if (host->data_cmd) {  in sdhci_timeout_data_timer()
3179 			host->data_cmd->error = -ETIMEDOUT;  in sdhci_timeout_data_timer()
3180 			sdhci_finish_mrq(host, host->data_cmd->mrq);  in sdhci_timeout_data_timer()
3182 			host->cmd->error = -ETIMEDOUT;  in sdhci_timeout_data_timer()
3183 			sdhci_finish_mrq(host, host->cmd->mrq);  in sdhci_timeout_data_timer()
3187 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_timeout_data_timer()
3198 	/* Handle auto-CMD12 error */  in sdhci_cmd_irq()
3199 	if (intmask & SDHCI_INT_AUTO_CMD_ERR && host->data_cmd) {  in sdhci_cmd_irq()
3200 		struct mmc_request *mrq = host->data_cmd->mrq;  in sdhci_cmd_irq()
3206 		/* Treat auto-CMD12 error the same as data error */  in sdhci_cmd_irq()
3207 		if (!mrq->sbc && (host->flags & SDHCI_AUTO_CMD12)) {  in sdhci_cmd_irq()
3213 	if (!host->cmd) {  in sdhci_cmd_irq()
3215 		 * SDHCI recovers from errors by resetting the cmd and data  in sdhci_cmd_irq()
3219 		if (host->pending_reset)  in sdhci_cmd_irq()
3222 		       mmc_hostname(host->mmc), (unsigned)intmask);  in sdhci_cmd_irq()
3230 			host->cmd->error = -ETIMEDOUT;  in sdhci_cmd_irq()
3232 			host->cmd->error = -EILSEQ;  in sdhci_cmd_irq()
3235 		if (host->cmd->data &&  in sdhci_cmd_irq()
3238 			host->cmd = NULL;  in sdhci_cmd_irq()
3243 		__sdhci_finish_mrq(host, host->cmd->mrq);  in sdhci_cmd_irq()
3247 	/* Handle auto-CMD23 error */  in sdhci_cmd_irq()
3249 		struct mmc_request *mrq = host->cmd->mrq;  in sdhci_cmd_irq()
3252 			  -ETIMEDOUT :  in sdhci_cmd_irq()
3253 			  -EILSEQ;  in sdhci_cmd_irq()
3255 		if (mrq->sbc && (host->flags & SDHCI_AUTO_CMD23)) {  in sdhci_cmd_irq()
3256 			mrq->sbc->error = err;  in sdhci_cmd_irq()
3268 	void *desc = host->adma_table;  in sdhci_adma_show_error()
3269 	dma_addr_t dma = host->adma_addr;  in sdhci_adma_show_error()
3276 		if (host->flags & SDHCI_USE_64_BIT_DMA)  in sdhci_adma_show_error()
3279 			    le32_to_cpu(dma_desc->addr_hi),  in sdhci_adma_show_error()
3280 			    le32_to_cpu(dma_desc->addr_lo),  in sdhci_adma_show_error()
3281 			    le16_to_cpu(dma_desc->len),  in sdhci_adma_show_error()
3282 			    le16_to_cpu(dma_desc->cmd));  in sdhci_adma_show_error()
3286 			    le32_to_cpu(dma_desc->addr_lo),  in sdhci_adma_show_error()
3287 			    le16_to_cpu(dma_desc->len),  in sdhci_adma_show_error()
3288 			    le16_to_cpu(dma_desc->cmd));  in sdhci_adma_show_error()
3290 		desc += host->desc_sz;  in sdhci_adma_show_error()
3291 		dma += host->desc_sz;  in sdhci_adma_show_error()
3293 		if (dma_desc->cmd & cpu_to_le16(ADMA2_END))  in sdhci_adma_show_error()
3307 			host->tuning_done = 1;  in sdhci_data_irq()
3308 			wake_up(&host->buf_ready_int);  in sdhci_data_irq()
3313 	if (!host->data) {  in sdhci_data_irq()
3314 		struct mmc_command *data_cmd = host->data_cmd;  in sdhci_data_irq()
3321 		if (data_cmd && (data_cmd->flags & MMC_RSP_BUSY)) {  in sdhci_data_irq()
3323 				host->data_cmd = NULL;  in sdhci_data_irq()
3324 				data_cmd->error = -ETIMEDOUT;  in sdhci_data_irq()
3325 				__sdhci_finish_mrq(host, data_cmd->mrq);  in sdhci_data_irq()
3329 				host->data_cmd = NULL;  in sdhci_data_irq()
3331 				 * Some cards handle busy-end interrupt  in sdhci_data_irq()
3335 				if (host->cmd == data_cmd)  in sdhci_data_irq()
3338 				__sdhci_finish_mrq(host, data_cmd->mrq);  in sdhci_data_irq()
3344 		 * SDHCI recovers from errors by resetting the cmd and data  in sdhci_data_irq()
3348 		if (host->pending_reset)  in sdhci_data_irq()
3352 		       mmc_hostname(host->mmc), (unsigned)intmask);  in sdhci_data_irq()
3359 		host->data->error = -ETIMEDOUT;  in sdhci_data_irq()
3361 		host->data->error = -EILSEQ;  in sdhci_data_irq()
3365 		host->data->error = -EILSEQ;  in sdhci_data_irq()
3367 		pr_err("%s: ADMA error: 0x%08x\n", mmc_hostname(host->mmc),  in sdhci_data_irq()
3370 		host->data->error = -EIO;  in sdhci_data_irq()
3371 		if (host->ops->adma_workaround)  in sdhci_data_irq()
3372 			host->ops->adma_workaround(host, intmask);  in sdhci_data_irq()
3375 	if (host->data->error)  in sdhci_data_irq()
3394 			dmanow = dmastart + host->data->bytes_xfered;  in sdhci_data_irq()
3399 				~((dma_addr_t)SDHCI_DEFAULT_BOUNDARY_SIZE - 1)) +  in sdhci_data_irq()
3401 			host->data->bytes_xfered = dmanow - dmastart;  in sdhci_data_irq()
3403 			    &dmastart, host->data->bytes_xfered, &dmanow);  in sdhci_data_irq()
3408 			if (host->cmd == host->data_cmd) {  in sdhci_data_irq()
3414 				host->data_early = 1;  in sdhci_data_irq()
3425 	struct mmc_data *data = mrq->data;  in sdhci_defer_done()
3427 	return host->pending_reset || host->always_defer_done ||  in sdhci_defer_done()
3428 	       ((host->flags & SDHCI_REQ_USE_DMA) && data &&  in sdhci_defer_done()
3429 		data->host_cookie == COOKIE_MAPPED);  in sdhci_defer_done()
3441 	spin_lock(&host->lock);  in sdhci_irq()
3443 	if (host->runtime_suspended) {  in sdhci_irq()
3444 		spin_unlock(&host->lock);  in sdhci_irq()
3457 		if (host->ops->irq) {  in sdhci_irq()
3458 			intmask = host->ops->irq(host, intmask);  in sdhci_irq()
3483 			host->ier &= ~(SDHCI_INT_CARD_INSERT |  in sdhci_irq()
3485 			host->ier |= present ? SDHCI_INT_CARD_REMOVE :  in sdhci_irq()
3487 			sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);  in sdhci_irq()
3488 			sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);  in sdhci_irq()
3493 			host->thread_isr |= intmask & (SDHCI_INT_CARD_INSERT |  in sdhci_irq()
3506 				mmc_hostname(host->mmc));  in sdhci_irq()
3509 			mmc_retune_needed(host->mmc);  in sdhci_irq()
3512 		    (host->ier & SDHCI_INT_CARD_INT)) {  in sdhci_irq()
3514 			sdio_signal_irq(host->mmc);  in sdhci_irq()
3531 	} while (intmask && --max_loops);  in sdhci_irq()
3535 		struct mmc_request *mrq = host->mrqs_done[i];  in sdhci_irq()
3544 			host->mrqs_done[i] = NULL;  in sdhci_irq()
3548 	if (host->deferred_cmd)  in sdhci_irq()
3551 	spin_unlock(&host->lock);  in sdhci_irq()
3558 		if (host->ops->request_done)  in sdhci_irq()
3559 			host->ops->request_done(host, mrqs_done[i]);  in sdhci_irq()
3561 			mmc_request_done(host->mmc, mrqs_done[i]);  in sdhci_irq()
3566 			   mmc_hostname(host->mmc), unexpected);  in sdhci_irq()
3576 	struct mmc_command *cmd;  in sdhci_thread_irq()  local
3583 	spin_lock_irqsave(&host->lock, flags);  in sdhci_thread_irq()
3585 	isr = host->thread_isr;  in sdhci_thread_irq()
3586 	host->thread_isr = 0;  in sdhci_thread_irq()
3588 	cmd = host->deferred_cmd;  in sdhci_thread_irq()
3589 	if (cmd && !sdhci_send_command_retry(host, cmd, flags))  in sdhci_thread_irq()
3590 		sdhci_finish_mrq(host, cmd->mrq);  in sdhci_thread_irq()
3592 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_thread_irq()
3595 		struct mmc_host *mmc = host->mmc;  in sdhci_thread_irq()
3597 		mmc->ops->card_event(mmc);  in sdhci_thread_irq()
3614 	return mmc_card_is_removable(host->mmc) &&  in sdhci_cd_irq_can_wakeup()
3615 	       !(host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) &&  in sdhci_cd_irq_can_wakeup()
3616 	       !mmc_can_gpio_cd(host->mmc);  in sdhci_cd_irq_can_wakeup()
3621  * the Interrupt Status Enable register too. See 'Table 1-6: Wakeup Signal
3640 	if (mmc_card_wake_sdio_irq(host->mmc)) {  in sdhci_enable_irq_wakeups()
3655 	host->irq_wake_enabled = !enable_irq_wake(host->irq);  in sdhci_enable_irq_wakeups()
3657 	return host->irq_wake_enabled;  in sdhci_enable_irq_wakeups()
3670 	disable_irq_wake(host->irq);  in sdhci_disable_irq_wakeups()
3672 	host->irq_wake_enabled = false;  in sdhci_disable_irq_wakeups()
3679 	mmc_retune_timer_stop(host->mmc);  in sdhci_suspend_host()
3681 	if (!device_may_wakeup(mmc_dev(host->mmc)) ||  in sdhci_suspend_host()
3683 		host->ier = 0;  in sdhci_suspend_host()
3686 		free_irq(host->irq, host);  in sdhci_suspend_host()
3696 	struct mmc_host *mmc = host->mmc;  in sdhci_resume_host()
3699 	if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {  in sdhci_resume_host()
3700 		if (host->ops->enable_dma)  in sdhci_resume_host()
3701 			host->ops->enable_dma(host);  in sdhci_resume_host()
3704 	if ((host->mmc->pm_flags & MMC_PM_KEEP_POWER) &&  in sdhci_resume_host()
3705 	    (host->quirks2 & SDHCI_QUIRK2_HOST_OFF_CARD_ON)) {  in sdhci_resume_host()
3708 		host->pwr = 0;  in sdhci_resume_host()
3709 		host->clock = 0;  in sdhci_resume_host()
3710 		mmc->ops->set_ios(mmc, &mmc->ios);  in sdhci_resume_host()
3712 		sdhci_init(host, (host->mmc->pm_flags & MMC_PM_KEEP_POWER));  in sdhci_resume_host()
3715 	if (host->irq_wake_enabled) {  in sdhci_resume_host()
3718 		ret = request_threaded_irq(host->irq, sdhci_irq,  in sdhci_resume_host()
3720 					   mmc_hostname(host->mmc), host);  in sdhci_resume_host()
3736 	mmc_retune_timer_stop(host->mmc);  in sdhci_runtime_suspend_host()
3738 	spin_lock_irqsave(&host->lock, flags);  in sdhci_runtime_suspend_host()
3739 	host->ier &= SDHCI_INT_CARD_INT;  in sdhci_runtime_suspend_host()
3740 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);  in sdhci_runtime_suspend_host()
3741 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);  in sdhci_runtime_suspend_host()
3742 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_runtime_suspend_host()
3744 	synchronize_hardirq(host->irq);  in sdhci_runtime_suspend_host()
3746 	spin_lock_irqsave(&host->lock, flags);  in sdhci_runtime_suspend_host()
3747 	host->runtime_suspended = true;  in sdhci_runtime_suspend_host()
3748 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_runtime_suspend_host()
3756 	struct mmc_host *mmc = host->mmc;  in sdhci_runtime_resume_host()
3758 	int host_flags = host->flags;  in sdhci_runtime_resume_host()
3761 		if (host->ops->enable_dma)  in sdhci_runtime_resume_host()
3762 			host->ops->enable_dma(host);  in sdhci_runtime_resume_host()
3767 	if (mmc->ios.power_mode != MMC_POWER_UNDEFINED &&  in sdhci_runtime_resume_host()
3768 	    mmc->ios.power_mode != MMC_POWER_OFF) {  in sdhci_runtime_resume_host()
3769 		/* Force clock and power re-program */  in sdhci_runtime_resume_host()
3770 		host->pwr = 0;  in sdhci_runtime_resume_host()
3771 		host->clock = 0;  in sdhci_runtime_resume_host()
3772 		mmc->ops->start_signal_voltage_switch(mmc, &mmc->ios);  in sdhci_runtime_resume_host()
3773 		mmc->ops->set_ios(mmc, &mmc->ios);  in sdhci_runtime_resume_host()
3776 		    !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN)) {  in sdhci_runtime_resume_host()
3777 			spin_lock_irqsave(&host->lock, flags);  in sdhci_runtime_resume_host()
3779 			spin_unlock_irqrestore(&host->lock, flags);  in sdhci_runtime_resume_host()
3782 		if ((mmc->caps2 & MMC_CAP2_HS400_ES) &&  in sdhci_runtime_resume_host()
3783 		    mmc->ops->hs400_enhanced_strobe)  in sdhci_runtime_resume_host()
3784 			mmc->ops->hs400_enhanced_strobe(mmc, &mmc->ios);  in sdhci_runtime_resume_host()
3787 	spin_lock_irqsave(&host->lock, flags);  in sdhci_runtime_resume_host()
3789 	host->runtime_suspended = false;  in sdhci_runtime_resume_host()
3798 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_runtime_resume_host()
3818 	spin_lock_irqsave(&host->lock, flags);  in sdhci_cqe_enable()
3825 	 * for cmd queuing to fetch both command and transfer descriptors.  in sdhci_cqe_enable()
3827 	if (host->v4_mode && (host->caps1 & SDHCI_CAN_DO_ADMA3))  in sdhci_cqe_enable()
3829 	else if (host->flags & SDHCI_USE_64_BIT_DMA)  in sdhci_cqe_enable()
3835 	sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, 512),  in sdhci_cqe_enable()
3841 	host->ier = host->cqe_ier;  in sdhci_cqe_enable()
3843 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);  in sdhci_cqe_enable()
3844 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);  in sdhci_cqe_enable()
3846 	host->cqe_on = true;  in sdhci_cqe_enable()
3849 		 mmc_hostname(mmc), host->ier,  in sdhci_cqe_enable()
3852 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_cqe_enable()
3861 	spin_lock_irqsave(&host->lock, flags);  in sdhci_cqe_disable()
3865 	host->cqe_on = false;  in sdhci_cqe_disable()
3873 		 mmc_hostname(mmc), host->ier,  in sdhci_cqe_disable()
3876 	spin_unlock_irqrestore(&host->lock, flags);  in sdhci_cqe_disable()
3885 	if (!host->cqe_on)  in sdhci_cqe_irq()
3889 		*cmd_error = -EILSEQ;  in sdhci_cqe_irq()
3891 		*cmd_error = -ETIMEDOUT;  in sdhci_cqe_irq()
3896 		*data_error = -EILSEQ;  in sdhci_cqe_irq()
3898 		*data_error = -ETIMEDOUT;  in sdhci_cqe_irq()
3900 		*data_error = -EIO;  in sdhci_cqe_irq()
3905 	mask = intmask & host->cqe_ier;  in sdhci_cqe_irq()
3910 		       mmc_hostname(host->mmc));  in sdhci_cqe_irq()
3912 	intmask &= ~(host->cqe_ier | SDHCI_INT_ERROR);  in sdhci_cqe_irq()
3916 		       mmc_hostname(host->mmc), intmask);  in sdhci_cqe_irq()
3940 		return ERR_PTR(-ENOMEM);  in sdhci_alloc_host()
3943 	host->mmc = mmc;  in sdhci_alloc_host()
3944 	host->mmc_host_ops = sdhci_ops;  in sdhci_alloc_host()
3945 	mmc->ops = &host->mmc_host_ops;  in sdhci_alloc_host()
3947 	host->flags = SDHCI_SIGNALING_330;  in sdhci_alloc_host()
3949 	host->cqe_ier     = SDHCI_CQE_INT_MASK;  in sdhci_alloc_host()
3950 	host->cqe_err_ier = SDHCI_CQE_INT_ERR_MASK;  in sdhci_alloc_host()
3952 	host->tuning_delay = -1;  in sdhci_alloc_host()
3953 	host->tuning_loop_count = MAX_TUNING_LOOP;  in sdhci_alloc_host()
3955 	host->sdma_boundary = SDHCI_DEFAULT_BOUNDARY_ARG;  in sdhci_alloc_host()
3962 	host->adma_table_cnt = SDHCI_MAX_SEGS * 2 + 1;  in sdhci_alloc_host()
3963 	host->max_adma = 65536;  in sdhci_alloc_host()
3972 	struct mmc_host *mmc = host->mmc;  in sdhci_set_dma_mask()
3974 	int ret = -EINVAL;  in sdhci_set_dma_mask()
3976 	if (host->quirks2 & SDHCI_QUIRK2_BROKEN_64_BIT_DMA)  in sdhci_set_dma_mask()
3977 		host->flags &= ~SDHCI_USE_64_BIT_DMA;  in sdhci_set_dma_mask()
3979 	/* Try 64-bit mask if hardware is capable  of it */  in sdhci_set_dma_mask()
3980 	if (host->flags & SDHCI_USE_64_BIT_DMA) {  in sdhci_set_dma_mask()
3983 			pr_warn("%s: Failed to set 64-bit DMA mask.\n",  in sdhci_set_dma_mask()
3985 			host->flags &= ~SDHCI_USE_64_BIT_DMA;  in sdhci_set_dma_mask()
3989 	/* 32-bit mask as default & fallback */  in sdhci_set_dma_mask()
3993 			pr_warn("%s: Failed to set 32-bit DMA mask.\n",  in sdhci_set_dma_mask()
4007 	if (host->read_caps)  in __sdhci_read_caps()
4010 	host->read_caps = true;  in __sdhci_read_caps()
4013 		host->quirks = debug_quirks;  in __sdhci_read_caps()
4016 		host->quirks2 = debug_quirks2;  in __sdhci_read_caps()
4020 	if (host->v4_mode)  in __sdhci_read_caps()
4023 	device_property_read_u64_array(mmc_dev(host->mmc),  in __sdhci_read_caps()
4024 				       "sdhci-caps-mask", &dt_caps_mask, 1);  in __sdhci_read_caps()
4025 	device_property_read_u64_array(mmc_dev(host->mmc),  in __sdhci_read_caps()
4026 				       "sdhci-caps", &dt_caps, 1);  in __sdhci_read_caps()
4029 	host->version = (v & SDHCI_SPEC_VER_MASK) >> SDHCI_SPEC_VER_SHIFT;  in __sdhci_read_caps()
4031 	if (host->quirks & SDHCI_QUIRK_MISSING_CAPS)  in __sdhci_read_caps()
4035 		host->caps = *caps;  in __sdhci_read_caps()
4037 		host->caps = sdhci_readl(host, SDHCI_CAPABILITIES);  in __sdhci_read_caps()
4038 		host->caps &= ~lower_32_bits(dt_caps_mask);  in __sdhci_read_caps()
4039 		host->caps |= lower_32_bits(dt_caps);  in __sdhci_read_caps()
4042 	if (host->version < SDHCI_SPEC_300)  in __sdhci_read_caps()
4046 		host->caps1 = *caps1;  in __sdhci_read_caps()
4048 		host->caps1 = sdhci_readl(host, SDHCI_CAPABILITIES_1);  in __sdhci_read_caps()
4049 		host->caps1 &= ~upper_32_bits(dt_caps_mask);  in __sdhci_read_caps()
4050 		host->caps1 |= upper_32_bits(dt_caps);  in __sdhci_read_caps()
4057 	struct mmc_host *mmc = host->mmc;  in sdhci_allocate_bounce_buffer()
4073 	if (mmc->max_req_size < bounce_size)  in sdhci_allocate_bounce_buffer()
4074 		bounce_size = mmc->max_req_size;  in sdhci_allocate_bounce_buffer()
4082 	host->bounce_buffer = devm_kmalloc(mmc->parent,  in sdhci_allocate_bounce_buffer()
4085 	if (!host->bounce_buffer) {  in sdhci_allocate_bounce_buffer()
4091 		 * mmc->max_segs == 1.  in sdhci_allocate_bounce_buffer()
4096 	host->bounce_addr = dma_map_single(mmc->parent,  in sdhci_allocate_bounce_buffer()
4097 					   host->bounce_buffer,  in sdhci_allocate_bounce_buffer()
4100 	ret = dma_mapping_error(mmc->parent, host->bounce_addr);  in sdhci_allocate_bounce_buffer()
4104 	host->bounce_buffer_size = bounce_size;  in sdhci_allocate_bounce_buffer()
4107 	mmc->max_segs = max_blocks;  in sdhci_allocate_bounce_buffer()
4108 	mmc->max_seg_size = bounce_size;  in sdhci_allocate_bounce_buffer()
4109 	mmc->max_req_size = bounce_size;  in sdhci_allocate_bounce_buffer()
4119 	 * version 4.10 in Capabilities Register is used as 64-bit System  in sdhci_can_64bit_dma()
4122 	if (host->version >= SDHCI_SPEC_410 && host->v4_mode)  in sdhci_can_64bit_dma()
4123 		return host->caps & SDHCI_CAN_64BIT_V4;  in sdhci_can_64bit_dma()
4125 	return host->caps & SDHCI_CAN_64BIT;  in sdhci_can_64bit_dma()
4140 		return -EINVAL;  in sdhci_setup_host()
4142 	mmc = host->mmc;  in sdhci_setup_host()
4150 	if (!mmc->supply.vqmmc) {  in sdhci_setup_host()
4166 	override_timeout_clk = host->timeout_clk;  in sdhci_setup_host()
4168 	if (host->version > SDHCI_SPEC_420) {  in sdhci_setup_host()
4170 		       mmc_hostname(mmc), host->version);  in sdhci_setup_host()
4173 	if (host->quirks & SDHCI_QUIRK_FORCE_DMA)  in sdhci_setup_host()
4174 		host->flags |= SDHCI_USE_SDMA;  in sdhci_setup_host()
4175 	else if (!(host->caps & SDHCI_CAN_DO_SDMA))  in sdhci_setup_host()
4178 		host->flags |= SDHCI_USE_SDMA;  in sdhci_setup_host()
4180 	if ((host->quirks & SDHCI_QUIRK_BROKEN_DMA) &&  in sdhci_setup_host()
4181 		(host->flags & SDHCI_USE_SDMA)) {  in sdhci_setup_host()
4183 		host->flags &= ~SDHCI_USE_SDMA;  in sdhci_setup_host()
4186 	if ((host->version >= SDHCI_SPEC_200) &&  in sdhci_setup_host()
4187 		(host->caps & SDHCI_CAN_DO_ADMA2))  in sdhci_setup_host()
4188 		host->flags |= SDHCI_USE_ADMA;  in sdhci_setup_host()
4190 	if ((host->quirks & SDHCI_QUIRK_BROKEN_ADMA) &&  in sdhci_setup_host()
4191 		(host->flags & SDHCI_USE_ADMA)) {  in sdhci_setup_host()
4193 		host->flags &= ~SDHCI_USE_ADMA;  in sdhci_setup_host()
4197 		host->flags |= SDHCI_USE_64_BIT_DMA;  in sdhci_setup_host()
4199 	if (host->use_external_dma) {  in sdhci_setup_host()
4201 		if (ret == -EPROBE_DEFER)  in sdhci_setup_host()
4211 			host->flags &= ~(SDHCI_USE_SDMA | SDHCI_USE_ADMA);  in sdhci_setup_host()
4214 	if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {  in sdhci_setup_host()
4215 		if (host->ops->set_dma_mask)  in sdhci_setup_host()
4216 			ret = host->ops->set_dma_mask(host);  in sdhci_setup_host()
4220 		if (!ret && host->ops->enable_dma)  in sdhci_setup_host()
4221 			ret = host->ops->enable_dma(host);  in sdhci_setup_host()
4224 			pr_warn("%s: No suitable DMA available - falling back to PIO\n",  in sdhci_setup_host()
4226 			host->flags &= ~(SDHCI_USE_SDMA | SDHCI_USE_ADMA);  in sdhci_setup_host()
4232 	/* SDMA does not support 64-bit DMA if v4 mode not set */  in sdhci_setup_host()
4233 	if ((host->flags & SDHCI_USE_64_BIT_DMA) && !host->v4_mode)  in sdhci_setup_host()
4234 		host->flags &= ~SDHCI_USE_SDMA;  in sdhci_setup_host()
4236 	if (host->flags & SDHCI_USE_ADMA) {  in sdhci_setup_host()
4240 		if (!(host->flags & SDHCI_USE_64_BIT_DMA))  in sdhci_setup_host()
4241 			host->alloc_desc_sz = SDHCI_ADMA2_32_DESC_SZ;  in sdhci_setup_host()
4242 		else if (!host->alloc_desc_sz)  in sdhci_setup_host()
4243 			host->alloc_desc_sz = SDHCI_ADMA2_64_DESC_SZ(host);  in sdhci_setup_host()
4245 		host->desc_sz = host->alloc_desc_sz;  in sdhci_setup_host()
4246 		host->adma_table_sz = host->adma_table_cnt * host->desc_sz;  in sdhci_setup_host()
4248 		host->align_buffer_sz = SDHCI_MAX_SEGS * SDHCI_ADMA2_ALIGN;  in sdhci_setup_host()
4250 		 * Use zalloc to zero the reserved high 32-bits of 128-bit  in sdhci_setup_host()
4254 					 host->align_buffer_sz + host->adma_table_sz,  in sdhci_setup_host()
4257 			pr_warn("%s: Unable to allocate ADMA buffers - falling back to standard DMA\n",  in sdhci_setup_host()
4259 			host->flags &= ~SDHCI_USE_ADMA;  in sdhci_setup_host()
4260 		} else if ((dma + host->align_buffer_sz) &  in sdhci_setup_host()
4261 			   (SDHCI_ADMA2_DESC_ALIGN - 1)) {  in sdhci_setup_host()
4264 			host->flags &= ~SDHCI_USE_ADMA;  in sdhci_setup_host()
4265 			dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +  in sdhci_setup_host()
4266 					  host->adma_table_sz, buf, dma);  in sdhci_setup_host()
4268 			host->align_buffer = buf;  in sdhci_setup_host()
4269 			host->align_addr = dma;  in sdhci_setup_host()
4271 			host->adma_table = buf + host->align_buffer_sz;  in sdhci_setup_host()
4272 			host->adma_addr = dma + host->align_buffer_sz;  in sdhci_setup_host()
4281 	if (!(host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA))) {  in sdhci_setup_host()
4282 		host->dma_mask = DMA_BIT_MASK(64);  in sdhci_setup_host()
4283 		mmc_dev(mmc)->dma_mask = &host->dma_mask;  in sdhci_setup_host()
4286 	if (host->version >= SDHCI_SPEC_300)  in sdhci_setup_host()
4287 		host->max_clk = FIELD_GET(SDHCI_CLOCK_V3_BASE_MASK, host->caps);  in sdhci_setup_host()
4289 		host->max_clk = FIELD_GET(SDHCI_CLOCK_BASE_MASK, host->caps);  in sdhci_setup_host()
4291 	host->max_clk *= 1000000;  in sdhci_setup_host()
4292 	if (host->max_clk == 0 || host->quirks &  in sdhci_setup_host()
4294 		if (!host->ops->get_max_clock) {  in sdhci_setup_host()
4297 			ret = -ENODEV;  in sdhci_setup_host()
4300 		host->max_clk = host->ops->get_max_clock(host);  in sdhci_setup_host()
4307 	host->clk_mul = FIELD_GET(SDHCI_CLOCK_MUL_MASK, host->caps1);  in sdhci_setup_host()
4315 	if (host->clk_mul)  in sdhci_setup_host()
4316 		host->clk_mul += 1;  in sdhci_setup_host()
4321 	max_clk = host->max_clk;  in sdhci_setup_host()
4323 	if (host->ops->get_min_clock)  in sdhci_setup_host()
4324 		mmc->f_min = host->ops->get_min_clock(host);  in sdhci_setup_host()
4325 	else if (host->version >= SDHCI_SPEC_300) {  in sdhci_setup_host()
4326 		if (host->clk_mul)  in sdhci_setup_host()
4327 			max_clk = host->max_clk * host->clk_mul;  in sdhci_setup_host()
4332 		mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_300;  in sdhci_setup_host()
4334 		mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_200;  in sdhci_setup_host()
4336 	if (!mmc->f_max || mmc->f_max > max_clk)  in sdhci_setup_host()
4337 		mmc->f_max = max_clk;  in sdhci_setup_host()
4339 	if (!(host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK)) {  in sdhci_setup_host()
4340 		host->timeout_clk = FIELD_GET(SDHCI_TIMEOUT_CLK_MASK, host->caps);  in sdhci_setup_host()
4342 		if (host->caps & SDHCI_TIMEOUT_CLK_UNIT)  in sdhci_setup_host()
4343 			host->timeout_clk *= 1000;  in sdhci_setup_host()
4345 		if (host->timeout_clk == 0) {  in sdhci_setup_host()
4346 			if (!host->ops->get_timeout_clock) {  in sdhci_setup_host()
4349 				ret = -ENODEV;  in sdhci_setup_host()
4353 			host->timeout_clk =  in sdhci_setup_host()
4354 				DIV_ROUND_UP(host->ops->get_timeout_clock(host),  in sdhci_setup_host()
4359 			host->timeout_clk = override_timeout_clk;  in sdhci_setup_host()
4361 		mmc->max_busy_timeout = host->ops->get_max_timeout_count ?  in sdhci_setup_host()
4362 			host->ops->get_max_timeout_count(host) : 1 << 27;  in sdhci_setup_host()
4363 		mmc->max_busy_timeout /= host->timeout_clk;  in sdhci_setup_host()
4366 	if (host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT &&  in sdhci_setup_host()
4367 	    !host->ops->get_max_timeout_count)  in sdhci_setup_host()
4368 		mmc->max_busy_timeout = 0;  in sdhci_setup_host()
4370 	mmc->caps |= MMC_CAP_SDIO_IRQ | MMC_CAP_CMD23;  in sdhci_setup_host()
4371 	mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;  in sdhci_setup_host()
4373 	if (host->quirks & SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12)  in sdhci_setup_host()
4374 		host->flags |= SDHCI_AUTO_CMD12;  in sdhci_setup_host()
4377 	 * For v3 mode, Auto-CMD23 stuff only works in ADMA or PIO.  in sdhci_setup_host()
4378 	 * For v4 mode, SDMA may use Auto-CMD23 as well.  in sdhci_setup_host()
4380 	if ((host->version >= SDHCI_SPEC_300) &&  in sdhci_setup_host()
4381 	    ((host->flags & SDHCI_USE_ADMA) ||  in sdhci_setup_host()
4382 	     !(host->flags & SDHCI_USE_SDMA) || host->v4_mode) &&  in sdhci_setup_host()
4383 	     !(host->quirks2 & SDHCI_QUIRK2_ACMD23_BROKEN)) {  in sdhci_setup_host()
4384 		host->flags |= SDHCI_AUTO_CMD23;  in sdhci_setup_host()
4385 		DBG("Auto-CMD23 available\n");  in sdhci_setup_host()
4387 		DBG("Auto-CMD23 unavailable\n");  in sdhci_setup_host()
4391 	 * A controller may support 8-bit width, but the board itself  in sdhci_setup_host()
4392 	 * might not have the pins brought out.  Boards that support  in sdhci_setup_host()
4393 	 * 8-bit width must set "mmc->caps |= MMC_CAP_8_BIT_DATA;" in  in sdhci_setup_host()
4395 	 * won't assume 8-bit width for hosts without that CAP.  in sdhci_setup_host()
4397 	if (!(host->quirks & SDHCI_QUIRK_FORCE_1_BIT_DATA))  in sdhci_setup_host()
4398 		mmc->caps |= MMC_CAP_4_BIT_DATA;  in sdhci_setup_host()
4400 	if (host->quirks2 & SDHCI_QUIRK2_HOST_NO_CMD23)  in sdhci_setup_host()
4401 		mmc->caps &= ~MMC_CAP_CMD23;  in sdhci_setup_host()
4403 	if (host->caps & SDHCI_CAN_DO_HISPD)  in sdhci_setup_host()
4404 		mmc->caps |= MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED;  in sdhci_setup_host()
4406 	if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) &&  in sdhci_setup_host()
4408 	    mmc_gpio_get_cd(host->mmc) < 0)  in sdhci_setup_host()
4409 		mmc->caps |= MMC_CAP_NEEDS_POLL;  in sdhci_setup_host()
4411 	if (!IS_ERR(mmc->supply.vqmmc)) {  in sdhci_setup_host()
4413 			ret = regulator_enable(mmc->supply.vqmmc);  in sdhci_setup_host()
4414 			host->sdhci_core_to_disable_vqmmc = !ret;  in sdhci_setup_host()
4418 		if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 1700000,  in sdhci_setup_host()
4420 			host->caps1 &= ~(SDHCI_SUPPORT_SDR104 |  in sdhci_setup_host()
4425 		if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 2700000,  in sdhci_setup_host()
4427 			host->flags &= ~SDHCI_SIGNALING_330;  in sdhci_setup_host()
4432 			mmc->supply.vqmmc = ERR_PTR(-EINVAL);  in sdhci_setup_host()
4437 	if (host->quirks2 & SDHCI_QUIRK2_NO_1_8_V) {  in sdhci_setup_host()
4438 		host->caps1 &= ~(SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |  in sdhci_setup_host()
4442 		 * (indicated using mmc-hs200-1_8v/mmc-hs400-1_8v dt property),  in sdhci_setup_host()
4448 		mmc->caps2 &= ~(MMC_CAP2_HSX00_1_8V | MMC_CAP2_HS400_ES);  in sdhci_setup_host()
4449 		mmc->caps &= ~(MMC_CAP_1_8V_DDR | MMC_CAP_UHS);  in sdhci_setup_host()
4452 	/* Any UHS-I mode in caps implies SDR12 and SDR25 support. */  in sdhci_setup_host()
4453 	if (host->caps1 & (SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |  in sdhci_setup_host()
4455 		mmc->caps |= MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;  in sdhci_setup_host()
4458 	if (host->caps1 & SDHCI_SUPPORT_SDR104) {  in sdhci_setup_host()
4459 		mmc->caps |= MMC_CAP_UHS_SDR104 | MMC_CAP_UHS_SDR50;  in sdhci_setup_host()
4463 		if (!(host->quirks2 & SDHCI_QUIRK2_BROKEN_HS200))  in sdhci_setup_host()
4464 			mmc->caps2 |= MMC_CAP2_HS200;  in sdhci_setup_host()
4465 	} else if (host->caps1 & SDHCI_SUPPORT_SDR50) {  in sdhci_setup_host()
4466 		mmc->caps |= MMC_CAP_UHS_SDR50;  in sdhci_setup_host()
4469 	if (host->quirks2 & SDHCI_QUIRK2_CAPS_BIT63_FOR_HS400 &&  in sdhci_setup_host()
4470 	    (host->caps1 & SDHCI_SUPPORT_HS400))  in sdhci_setup_host()
4471 		mmc->caps2 |= MMC_CAP2_HS400;  in sdhci_setup_host()
4473 	if ((mmc->caps2 & MMC_CAP2_HSX00_1_2V) &&  in sdhci_setup_host()
4474 	    (IS_ERR(mmc->supply.vqmmc) ||  in sdhci_setup_host()
4475 	     !regulator_is_supported_voltage(mmc->supply.vqmmc, 1100000,  in sdhci_setup_host()
4477 		mmc->caps2 &= ~MMC_CAP2_HSX00_1_2V;  in sdhci_setup_host()
4479 	if ((host->caps1 & SDHCI_SUPPORT_DDR50) &&  in sdhci_setup_host()
4480 	    !(host->quirks2 & SDHCI_QUIRK2_BROKEN_DDR50))  in sdhci_setup_host()
4481 		mmc->caps |= MMC_CAP_UHS_DDR50;  in sdhci_setup_host()
4484 	if (host->caps1 & SDHCI_USE_SDR50_TUNING)  in sdhci_setup_host()
4485 		host->flags |= SDHCI_SDR50_NEEDS_TUNING;  in sdhci_setup_host()
4488 	if (host->caps1 & SDHCI_DRIVER_TYPE_A)  in sdhci_setup_host()
4489 		mmc->caps |= MMC_CAP_DRIVER_TYPE_A;  in sdhci_setup_host()
4490 	if (host->caps1 & SDHCI_DRIVER_TYPE_C)  in sdhci_setup_host()
4491 		mmc->caps |= MMC_CAP_DRIVER_TYPE_C;  in sdhci_setup_host()
4492 	if (host->caps1 & SDHCI_DRIVER_TYPE_D)  in sdhci_setup_host()
4493 		mmc->caps |= MMC_CAP_DRIVER_TYPE_D;  in sdhci_setup_host()
4495 	/* Initial value for re-tuning timer count */  in sdhci_setup_host()
4496 	host->tuning_count = FIELD_GET(SDHCI_RETUNING_TIMER_COUNT_MASK,  in sdhci_setup_host()
4497 				       host->caps1);  in sdhci_setup_host()
4500 	 * In case Re-tuning Timer is not disabled, the actual value of  in sdhci_setup_host()
4501 	 * re-tuning timer will be 2 ^ (n - 1).  in sdhci_setup_host()
4503 	if (host->tuning_count)  in sdhci_setup_host()
4504 		host->tuning_count = 1 << (host->tuning_count - 1);  in sdhci_setup_host()
4506 	/* Re-tuning mode supported by the Host Controller */  in sdhci_setup_host()
4507 	host->tuning_mode = FIELD_GET(SDHCI_RETUNING_MODE_MASK, host->caps1);  in sdhci_setup_host()
4519 	if (!max_current_caps && !IS_ERR(mmc->supply.vmmc)) {  in sdhci_setup_host()
4520 		int curr = regulator_get_current_limit(mmc->supply.vmmc);  in sdhci_setup_host()
4535 	if (host->caps & SDHCI_CAN_VDD_330) {  in sdhci_setup_host()
4538 		mmc->max_current_330 = FIELD_GET(SDHCI_MAX_CURRENT_330_MASK,  in sdhci_setup_host()
4542 	if (host->caps & SDHCI_CAN_VDD_300) {  in sdhci_setup_host()
4545 		mmc->max_current_300 = FIELD_GET(SDHCI_MAX_CURRENT_300_MASK,  in sdhci_setup_host()
4549 	if (host->caps & SDHCI_CAN_VDD_180) {  in sdhci_setup_host()
4552 		mmc->max_current_180 = FIELD_GET(SDHCI_MAX_CURRENT_180_MASK,  in sdhci_setup_host()
4558 	if (host->ocr_mask)  in sdhci_setup_host()
4559 		ocr_avail = host->ocr_mask;  in sdhci_setup_host()
4562 	if (mmc->ocr_avail)  in sdhci_setup_host()
4563 		ocr_avail = mmc->ocr_avail;  in sdhci_setup_host()
4565 	mmc->ocr_avail = ocr_avail;  in sdhci_setup_host()
4566 	mmc->ocr_avail_sdio = ocr_avail;  in sdhci_setup_host()
4567 	if (host->ocr_avail_sdio)  in sdhci_setup_host()
4568 		mmc->ocr_avail_sdio &= host->ocr_avail_sdio;  in sdhci_setup_host()
4569 	mmc->ocr_avail_sd = ocr_avail;  in sdhci_setup_host()
4570 	if (host->ocr_avail_sd)  in sdhci_setup_host()
4571 		mmc->ocr_avail_sd &= host->ocr_avail_sd;  in sdhci_setup_host()
4573 		mmc->ocr_avail_sd &= ~MMC_VDD_165_195;  in sdhci_setup_host()
4574 	mmc->ocr_avail_mmc = ocr_avail;  in sdhci_setup_host()
4575 	if (host->ocr_avail_mmc)  in sdhci_setup_host()
4576 		mmc->ocr_avail_mmc &= host->ocr_avail_mmc;  in sdhci_setup_host()
4578 	if (mmc->ocr_avail == 0) {  in sdhci_setup_host()
4581 		ret = -ENODEV;  in sdhci_setup_host()
4585 	if ((mmc->caps & (MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 |  in sdhci_setup_host()
4588 	    (mmc->caps2 & (MMC_CAP2_HS200_1_8V_SDR | MMC_CAP2_HS400_1_8V)))  in sdhci_setup_host()
4589 		host->flags |= SDHCI_SIGNALING_180;  in sdhci_setup_host()
4591 	if (mmc->caps2 & MMC_CAP2_HSX00_1_2V)  in sdhci_setup_host()
4592 		host->flags |= SDHCI_SIGNALING_120;  in sdhci_setup_host()
4594 	spin_lock_init(&host->lock);  in sdhci_setup_host()
4601 	mmc->max_req_size = 524288;  in sdhci_setup_host()
4607 	if (host->flags & SDHCI_USE_ADMA) {  in sdhci_setup_host()
4608 		mmc->max_segs = SDHCI_MAX_SEGS;  in sdhci_setup_host()
4609 	} else if (host->flags & SDHCI_USE_SDMA) {  in sdhci_setup_host()
4610 		mmc->max_segs = 1;  in sdhci_setup_host()
4614 			mmc->max_req_size = min(mmc->max_req_size,  in sdhci_setup_host()
4618 		mmc->max_segs = SDHCI_MAX_SEGS;  in sdhci_setup_host()
4626 	if (host->flags & SDHCI_USE_ADMA) {  in sdhci_setup_host()
4627 		if (host->quirks & SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC) {  in sdhci_setup_host()
4628 			host->max_adma = 65532; /* 32-bit alignment */  in sdhci_setup_host()
4629 			mmc->max_seg_size = 65535;  in sdhci_setup_host()
4631 			mmc->max_seg_size = 65536;  in sdhci_setup_host()
4634 		mmc->max_seg_size = mmc->max_req_size;  in sdhci_setup_host()
4641 	if (host->quirks & SDHCI_QUIRK_FORCE_BLK_SZ_2048) {  in sdhci_setup_host()
4642 		mmc->max_blk_size = 2;  in sdhci_setup_host()
4644 		mmc->max_blk_size = (host->caps & SDHCI_MAX_BLOCK_MASK) >>  in sdhci_setup_host()
4646 		if (mmc->max_blk_size >= 3) {  in sdhci_setup_host()
4649 			mmc->max_blk_size = 0;  in sdhci_setup_host()
4653 	mmc->max_blk_size = 512 << mmc->max_blk_size;  in sdhci_setup_host()
4658 	mmc->max_blk_count = (host->quirks & SDHCI_QUIRK_NO_MULTIBLOCK) ? 1 : 65535;  in sdhci_setup_host()
4660 	if (mmc->max_segs == 1)  in sdhci_setup_host()
4661 		/* This may alter mmc->*_blk_* parameters */  in sdhci_setup_host()
4667 	if (host->sdhci_core_to_disable_vqmmc)  in sdhci_setup_host()
4668 		regulator_disable(mmc->supply.vqmmc);  in sdhci_setup_host()
4670 	if (host->align_buffer)  in sdhci_setup_host()
4671 		dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +  in sdhci_setup_host()
4672 				  host->adma_table_sz, host->align_buffer,  in sdhci_setup_host()
4673 				  host->align_addr);  in sdhci_setup_host()
4674 	host->adma_table = NULL;  in sdhci_setup_host()
4675 	host->align_buffer = NULL;  in sdhci_setup_host()
4683 	struct mmc_host *mmc = host->mmc;  in sdhci_cleanup_host()
4685 	if (host->sdhci_core_to_disable_vqmmc)  in sdhci_cleanup_host()
4686 		regulator_disable(mmc->supply.vqmmc);  in sdhci_cleanup_host()
4688 	if (host->align_buffer)  in sdhci_cleanup_host()
4689 		dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +  in sdhci_cleanup_host()
4690 				  host->adma_table_sz, host->align_buffer,  in sdhci_cleanup_host()
4691 				  host->align_addr);  in sdhci_cleanup_host()
4693 	if (host->use_external_dma)  in sdhci_cleanup_host()
4696 	host->adma_table = NULL;  in sdhci_cleanup_host()
4697 	host->align_buffer = NULL;  in sdhci_cleanup_host()
4704 	struct mmc_host *mmc = host->mmc;  in __sdhci_add_host()
4707 	if ((mmc->caps2 & MMC_CAP2_CQE) &&  in __sdhci_add_host()
4708 	    (host->quirks & SDHCI_QUIRK_BROKEN_CQE)) {  in __sdhci_add_host()
4709 		mmc->caps2 &= ~MMC_CAP2_CQE;  in __sdhci_add_host()
4710 		mmc->cqe_ops = NULL;  in __sdhci_add_host()
4713 	host->complete_wq = alloc_workqueue("sdhci", flags, 0);  in __sdhci_add_host()
4714 	if (!host->complete_wq)  in __sdhci_add_host()
4715 		return -ENOMEM;  in __sdhci_add_host()
4717 	INIT_WORK(&host->complete_work, sdhci_complete_work);  in __sdhci_add_host()
4719 	timer_setup(&host->timer, sdhci_timeout_timer, 0);  in __sdhci_add_host()
4720 	timer_setup(&host->data_timer, sdhci_timeout_data_timer, 0);  in __sdhci_add_host()
4722 	init_waitqueue_head(&host->buf_ready_int);  in __sdhci_add_host()
4726 	ret = request_threaded_irq(host->irq, sdhci_irq, sdhci_thread_irq,  in __sdhci_add_host()
4730 		       mmc_hostname(mmc), host->irq, ret);  in __sdhci_add_host()
4746 		mmc_hostname(mmc), host->hw_name, dev_name(mmc_dev(mmc)),  in __sdhci_add_host()
4747 		host->use_external_dma ? "External DMA" :  in __sdhci_add_host()
4748 		(host->flags & SDHCI_USE_ADMA) ?  in __sdhci_add_host()
4749 		(host->flags & SDHCI_USE_64_BIT_DMA) ? "ADMA 64-bit" : "ADMA" :  in __sdhci_add_host()
4750 		(host->flags & SDHCI_USE_SDMA) ? "DMA" : "PIO");  in __sdhci_add_host()
4762 	free_irq(host->irq, host);  in __sdhci_add_host()
4764 	destroy_workqueue(host->complete_wq);  in __sdhci_add_host()
4793 	struct mmc_host *mmc = host->mmc;  in sdhci_remove_host()
4797 		spin_lock_irqsave(&host->lock, flags);  in sdhci_remove_host()
4799 		host->flags |= SDHCI_DEVICE_DEAD;  in sdhci_remove_host()
4804 			sdhci_error_out_mrqs(host, -ENOMEDIUM);  in sdhci_remove_host()
4807 		spin_unlock_irqrestore(&host->lock, flags);  in sdhci_remove_host()
4821 	free_irq(host->irq, host);  in sdhci_remove_host()
4823 	del_timer_sync(&host->timer);  in sdhci_remove_host()
4824 	del_timer_sync(&host->data_timer);  in sdhci_remove_host()
4826 	destroy_workqueue(host->complete_wq);  in sdhci_remove_host()
4828 	if (host->sdhci_core_to_disable_vqmmc)  in sdhci_remove_host()
4829 		regulator_disable(mmc->supply.vqmmc);  in sdhci_remove_host()
4831 	if (host->align_buffer)  in sdhci_remove_host()
4832 		dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +  in sdhci_remove_host()
4833 				  host->adma_table_sz, host->align_buffer,  in sdhci_remove_host()
4834 				  host->align_addr);  in sdhci_remove_host()
4836 	if (host->use_external_dma)  in sdhci_remove_host()
4839 	host->adma_table = NULL;  in sdhci_remove_host()
4840 	host->align_buffer = NULL;  in sdhci_remove_host()
4847 	mmc_free_host(host->mmc);  in sdhci_free_host()