237 	mutex_lock(&padctl->lock);  in tegra124_xusb_padctl_enable()
239 	if (padctl->enable++ > 0)  in tegra124_xusb_padctl_enable()
259 	mutex_unlock(&padctl->lock);  in tegra124_xusb_padctl_enable()
267 	mutex_lock(&padctl->lock);  in tegra124_xusb_padctl_disable()
269 	if (WARN_ON(padctl->enable == 0))  in tegra124_xusb_padctl_disable()
272 	if (--padctl->enable > 0)  in tegra124_xusb_padctl_disable()
292 	mutex_unlock(&padctl->lock);  in tegra124_xusb_padctl_disable()
299 	struct tegra_xusb_usb3_port *port;  in tegra124_usb3_save_context()  local
303 	port = tegra_xusb_find_usb3_port(padctl, index);  in tegra124_usb3_save_context()
304 	if (!port)  in tegra124_usb3_save_context()
305 		return -ENODEV;  in tegra124_usb3_save_context()
307 	port->context_saved = true;  in tegra124_usb3_save_context()
308 	lane = port->base.lane;  in tegra124_usb3_save_context()
310 	if (lane->pad == padctl->pcie)  in tegra124_usb3_save_context()
311 		offset = XUSB_PADCTL_IOPHY_MISC_PAD_PX_CTL6(lane->index);  in tegra124_usb3_save_context()
324 	port->tap1 = value & XUSB_PADCTL_IOPHY_MISC_PAD_CTL6_MISC_OUT_TAP_MASK;  in tegra124_usb3_save_context()
335 	port->amp = value & XUSB_PADCTL_IOPHY_MISC_PAD_CTL6_MISC_OUT_AMP_MASK;  in tegra124_usb3_save_context()
342 	value |= (port->tap1 <<  in tegra124_usb3_save_context()
344 		 (port->amp <<  in tegra124_usb3_save_context()
364 	port->ctle_g = value &  in tegra124_usb3_save_context()
376 	port->ctle_z = value &  in tegra124_usb3_save_context()
384 	value |= (port->ctle_g <<  in tegra124_usb3_save_context()
386 		 (port->ctle_z <<  in tegra124_usb3_save_context()
429 	TEGRA124_LANE("usb2-0", 0x004,  0, 0x3, usb2),
430 	TEGRA124_LANE("usb2-1", 0x004,  2, 0x3, usb2),
431 	TEGRA124_LANE("usb2-2", 0x004,  4, 0x3, usb2),
443 		return ERR_PTR(-ENOMEM);  in tegra124_usb2_lane_probe()
445 	INIT_LIST_HEAD(&usb2->base.list);  in tegra124_usb2_lane_probe()
446 	usb2->base.soc = &pad->soc->lanes[index];  in tegra124_usb2_lane_probe()
447 	usb2->base.index = index;  in tegra124_usb2_lane_probe()
448 	usb2->base.pad = pad;  in tegra124_usb2_lane_probe()
449 	usb2->base.np = np;  in tegra124_usb2_lane_probe()
451 	err = tegra_xusb_lane_parse_dt(&usb2->base, np);  in tegra124_usb2_lane_probe()
457 	return &usb2->base;  in tegra124_usb2_lane_probe()
476 	return tegra124_xusb_padctl_enable(lane->pad->padctl);  in tegra124_usb2_phy_init()
483 	return tegra124_xusb_padctl_disable(lane->pad->padctl);  in tegra124_usb2_phy_exit()
490 	struct tegra_xusb_usb2_pad *pad = to_usb2_pad(lane->pad);  in tegra124_usb2_phy_power_on()
491 	struct tegra_xusb_padctl *padctl = lane->pad->padctl;  in tegra124_usb2_phy_power_on()
493 	struct tegra_xusb_usb2_port *port;  in tegra124_usb2_phy_power_on()  local
494 	unsigned int index = lane->index;  in tegra124_usb2_phy_power_on()
498 	port = tegra_xusb_find_usb2_port(padctl, index);  in tegra124_usb2_phy_power_on()
499 	if (!port) {  in tegra124_usb2_phy_power_on()
500 		dev_err(&phy->dev, "no port found for USB2 lane %u\n", index);  in tegra124_usb2_phy_power_on()
501 		return -ENODEV;  in tegra124_usb2_phy_power_on()
511 	value |= (priv->fuse.hs_squelch_level <<  in tegra124_usb2_phy_power_on()
534 	value |= (priv->fuse.hs_curr_level[index] +  in tegra124_usb2_phy_power_on()
535 		  usb2->hs_curr_level_offset) <<  in tegra124_usb2_phy_power_on()
551 	value |= (priv->fuse.hs_term_range_adj <<  in tegra124_usb2_phy_power_on()
553 		 (priv->fuse.hs_iref_cap <<  in tegra124_usb2_phy_power_on()
557 	err = regulator_enable(port->supply);  in tegra124_usb2_phy_power_on()
561 	mutex_lock(&pad->lock);  in tegra124_usb2_phy_power_on()
563 	if (pad->enable++ > 0)  in tegra124_usb2_phy_power_on()
571 	mutex_unlock(&pad->lock);  in tegra124_usb2_phy_power_on()
578 	struct tegra_xusb_usb2_pad *pad = to_usb2_pad(lane->pad);  in tegra124_usb2_phy_power_off()
579 	struct tegra_xusb_padctl *padctl = lane->pad->padctl;  in tegra124_usb2_phy_power_off()
580 	struct tegra_xusb_usb2_port *port;  in tegra124_usb2_phy_power_off()  local
583 	port = tegra_xusb_find_usb2_port(padctl, lane->index);  in tegra124_usb2_phy_power_off()
584 	if (!port) {  in tegra124_usb2_phy_power_off()
585 		dev_err(&phy->dev, "no port found for USB2 lane %u\n",  in tegra124_usb2_phy_power_off()
586 			lane->index);  in tegra124_usb2_phy_power_off()
587 		return -ENODEV;  in tegra124_usb2_phy_power_off()
590 	mutex_lock(&pad->lock);  in tegra124_usb2_phy_power_off()
592 	if (WARN_ON(pad->enable == 0))  in tegra124_usb2_phy_power_off()
595 	if (--pad->enable > 0)  in tegra124_usb2_phy_power_off()
603 	regulator_disable(port->supply);  in tegra124_usb2_phy_power_off()
604 	mutex_unlock(&pad->lock);  in tegra124_usb2_phy_power_off()
627 		return ERR_PTR(-ENOMEM);  in tegra124_usb2_pad_probe()
629 	mutex_init(&usb2->lock);  in tegra124_usb2_pad_probe()
631 	pad = &usb2->base;  in tegra124_usb2_pad_probe()
632 	pad->ops = &tegra124_usb2_lane_ops;  in tegra124_usb2_pad_probe()
633 	pad->soc = soc;  in tegra124_usb2_pad_probe()
645 	dev_set_drvdata(&pad->dev, pad);  in tegra124_usb2_pad_probe()
650 	device_unregister(&pad->dev);  in tegra124_usb2_pad_probe()
680 	TEGRA124_LANE("ulpi-0", 0x004, 12, 0x1, ulpi),
692 		return ERR_PTR(-ENOMEM);  in tegra124_ulpi_lane_probe()
694 	INIT_LIST_HEAD(&ulpi->base.list);  in tegra124_ulpi_lane_probe()
695 	ulpi->base.soc = &pad->soc->lanes[index];  in tegra124_ulpi_lane_probe()
696 	ulpi->base.index = index;  in tegra124_ulpi_lane_probe()
697 	ulpi->base.pad = pad;  in tegra124_ulpi_lane_probe()
698 	ulpi->base.np = np;  in tegra124_ulpi_lane_probe()
700 	err = tegra_xusb_lane_parse_dt(&ulpi->base, np);  in tegra124_ulpi_lane_probe()
706 	return &ulpi->base;  in tegra124_ulpi_lane_probe()
725 	return tegra124_xusb_padctl_enable(lane->pad->padctl);  in tegra124_ulpi_phy_init()
732 	return tegra124_xusb_padctl_disable(lane->pad->padctl);  in tegra124_ulpi_phy_exit()
764 		return ERR_PTR(-ENOMEM);  in tegra124_ulpi_pad_probe()
766 	pad = &ulpi->base;  in tegra124_ulpi_pad_probe()
767 	pad->ops = &tegra124_ulpi_lane_ops;  in tegra124_ulpi_pad_probe()
768 	pad->soc = soc;  in tegra124_ulpi_pad_probe()
780 	dev_set_drvdata(&pad->dev, pad);  in tegra124_ulpi_pad_probe()
785 	device_unregister(&pad->dev);  in tegra124_ulpi_pad_probe()
815 	TEGRA124_LANE("hsic-0", 0x004, 14, 0x1, hsic),
816 	TEGRA124_LANE("hsic-1", 0x004, 15, 0x1, hsic),
828 		return ERR_PTR(-ENOMEM);  in tegra124_hsic_lane_probe()
830 	INIT_LIST_HEAD(&hsic->base.list);  in tegra124_hsic_lane_probe()
831 	hsic->base.soc = &pad->soc->lanes[index];  in tegra124_hsic_lane_probe()
832 	hsic->base.index = index;  in tegra124_hsic_lane_probe()
833 	hsic->base.pad = pad;  in tegra124_hsic_lane_probe()
834 	hsic->base.np = np;  in tegra124_hsic_lane_probe()
836 	err = tegra_xusb_lane_parse_dt(&hsic->base, np);  in tegra124_hsic_lane_probe()
842 	return &hsic->base;  in tegra124_hsic_lane_probe()
861 	return tegra124_xusb_padctl_enable(lane->pad->padctl);  in tegra124_hsic_phy_init()
868 	return tegra124_xusb_padctl_disable(lane->pad->padctl);  in tegra124_hsic_phy_exit()
875 	struct tegra_xusb_hsic_pad *pad = to_hsic_pad(lane->pad);  in tegra124_hsic_phy_power_on()
876 	struct tegra_xusb_padctl *padctl = lane->pad->padctl;  in tegra124_hsic_phy_power_on()
877 	unsigned int index = lane->index;  in tegra124_hsic_phy_power_on()
881 	err = regulator_enable(pad->supply);  in tegra124_hsic_phy_power_on()
885 	padctl_writel(padctl, hsic->strobe_trim,  in tegra124_hsic_phy_power_on()
890 	if (hsic->auto_term)  in tegra124_hsic_phy_power_on()
906 	value |= (hsic->tx_rtune_n <<  in tegra124_hsic_phy_power_on()
908 		(hsic->tx_rtune_p <<  in tegra124_hsic_phy_power_on()
910 		(hsic->tx_rslew_n <<  in tegra124_hsic_phy_power_on()
912 		(hsic->tx_rslew_p <<  in tegra124_hsic_phy_power_on()
921 	value |= (hsic->rx_strobe_trim <<  in tegra124_hsic_phy_power_on()
923 		(hsic->rx_data_trim <<  in tegra124_hsic_phy_power_on()
944 	struct tegra_xusb_hsic_pad *pad = to_hsic_pad(lane->pad);  in tegra124_hsic_phy_power_off()
945 	struct tegra_xusb_padctl *padctl = lane->pad->padctl;  in tegra124_hsic_phy_power_off()
946 	unsigned int index = lane->index;  in tegra124_hsic_phy_power_off()
956 	regulator_disable(pad->supply);  in tegra124_hsic_phy_power_off()
980 		return ERR_PTR(-ENOMEM);  in tegra124_hsic_pad_probe()
982 	pad = &hsic->base;  in tegra124_hsic_pad_probe()
983 	pad->ops = &tegra124_hsic_lane_ops;  in tegra124_hsic_pad_probe()
984 	pad->soc = soc;  in tegra124_hsic_pad_probe()
996 	dev_set_drvdata(&pad->dev, pad);  in tegra124_hsic_pad_probe()
1001 	device_unregister(&pad->dev);  in tegra124_hsic_pad_probe()
1027 	"usb3-ss",
1028 	"sata",
1032 	TEGRA124_LANE("pcie-0", 0x134, 16, 0x3, pcie),
1033 	TEGRA124_LANE("pcie-1", 0x134, 18, 0x3, pcie),
1034 	TEGRA124_LANE("pcie-2", 0x134, 20, 0x3, pcie),
1035 	TEGRA124_LANE("pcie-3", 0x134, 22, 0x3, pcie),
1036 	TEGRA124_LANE("pcie-4", 0x134, 24, 0x3, pcie),
1048 		return ERR_PTR(-ENOMEM);  in tegra124_pcie_lane_probe()
1050 	INIT_LIST_HEAD(&pcie->base.list);  in tegra124_pcie_lane_probe()
1051 	pcie->base.soc = &pad->soc->lanes[index];  in tegra124_pcie_lane_probe()
1052 	pcie->base.index = index;  in tegra124_pcie_lane_probe()
1053 	pcie->base.pad = pad;  in tegra124_pcie_lane_probe()
1054 	pcie->base.np = np;  in tegra124_pcie_lane_probe()
1056 	err = tegra_xusb_lane_parse_dt(&pcie->base, np);  in tegra124_pcie_lane_probe()
1062 	return &pcie->base;  in tegra124_pcie_lane_probe()
1081 	return tegra124_xusb_padctl_enable(lane->pad->padctl);  in tegra124_pcie_phy_init()
1088 	return tegra124_xusb_padctl_disable(lane->pad->padctl);  in tegra124_pcie_phy_exit()
1094 	struct tegra_xusb_padctl *padctl = lane->pad->padctl;  in tegra124_pcie_phy_power_on()
1096 	int err = -ETIMEDOUT;  in tegra124_pcie_phy_power_on()
1126 	value |= XUSB_PADCTL_USB3_PAD_MUX_PCIE_IDDQ_DISABLE(lane->index);  in tegra124_pcie_phy_power_on()
1135 	struct tegra_xusb_padctl *padctl = lane->pad->padctl;  in tegra124_pcie_phy_power_off()
1139 	value &= ~XUSB_PADCTL_USB3_PAD_MUX_PCIE_IDDQ_DISABLE(lane->index);  in tegra124_pcie_phy_power_off()
1168 		return ERR_PTR(-ENOMEM);  in tegra124_pcie_pad_probe()
1170 	pad = &pcie->base;  in tegra124_pcie_pad_probe()
1171 	pad->ops = &tegra124_pcie_lane_ops;  in tegra124_pcie_pad_probe()
1172 	pad->soc = soc;  in tegra124_pcie_pad_probe()
1184 	dev_set_drvdata(&pad->dev, pad);  in tegra124_pcie_pad_probe()
1189 	device_unregister(&pad->dev);  in tegra124_pcie_pad_probe()
1214 	TEGRA124_LANE("sata-0", 0x134, 26, 0x3, pcie),
1221 	struct tegra_xusb_sata_lane *sata;  in tegra124_sata_lane_probe()  local
1224 	sata = kzalloc(sizeof(*sata), GFP_KERNEL);  in tegra124_sata_lane_probe()
1225 	if (!sata)  in tegra124_sata_lane_probe()
1226 		return ERR_PTR(-ENOMEM);  in tegra124_sata_lane_probe()
1228 	INIT_LIST_HEAD(&sata->base.list);  in tegra124_sata_lane_probe()
1229 	sata->base.soc = &pad->soc->lanes[index];  in tegra124_sata_lane_probe()
1230 	sata->base.index = index;  in tegra124_sata_lane_probe()
1231 	sata->base.pad = pad;  in tegra124_sata_lane_probe()
1232 	sata->base.np = np;  in tegra124_sata_lane_probe()
1234 	err = tegra_xusb_lane_parse_dt(&sata->base, np);  in tegra124_sata_lane_probe()
1236 		kfree(sata);  in tegra124_sata_lane_probe()
1240 	return &sata->base;  in tegra124_sata_lane_probe()
1245 	struct tegra_xusb_sata_lane *sata = to_sata_lane(lane);  in tegra124_sata_lane_remove()  local
1247 	kfree(sata);  in tegra124_sata_lane_remove()
1259 	return tegra124_xusb_padctl_enable(lane->pad->padctl);  in tegra124_sata_phy_init()
1266 	return tegra124_xusb_padctl_disable(lane->pad->padctl);  in tegra124_sata_phy_exit()
1272 	struct tegra_xusb_padctl *padctl = lane->pad->padctl;  in tegra124_sata_phy_power_on()
1274 	int err = -ETIMEDOUT;  in tegra124_sata_phy_power_on()
1308 	value |= XUSB_PADCTL_USB3_PAD_MUX_SATA_IDDQ_DISABLE(lane->index);  in tegra124_sata_phy_power_on()
1317 	struct tegra_xusb_padctl *padctl = lane->pad->padctl;  in tegra124_sata_phy_power_off()
1321 	value &= ~XUSB_PADCTL_USB3_PAD_MUX_SATA_IDDQ_DISABLE(lane->index);  in tegra124_sata_phy_power_off()
1358 	struct tegra_xusb_sata_pad *sata;  in tegra124_sata_pad_probe()  local
1362 	sata = kzalloc(sizeof(*sata), GFP_KERNEL);  in tegra124_sata_pad_probe()
1363 	if (!sata)  in tegra124_sata_pad_probe()
1364 		return ERR_PTR(-ENOMEM);  in tegra124_sata_pad_probe()
1366 	pad = &sata->base;  in tegra124_sata_pad_probe()
1367 	pad->ops = &tegra124_sata_lane_ops;  in tegra124_sata_pad_probe()
1368 	pad->soc = soc;  in tegra124_sata_pad_probe()
1372 		kfree(sata);  in tegra124_sata_pad_probe()
1380 	dev_set_drvdata(&pad->dev, pad);  in tegra124_sata_pad_probe()
1385 	device_unregister(&pad->dev);  in tegra124_sata_pad_probe()
1392 	struct tegra_xusb_sata_pad *sata = to_sata_pad(pad);  in tegra124_sata_pad_remove()  local
1394 	kfree(sata);  in tegra124_sata_pad_remove()
1403 	.name = "sata",
1417 static int tegra124_usb2_port_enable(struct tegra_xusb_port *port)  in tegra124_usb2_port_enable()  argument
1422 static void tegra124_usb2_port_disable(struct tegra_xusb_port *port)  in tegra124_usb2_port_disable()  argument
1427 tegra124_usb2_port_map(struct tegra_xusb_port *port)  in tegra124_usb2_port_map()  argument
1429 	return tegra_xusb_find_lane(port->padctl, "usb2", port->index);  in tegra124_usb2_port_map()
1438 static int tegra124_ulpi_port_enable(struct tegra_xusb_port *port)  in tegra124_ulpi_port_enable()  argument
1443 static void tegra124_ulpi_port_disable(struct tegra_xusb_port *port)  in tegra124_ulpi_port_disable()  argument
1448 tegra124_ulpi_port_map(struct tegra_xusb_port *port)  in tegra124_ulpi_port_map()  argument
1450 	return tegra_xusb_find_lane(port->padctl, "ulpi", port->index);  in tegra124_ulpi_port_map()
1459 static int tegra124_hsic_port_enable(struct tegra_xusb_port *port)  in tegra124_hsic_port_enable()  argument
1464 static void tegra124_hsic_port_disable(struct tegra_xusb_port *port)  in tegra124_hsic_port_disable()  argument
1469 tegra124_hsic_port_map(struct tegra_xusb_port *port)  in tegra124_hsic_port_map()  argument
1471 	return tegra_xusb_find_lane(port->padctl, "hsic", port->index);  in tegra124_hsic_port_map()
1480 static int tegra124_usb3_port_enable(struct tegra_xusb_port *port)  in tegra124_usb3_port_enable()  argument
1482 	struct tegra_xusb_usb3_port *usb3 = to_usb3_port(port);  in tegra124_usb3_port_enable()
1483 	struct tegra_xusb_padctl *padctl = port->padctl;  in tegra124_usb3_port_enable()
1484 	struct tegra_xusb_lane *lane = usb3->base.lane;  in tegra124_usb3_port_enable()
1485 	unsigned int index = port->index, offset;  in tegra124_usb3_port_enable()
1490 	if (!usb3->internal)  in tegra124_usb3_port_enable()
1496 	value |= XUSB_PADCTL_SS_PORT_MAP_PORTX_MAP(index, usb3->port);  in tegra124_usb3_port_enable()
1500 	 * TODO: move this code into the PCIe/SATA PHY ->power_on() callbacks  in tegra124_usb3_port_enable()
1518 	if (usb3->context_saved) {  in tegra124_usb3_port_enable()
1523 		value |= (usb3->ctle_g <<  in tegra124_usb3_port_enable()
1525 			 (usb3->ctle_z <<  in tegra124_usb3_port_enable()
1533 	if (usb3->context_saved) {  in tegra124_usb3_port_enable()
1538 		value |= (usb3->tap1 <<  in tegra124_usb3_port_enable()
1540 			 (usb3->amp <<  in tegra124_usb3_port_enable()
1546 	if (lane->pad == padctl->pcie)  in tegra124_usb3_port_enable()
1547 		offset = XUSB_PADCTL_IOPHY_MISC_PAD_PX_CTL2(lane->index);  in tegra124_usb3_port_enable()
1558 	if (lane->pad == padctl->pcie)  in tegra124_usb3_port_enable()
1559 		offset = XUSB_PADCTL_IOPHY_MISC_PAD_PX_CTL5(lane->index);  in tegra124_usb3_port_enable()
1567 	/* Enable SATA PHY when SATA lane is used */  in tegra124_usb3_port_enable()
1568 	if (lane->pad == padctl->sata) {  in tegra124_usb3_port_enable()
1617 static void tegra124_usb3_port_disable(struct tegra_xusb_port *port)  in tegra124_usb3_port_disable()  argument
1619 	struct tegra_xusb_padctl *padctl = port->padctl;  in tegra124_usb3_port_disable()
1623 	value |= XUSB_PADCTL_ELPG_PROGRAM_SSPX_ELPG_CLAMP_EN_EARLY(port->index);  in tegra124_usb3_port_disable()
1629 	value |= XUSB_PADCTL_ELPG_PROGRAM_SSPX_ELPG_CLAMP_EN(port->index);  in tegra124_usb3_port_disable()
1635 	value |= XUSB_PADCTL_ELPG_PROGRAM_SSPX_ELPG_VCORE_DOWN(port->index);  in tegra124_usb3_port_disable()
1639 	value &= ~XUSB_PADCTL_SS_PORT_MAP_PORTX_MAP_MASK(port->index);  in tegra124_usb3_port_disable()
1640 	value |= XUSB_PADCTL_SS_PORT_MAP_PORTX_MAP(port->index, 0x7);  in tegra124_usb3_port_disable()
1647 	{ 1, "sata", 0 },
1652 tegra124_usb3_port_map(struct tegra_xusb_port *port)  in tegra124_usb3_port_map()  argument
1654 	return tegra_xusb_port_find_lane(port, tegra124_usb3_map, "usb3-ss");  in tegra124_usb3_port_map()
1674 	for (i = 0; i < ARRAY_SIZE(fuse->hs_curr_level); i++) {  in tegra124_xusb_read_fuse_calibration()
1675 		fuse->hs_curr_level[i] =  in tegra124_xusb_read_fuse_calibration()
1679 	fuse->hs_iref_cap =  in tegra124_xusb_read_fuse_calibration()
1682 	fuse->hs_term_range_adj =  in tegra124_xusb_read_fuse_calibration()
1685 	fuse->hs_squelch_level =  in tegra124_xusb_read_fuse_calibration()
1701 		return ERR_PTR(-ENOMEM);  in tegra124_xusb_padctl_probe()
1703 	padctl->base.dev = dev;  in tegra124_xusb_padctl_probe()
1704 	padctl->base.soc = soc;  in tegra124_xusb_padctl_probe()
1706 	err = tegra124_xusb_read_fuse_calibration(&padctl->fuse);  in tegra124_xusb_padctl_probe()
1710 	return &padctl->base;  in tegra124_xusb_padctl_probe()