Lines Matching +full:aspm +full:- +full:no +full:- +full:l0s
1 // SPDX-License-Identifier: GPL-2.0
3 * Enable PCIe link L0s/L1 state and Clock Power Management
29 #define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */
30 #define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */
32 #define ASPM_STATE_L1_1 (8) /* ASPM L1.1 state */
33 #define ASPM_STATE_L1_2 (0x10) /* ASPM L1.2 state */
45 u32 l0s; /* L0s latency (nsec) */ member
56 /* ASPM state */
57 u32 aspm_support:7; /* Supported ASPM state */
58 u32 aspm_enabled:7; /* Enabled ASPM state */
59 u32 aspm_capable:7; /* Capable ASPM state with latency */
60 u32 aspm_default:7; /* Default ASPM state by BIOS */
61 u32 aspm_disable:7; /* Disabled ASPM state */
112 /* Disable ASPM and Clock PM */ in policy_to_aspm_state()
115 /* Enable ASPM L0s/L1 */ in policy_to_aspm_state()
121 return link->aspm_default; in policy_to_aspm_state()
130 /* Disable ASPM and Clock PM */ in policy_to_clkpm_state()
137 return link->clkpm_default; in policy_to_clkpm_state()
145 struct pci_bus *linkbus = link->pdev->subordinate; in pcie_set_clkpm_nocheck()
148 list_for_each_entry(child, &linkbus->devices, bus_list) in pcie_set_clkpm_nocheck()
152 link->clkpm_enabled = !!enable; in pcie_set_clkpm_nocheck()
161 if (!link->clkpm_capable || link->clkpm_disable) in pcie_set_clkpm()
164 if (link->clkpm_enabled == enable) in pcie_set_clkpm()
175 struct pci_bus *linkbus = link->pdev->subordinate; in pcie_clkpm_cap_init()
178 list_for_each_entry(child, &linkbus->devices, bus_list) { in pcie_clkpm_cap_init()
189 link->clkpm_enabled = enabled; in pcie_clkpm_cap_init()
190 link->clkpm_default = enabled; in pcie_clkpm_cap_init()
191 link->clkpm_capable = capable; in pcie_clkpm_cap_init()
192 link->clkpm_disable = blacklist ? 1 : 0; in pcie_clkpm_cap_init()
209 return -ETIMEDOUT; in pcie_wait_for_retrain()
214 struct pci_dev *parent = link->pdev; in pcie_retrain_link()
220 * training by checking that there is no ongoing link training to in pcie_retrain_link()
231 if (parent->clear_retrain_link) { in pcie_retrain_link()
247 * common clock. That will reduce the ASPM state exit latency.
253 struct pci_dev *child, *parent = link->pdev; in pcie_aspm_configure_common_clock()
254 struct pci_bus *linkbus = parent->subordinate; in pcie_aspm_configure_common_clock()
259 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); in pcie_aspm_configure_common_clock()
278 list_for_each_entry(child, &linkbus->devices, bus_list) { in pcie_aspm_configure_common_clock()
288 pci_info(parent, "ASPM: current common clock configuration is inconsistent, reconfiguring\n"); in pcie_aspm_configure_common_clock()
293 list_for_each_entry(child, &linkbus->devices, bus_list) { in pcie_aspm_configure_common_clock()
295 child_old_ccc[PCI_FUNC(child->devfn)] = reg16 & PCI_EXP_LNKCTL_CCC; in pcie_aspm_configure_common_clock()
307 pci_err(parent, "ASPM: Could not configure common clock\n"); in pcie_aspm_configure_common_clock()
308 list_for_each_entry(child, &linkbus->devices, bus_list) in pcie_aspm_configure_common_clock()
311 child_old_ccc[PCI_FUNC(child->devfn)]); in pcie_aspm_configure_common_clock()
317 /* Convert L0s latency encoding to ns */
327 /* Convert L0s acceptable latency encoding to ns */
331 return -1U; in calc_l0s_acceptable()
349 return -1U; in calc_l1_acceptable()
401 if ((endpoint->current_state != PCI_D0) && in pcie_aspm_check_latency()
402 (endpoint->current_state != PCI_UNKNOWN)) in pcie_aspm_check_latency()
405 link = endpoint->bus->self->link_state; in pcie_aspm_check_latency()
406 acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)]; in pcie_aspm_check_latency()
409 /* Check upstream direction L0s latency */ in pcie_aspm_check_latency()
410 if ((link->aspm_capable & ASPM_STATE_L0S_UP) && in pcie_aspm_check_latency()
411 (link->latency_up.l0s > acceptable->l0s)) in pcie_aspm_check_latency()
412 link->aspm_capable &= ~ASPM_STATE_L0S_UP; in pcie_aspm_check_latency()
414 /* Check downstream direction L0s latency */ in pcie_aspm_check_latency()
415 if ((link->aspm_capable & ASPM_STATE_L0S_DW) && in pcie_aspm_check_latency()
416 (link->latency_dw.l0s > acceptable->l0s)) in pcie_aspm_check_latency()
417 link->aspm_capable &= ~ASPM_STATE_L0S_DW; in pcie_aspm_check_latency()
421 * more microsecond for L1. Spec doesn't mention L0s. in pcie_aspm_check_latency()
431 latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1); in pcie_aspm_check_latency()
432 if ((link->aspm_capable & ASPM_STATE_L1) && in pcie_aspm_check_latency()
433 (latency + l1_switch_latency > acceptable->l1)) in pcie_aspm_check_latency()
434 link->aspm_capable &= ~ASPM_STATE_L1; in pcie_aspm_check_latency()
437 link = link->parent; in pcie_aspm_check_latency()
449 list_for_each_entry(child, &linkbus->devices, bus_list) in pci_function_0()
450 if (PCI_FUNC(child->devfn) == 0) in pci_function_0()
470 struct pci_dev *child = link->downstream, *parent = link->pdev; in aspm_calc_l1ss_info()
477 if (!(link->aspm_support & ASPM_STATE_L1_2_MASK)) in aspm_calc_l1ss_info()
506 * Based on PCIe r3.1, sec 5.5.3.3.1, Figures 5-16 and 5-17, and in aspm_calc_l1ss_info()
507 * Table 5-11. T(POWER_OFF) is at most 2us and T(L1.2) is at in aspm_calc_l1ss_info()
514 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, &pctl1); in aspm_calc_l1ss_info()
515 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, &pctl2); in aspm_calc_l1ss_info()
516 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, &cctl1); in aspm_calc_l1ss_info()
517 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL2, &cctl2); in aspm_calc_l1ss_info()
528 pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, in aspm_calc_l1ss_info()
530 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in aspm_calc_l1ss_info()
535 pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2); in aspm_calc_l1ss_info()
536 pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2); in aspm_calc_l1ss_info()
539 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in aspm_calc_l1ss_info()
543 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in aspm_calc_l1ss_info()
546 pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, in aspm_calc_l1ss_info()
551 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 0, in aspm_calc_l1ss_info()
553 pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 0, in aspm_calc_l1ss_info()
560 struct pci_dev *child = link->downstream, *parent = link->pdev; in pcie_aspm_cap_init()
565 struct pci_bus *linkbus = parent->subordinate; in pcie_aspm_cap_init()
568 /* Set enabled/disable so that we will disable ASPM later */ in pcie_aspm_cap_init()
569 link->aspm_enabled = ASPM_STATE_ALL; in pcie_aspm_cap_init()
570 link->aspm_disable = ASPM_STATE_ALL; in pcie_aspm_cap_init()
575 * If ASPM not supported, don't mess with the clocks and link, in pcie_aspm_cap_init()
587 * Re-read upstream/downstream components' register state after in pcie_aspm_cap_init()
588 * clock configuration. L0s & L1 exit latencies in the otherwise in pcie_aspm_cap_init()
589 * read-only Link Capabilities may change depending on common clock in pcie_aspm_cap_init()
598 * Setup L0s state in pcie_aspm_cap_init()
600 * Note that we must not enable L0s in either direction on a in pcie_aspm_cap_init()
602 * support L0s. in pcie_aspm_cap_init()
605 link->aspm_support |= ASPM_STATE_L0S; in pcie_aspm_cap_init()
608 link->aspm_enabled |= ASPM_STATE_L0S_UP; in pcie_aspm_cap_init()
610 link->aspm_enabled |= ASPM_STATE_L0S_DW; in pcie_aspm_cap_init()
611 link->latency_up.l0s = calc_l0s_latency(parent_lnkcap); in pcie_aspm_cap_init()
612 link->latency_dw.l0s = calc_l0s_latency(child_lnkcap); in pcie_aspm_cap_init()
616 link->aspm_support |= ASPM_STATE_L1; in pcie_aspm_cap_init()
619 link->aspm_enabled |= ASPM_STATE_L1; in pcie_aspm_cap_init()
620 link->latency_up.l1 = calc_l1_latency(parent_lnkcap); in pcie_aspm_cap_init()
621 link->latency_dw.l1 = calc_l1_latency(child_lnkcap); in pcie_aspm_cap_init()
624 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP, in pcie_aspm_cap_init()
626 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP, in pcie_aspm_cap_init()
636 * to this device, we can't use ASPM L1.2 because it relies on the in pcie_aspm_cap_init()
639 if (!child->ltr_path) in pcie_aspm_cap_init()
643 link->aspm_support |= ASPM_STATE_L1_1; in pcie_aspm_cap_init()
645 link->aspm_support |= ASPM_STATE_L1_2; in pcie_aspm_cap_init()
647 link->aspm_support |= ASPM_STATE_L1_1_PCIPM; in pcie_aspm_cap_init()
649 link->aspm_support |= ASPM_STATE_L1_2_PCIPM; in pcie_aspm_cap_init()
652 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in pcie_aspm_cap_init()
655 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, in pcie_aspm_cap_init()
659 link->aspm_enabled |= ASPM_STATE_L1_1; in pcie_aspm_cap_init()
661 link->aspm_enabled |= ASPM_STATE_L1_2; in pcie_aspm_cap_init()
663 link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM; in pcie_aspm_cap_init()
665 link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM; in pcie_aspm_cap_init()
667 if (link->aspm_support & ASPM_STATE_L1SS) in pcie_aspm_cap_init()
671 link->aspm_default = link->aspm_enabled; in pcie_aspm_cap_init()
674 link->aspm_capable = link->aspm_support; in pcie_aspm_cap_init()
677 list_for_each_entry(child, &linkbus->devices, bus_list) { in pcie_aspm_cap_init()
680 &link->acceptable[PCI_FUNC(child->devfn)]; in pcie_aspm_cap_init()
687 /* Calculate endpoint L0s acceptable latency */ in pcie_aspm_cap_init()
689 acceptable->l0s = calc_l0s_acceptable(encoding); in pcie_aspm_cap_init()
692 acceptable->l1 = calc_l1_acceptable(encoding); in pcie_aspm_cap_init()
698 /* Configure the ASPM L1 substates */
702 struct pci_dev *child = link->downstream, *parent = link->pdev; in pcie_config_aspm_l1ss()
704 enable_req = (link->aspm_enabled ^ state) & state; in pcie_config_aspm_l1ss()
708 * - When enabling L1.x, enable bit at parent first, then at child in pcie_config_aspm_l1ss()
709 * - When disabling L1.x, disable bit at child first, then at parent in pcie_config_aspm_l1ss()
710 * - When enabling ASPM L1.x, need to disable L1 in pcie_config_aspm_l1ss()
712 * - The ASPM/PCIPM L1.2 must be disabled while programming timing in pcie_config_aspm_l1ss()
720 pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, in pcie_config_aspm_l1ss()
722 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in pcie_config_aspm_l1ss()
746 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in pcie_config_aspm_l1ss()
748 pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, in pcie_config_aspm_l1ss()
761 struct pci_dev *child = link->downstream, *parent = link->pdev; in pcie_config_aspm_link()
762 struct pci_bus *linkbus = parent->subordinate; in pcie_config_aspm_link()
765 state &= (link->aspm_capable & ~link->aspm_disable); in pcie_config_aspm_link()
772 if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) { in pcie_config_aspm_link()
774 state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM); in pcie_config_aspm_link()
778 if (link->aspm_enabled == state) in pcie_config_aspm_link()
780 /* Convert ASPM state to upstream/downstream ASPM register state */ in pcie_config_aspm_link()
790 if (link->aspm_capable & ASPM_STATE_L1SS) in pcie_config_aspm_link()
795 * same setting for ASPM. Enabling ASPM L1 should be done in in pcie_config_aspm_link()
797 * versa for disabling ASPM L1. Spec doesn't mention L0S. in pcie_config_aspm_link()
801 list_for_each_entry(child, &linkbus->devices, bus_list) in pcie_config_aspm_link()
806 link->aspm_enabled = state; in pcie_config_aspm_link()
813 link = link->parent; in pcie_config_aspm_path()
819 link->pdev->link_state = NULL; in free_link_state()
830 * very strange. Disable ASPM for the whole slot in pcie_aspm_sanity_check()
832 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) { in pcie_aspm_sanity_check()
834 return -EINVAL; in pcie_aspm_sanity_check()
837 * If ASPM is disabled then we're not going to change in pcie_aspm_sanity_check()
839 * pre-1.1 device in pcie_aspm_sanity_check()
846 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use in pcie_aspm_sanity_check()
851 …pci_info(child, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\… in pcie_aspm_sanity_check()
852 return -EINVAL; in pcie_aspm_sanity_check()
866 INIT_LIST_HEAD(&link->sibling); in alloc_pcie_link_state()
867 link->pdev = pdev; in alloc_pcie_link_state()
868 link->downstream = pci_function_0(pdev->subordinate); in alloc_pcie_link_state()
871 * Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe in alloc_pcie_link_state()
879 !pdev->bus->parent->self) { in alloc_pcie_link_state()
880 link->root = link; in alloc_pcie_link_state()
884 parent = pdev->bus->parent->self->link_state; in alloc_pcie_link_state()
890 link->parent = parent; in alloc_pcie_link_state()
891 link->root = link->parent->root; in alloc_pcie_link_state()
894 list_add(&link->sibling, &link_list); in alloc_pcie_link_state()
895 pdev->link_state = link; in alloc_pcie_link_state()
903 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) in pcie_aspm_update_sysfs_visibility()
904 sysfs_update_group(&child->dev.kobj, &aspm_ctrl_attr_group); in pcie_aspm_update_sysfs_visibility()
920 if (pdev->link_state) in pcie_aspm_init_link_state()
933 pdev->bus->self) in pcie_aspm_init_link_state()
937 if (list_empty(&pdev->subordinate->devices)) in pcie_aspm_init_link_state()
945 * Setup initial ASPM state. Note that we need to configure in pcie_aspm_init_link_state()
956 * link policy setting. Enabling ASPM on broken hardware can cripple in pcie_aspm_init_link_state()
957 * it even before the driver has had a chance to disable ASPM, so in pcie_aspm_init_link_state()
958 * default to a safe level right now. If we're enabling ASPM beyond in pcie_aspm_init_link_state()
980 BUG_ON(root->parent); in pcie_update_aspm_capable()
982 if (link->root != root) in pcie_update_aspm_capable()
984 link->aspm_capable = link->aspm_support; in pcie_update_aspm_capable()
988 struct pci_bus *linkbus = link->pdev->subordinate; in pcie_update_aspm_capable()
989 if (link->root != root) in pcie_update_aspm_capable()
991 list_for_each_entry(child, &linkbus->devices, bus_list) { in pcie_update_aspm_capable()
1003 struct pci_dev *parent = pdev->bus->self; in pcie_aspm_exit_link_state()
1006 if (!parent || !parent->link_state) in pcie_aspm_exit_link_state()
1012 link = parent->link_state; in pcie_aspm_exit_link_state()
1013 root = link->root; in pcie_aspm_exit_link_state()
1014 parent_link = link->parent; in pcie_aspm_exit_link_state()
1017 * Free the parent link state, no later than function 0 (i.e. in pcie_aspm_exit_link_state()
1018 * link->downstream) being removed. in pcie_aspm_exit_link_state()
1024 if (pdev != link->downstream) in pcie_aspm_exit_link_state()
1028 list_del(&link->sibling); in pcie_aspm_exit_link_state()
1045 struct pcie_link_state *link = pdev->link_state; in pcie_aspm_pm_state_change()
1055 pcie_update_aspm_capable(link->root); in pcie_aspm_pm_state_change()
1063 struct pcie_link_state *link = pdev->link_state; in pcie_aspm_powersave_config_link()
1091 return bridge->link_state; in pcie_aspm_get_link()
1099 return -EINVAL; in __pci_disable_link_state()
1101 * A driver requested that ASPM be disabled on this device, but in __pci_disable_link_state()
1102 * if we don't have permission to manage ASPM (e.g., on ACPI in __pci_disable_link_state()
1109 pci_warn(pdev, "can't disable ASPM; OS doesn't have ASPM control\n"); in __pci_disable_link_state()
1110 return -EPERM; in __pci_disable_link_state()
1117 link->aspm_disable |= ASPM_STATE_L0S; in __pci_disable_link_state()
1120 link->aspm_disable |= ASPM_STATE_L1 | ASPM_STATE_L1SS; in __pci_disable_link_state()
1122 link->aspm_disable |= ASPM_STATE_L1_1; in __pci_disable_link_state()
1124 link->aspm_disable |= ASPM_STATE_L1_2; in __pci_disable_link_state()
1126 link->aspm_disable |= ASPM_STATE_L1_1_PCIPM; in __pci_disable_link_state()
1128 link->aspm_disable |= ASPM_STATE_L1_2_PCIPM; in __pci_disable_link_state()
1132 link->clkpm_disable = 1; in __pci_disable_link_state()
1148 * pci_disable_link_state - Disable device's link state, so the link will
1149 * never enter specific states. Note that if the BIOS didn't grant ASPM
1154 * @state: ASPM link state to disable
1169 return -EPERM; in pcie_aspm_set_policy()
1204 * pcie_aspm_enabled - Check if PCIe ASPM has been enabled for a device.
1219 return link->aspm_enabled; in pcie_aspm_enabled()
1230 return sprintf(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0); in aspm_attr_show_common()
1242 return -EINVAL; in aspm_attr_store_common()
1248 link->aspm_disable &= ~state; in aspm_attr_store_common()
1251 link->aspm_disable &= ~ASPM_STATE_L1; in aspm_attr_store_common()
1253 link->aspm_disable |= state; in aspm_attr_store_common()
1255 link->aspm_disable |= ASPM_STATE_L1SS; in aspm_attr_store_common()
1276 ASPM_ATTR(l0s_aspm, L0S) in ASPM_ATTR() argument
1289 return sprintf(buf, "%d\n", link->clkpm_enabled); in ASPM_ATTR()
1301 return -EINVAL; in clkpm_store()
1306 link->clkpm_disable = !state_enable; in clkpm_store()
1353 return link->clkpm_capable ? a->mode : 0; in aspm_ctrl_attrs_are_visible()
1355 return link->aspm_capable & aspm_state_map[n - 1] ? a->mode : 0; in aspm_ctrl_attrs_are_visible()
1370 printk(KERN_INFO "PCIe ASPM is disabled\n"); in pcie_aspm_disable()
1373 printk(KERN_INFO "PCIe ASPM is forcibly enabled\n"); in pcie_aspm_disable()
1383 * Disabling ASPM is intended to prevent the kernel from modifying in pcie_no_aspm()