1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * File: drivers/pci/pcie/aspm.c
4 * Enabling PCIe link L0s/L1 state and Clock Power Management
5 *
6 * Copyright (C) 2007 Intel
7 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
8 * Copyright (C) Shaohua Li (shaohua.li@intel.com)
9 */
10
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/moduleparam.h>
14 #include <linux/pci.h>
15 #include <linux/pci_regs.h>
16 #include <linux/errno.h>
17 #include <linux/pm.h>
18 #include <linux/init.h>
19 #include <linux/slab.h>
20 #include <linux/jiffies.h>
21 #include <linux/delay.h>
22 #include <linux/pci-aspm.h>
23 #include "../pci.h"
24
25 #ifdef MODULE_PARAM_PREFIX
26 #undef MODULE_PARAM_PREFIX
27 #endif
28 #define MODULE_PARAM_PREFIX "pcie_aspm."
29
30 /* Note: those are not register definitions */
31 #define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */
32 #define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */
33 #define ASPM_STATE_L1 (4) /* L1 state */
34 #define ASPM_STATE_L1_1 (8) /* ASPM L1.1 state */
35 #define ASPM_STATE_L1_2 (0x10) /* ASPM L1.2 state */
36 #define ASPM_STATE_L1_1_PCIPM (0x20) /* PCI PM L1.1 state */
37 #define ASPM_STATE_L1_2_PCIPM (0x40) /* PCI PM L1.2 state */
38 #define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM)
39 #define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM)
40 #define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\
41 ASPM_STATE_L1_2_MASK)
42 #define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
43 #define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \
44 ASPM_STATE_L1SS)
45
46 /*
47 * When L1 substates are enabled, the LTR L1.2 threshold is a timing parameter
48 * that decides whether L1.1 or L1.2 is entered (Refer PCIe spec for details).
49 * Not sure is there is a way to "calculate" this on the fly, but maybe we
50 * could turn it into a parameter in future. This value has been taken from
51 * the following files from Intel's coreboot (which is the only code I found
52 * to have used this):
53 * https://www.coreboot.org/pipermail/coreboot-gerrit/2015-March/021134.html
54 * https://review.coreboot.org/#/c/8832/
55 */
56 #define LTR_L1_2_THRESHOLD_BITS ((1 << 21) | (1 << 23) | (1 << 30))
57
58 struct aspm_latency {
59 u32 l0s; /* L0s latency (nsec) */
60 u32 l1; /* L1 latency (nsec) */
61 };
62
63 struct pcie_link_state {
64 struct pci_dev *pdev; /* Upstream component of the Link */
65 struct pci_dev *downstream; /* Downstream component, function 0 */
66 struct pcie_link_state *root; /* pointer to the root port link */
67 struct pcie_link_state *parent; /* pointer to the parent Link state */
68 struct list_head sibling; /* node in link_list */
69 struct list_head children; /* list of child link states */
70 struct list_head link; /* node in parent's children list */
71
72 /* ASPM state */
73 u32 aspm_support:7; /* Supported ASPM state */
74 u32 aspm_enabled:7; /* Enabled ASPM state */
75 u32 aspm_capable:7; /* Capable ASPM state with latency */
76 u32 aspm_default:7; /* Default ASPM state by BIOS */
77 u32 aspm_disable:7; /* Disabled ASPM state */
78
79 /* Clock PM state */
80 u32 clkpm_capable:1; /* Clock PM capable? */
81 u32 clkpm_enabled:1; /* Current Clock PM state */
82 u32 clkpm_default:1; /* Default Clock PM state by BIOS */
83
84 /* Exit latencies */
85 struct aspm_latency latency_up; /* Upstream direction exit latency */
86 struct aspm_latency latency_dw; /* Downstream direction exit latency */
87 /*
88 * Endpoint acceptable latencies. A pcie downstream port only
89 * has one slot under it, so at most there are 8 functions.
90 */
91 struct aspm_latency acceptable[8];
92
93 /* L1 PM Substate info */
94 struct {
95 u32 up_cap_ptr; /* L1SS cap ptr in upstream dev */
96 u32 dw_cap_ptr; /* L1SS cap ptr in downstream dev */
97 u32 ctl1; /* value to be programmed in ctl1 */
98 u32 ctl2; /* value to be programmed in ctl2 */
99 } l1ss;
100 };
101
102 static int aspm_disabled, aspm_force;
103 static bool aspm_support_enabled = true;
104 static DEFINE_MUTEX(aspm_lock);
105 static LIST_HEAD(link_list);
106
107 #define POLICY_DEFAULT 0 /* BIOS default setting */
108 #define POLICY_PERFORMANCE 1 /* high performance */
109 #define POLICY_POWERSAVE 2 /* high power saving */
110 #define POLICY_POWER_SUPERSAVE 3 /* possibly even more power saving */
111
112 #ifdef CONFIG_PCIEASPM_PERFORMANCE
113 static int aspm_policy = POLICY_PERFORMANCE;
114 #elif defined CONFIG_PCIEASPM_POWERSAVE
115 static int aspm_policy = POLICY_POWERSAVE;
116 #elif defined CONFIG_PCIEASPM_POWER_SUPERSAVE
117 static int aspm_policy = POLICY_POWER_SUPERSAVE;
118 #else
119 static int aspm_policy;
120 #endif
121
122 static const char *policy_str[] = {
123 [POLICY_DEFAULT] = "default",
124 [POLICY_PERFORMANCE] = "performance",
125 [POLICY_POWERSAVE] = "powersave",
126 [POLICY_POWER_SUPERSAVE] = "powersupersave"
127 };
128
129 #define LINK_RETRAIN_TIMEOUT HZ
130
policy_to_aspm_state(struct pcie_link_state * link)131 static int policy_to_aspm_state(struct pcie_link_state *link)
132 {
133 switch (aspm_policy) {
134 case POLICY_PERFORMANCE:
135 /* Disable ASPM and Clock PM */
136 return 0;
137 case POLICY_POWERSAVE:
138 /* Enable ASPM L0s/L1 */
139 return (ASPM_STATE_L0S | ASPM_STATE_L1);
140 case POLICY_POWER_SUPERSAVE:
141 /* Enable Everything */
142 return ASPM_STATE_ALL;
143 case POLICY_DEFAULT:
144 return link->aspm_default;
145 }
146 return 0;
147 }
148
policy_to_clkpm_state(struct pcie_link_state * link)149 static int policy_to_clkpm_state(struct pcie_link_state *link)
150 {
151 switch (aspm_policy) {
152 case POLICY_PERFORMANCE:
153 /* Disable ASPM and Clock PM */
154 return 0;
155 case POLICY_POWERSAVE:
156 case POLICY_POWER_SUPERSAVE:
157 /* Enable Clock PM */
158 return 1;
159 case POLICY_DEFAULT:
160 return link->clkpm_default;
161 }
162 return 0;
163 }
164
pcie_set_clkpm_nocheck(struct pcie_link_state * link,int enable)165 static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
166 {
167 struct pci_dev *child;
168 struct pci_bus *linkbus = link->pdev->subordinate;
169 u32 val = enable ? PCI_EXP_LNKCTL_CLKREQ_EN : 0;
170
171 list_for_each_entry(child, &linkbus->devices, bus_list)
172 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,
173 PCI_EXP_LNKCTL_CLKREQ_EN,
174 val);
175 link->clkpm_enabled = !!enable;
176 }
177
pcie_set_clkpm(struct pcie_link_state * link,int enable)178 static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
179 {
180 /* Don't enable Clock PM if the link is not Clock PM capable */
181 if (!link->clkpm_capable)
182 enable = 0;
183 /* Need nothing if the specified equals to current state */
184 if (link->clkpm_enabled == enable)
185 return;
186 pcie_set_clkpm_nocheck(link, enable);
187 }
188
pcie_clkpm_cap_init(struct pcie_link_state * link,int blacklist)189 static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
190 {
191 int capable = 1, enabled = 1;
192 u32 reg32;
193 u16 reg16;
194 struct pci_dev *child;
195 struct pci_bus *linkbus = link->pdev->subordinate;
196
197 /* All functions should have the same cap and state, take the worst */
198 list_for_each_entry(child, &linkbus->devices, bus_list) {
199 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, ®32);
200 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
201 capable = 0;
202 enabled = 0;
203 break;
204 }
205 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16);
206 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
207 enabled = 0;
208 }
209 link->clkpm_enabled = enabled;
210 link->clkpm_default = enabled;
211 link->clkpm_capable = (blacklist) ? 0 : capable;
212 }
213
pcie_retrain_link(struct pcie_link_state * link)214 static bool pcie_retrain_link(struct pcie_link_state *link)
215 {
216 struct pci_dev *parent = link->pdev;
217 unsigned long start_jiffies;
218 u16 reg16;
219
220 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16);
221 reg16 |= PCI_EXP_LNKCTL_RL;
222 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
223 if (parent->clear_retrain_link) {
224 /*
225 * Due to an erratum in some devices the Retrain Link bit
226 * needs to be cleared again manually to allow the link
227 * training to succeed.
228 */
229 reg16 &= ~PCI_EXP_LNKCTL_RL;
230 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
231 }
232
233 /* Wait for link training end. Break out after waiting for timeout */
234 start_jiffies = jiffies;
235 for (;;) {
236 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16);
237 if (!(reg16 & PCI_EXP_LNKSTA_LT))
238 break;
239 if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT))
240 break;
241 msleep(1);
242 }
243 return !(reg16 & PCI_EXP_LNKSTA_LT);
244 }
245
246 /*
247 * pcie_aspm_configure_common_clock: check if the 2 ends of a link
248 * could use common clock. If they are, configure them to use the
249 * common clock. That will reduce the ASPM state exit latency.
250 */
pcie_aspm_configure_common_clock(struct pcie_link_state * link)251 static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
252 {
253 int same_clock = 1;
254 u16 reg16, parent_reg, child_reg[8];
255 struct pci_dev *child, *parent = link->pdev;
256 struct pci_bus *linkbus = parent->subordinate;
257 /*
258 * All functions of a slot should have the same Slot Clock
259 * Configuration, so just check one function
260 */
261 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
262 BUG_ON(!pci_is_pcie(child));
263
264 /* Check downstream component if bit Slot Clock Configuration is 1 */
265 pcie_capability_read_word(child, PCI_EXP_LNKSTA, ®16);
266 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
267 same_clock = 0;
268
269 /* Check upstream component if bit Slot Clock Configuration is 1 */
270 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16);
271 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
272 same_clock = 0;
273
274 /* Configure downstream component, all functions */
275 list_for_each_entry(child, &linkbus->devices, bus_list) {
276 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16);
277 child_reg[PCI_FUNC(child->devfn)] = reg16;
278 if (same_clock)
279 reg16 |= PCI_EXP_LNKCTL_CCC;
280 else
281 reg16 &= ~PCI_EXP_LNKCTL_CCC;
282 pcie_capability_write_word(child, PCI_EXP_LNKCTL, reg16);
283 }
284
285 /* Configure upstream component */
286 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16);
287 parent_reg = reg16;
288 if (same_clock)
289 reg16 |= PCI_EXP_LNKCTL_CCC;
290 else
291 reg16 &= ~PCI_EXP_LNKCTL_CCC;
292 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
293
294 if (pcie_retrain_link(link))
295 return;
296
297 /* Training failed. Restore common clock configurations */
298 dev_err(&parent->dev, "ASPM: Could not configure common clock\n");
299 list_for_each_entry(child, &linkbus->devices, bus_list)
300 pcie_capability_write_word(child, PCI_EXP_LNKCTL,
301 child_reg[PCI_FUNC(child->devfn)]);
302 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg);
303 }
304
305 /* Convert L0s latency encoding to ns */
calc_l0s_latency(u32 encoding)306 static u32 calc_l0s_latency(u32 encoding)
307 {
308 if (encoding == 0x7)
309 return (5 * 1000); /* > 4us */
310 return (64 << encoding);
311 }
312
313 /* Convert L0s acceptable latency encoding to ns */
calc_l0s_acceptable(u32 encoding)314 static u32 calc_l0s_acceptable(u32 encoding)
315 {
316 if (encoding == 0x7)
317 return -1U;
318 return (64 << encoding);
319 }
320
321 /* Convert L1 latency encoding to ns */
calc_l1_latency(u32 encoding)322 static u32 calc_l1_latency(u32 encoding)
323 {
324 if (encoding == 0x7)
325 return (65 * 1000); /* > 64us */
326 return (1000 << encoding);
327 }
328
329 /* Convert L1 acceptable latency encoding to ns */
calc_l1_acceptable(u32 encoding)330 static u32 calc_l1_acceptable(u32 encoding)
331 {
332 if (encoding == 0x7)
333 return -1U;
334 return (1000 << encoding);
335 }
336
337 /* Convert L1SS T_pwr encoding to usec */
calc_l1ss_pwron(struct pci_dev * pdev,u32 scale,u32 val)338 static u32 calc_l1ss_pwron(struct pci_dev *pdev, u32 scale, u32 val)
339 {
340 switch (scale) {
341 case 0:
342 return val * 2;
343 case 1:
344 return val * 10;
345 case 2:
346 return val * 100;
347 }
348 dev_err(&pdev->dev, "%s: Invalid T_PwrOn scale: %u\n",
349 __func__, scale);
350 return 0;
351 }
352
353 struct aspm_register_info {
354 u32 support:2;
355 u32 enabled:2;
356 u32 latency_encoding_l0s;
357 u32 latency_encoding_l1;
358
359 /* L1 substates */
360 u32 l1ss_cap_ptr;
361 u32 l1ss_cap;
362 u32 l1ss_ctl1;
363 u32 l1ss_ctl2;
364 };
365
pcie_get_aspm_reg(struct pci_dev * pdev,struct aspm_register_info * info)366 static void pcie_get_aspm_reg(struct pci_dev *pdev,
367 struct aspm_register_info *info)
368 {
369 u16 reg16;
370 u32 reg32;
371
372 pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, ®32);
373 info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
374 info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
375 info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
376 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, ®16);
377 info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC;
378
379 /* Read L1 PM substate capabilities */
380 info->l1ss_cap = info->l1ss_ctl1 = info->l1ss_ctl2 = 0;
381 info->l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS);
382 if (!info->l1ss_cap_ptr)
383 return;
384 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CAP,
385 &info->l1ss_cap);
386 if (!(info->l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) {
387 info->l1ss_cap = 0;
388 return;
389 }
390 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL1,
391 &info->l1ss_ctl1);
392 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL2,
393 &info->l1ss_ctl2);
394 }
395
pcie_aspm_check_latency(struct pci_dev * endpoint)396 static void pcie_aspm_check_latency(struct pci_dev *endpoint)
397 {
398 u32 latency, l1_switch_latency = 0;
399 struct aspm_latency *acceptable;
400 struct pcie_link_state *link;
401
402 /* Device not in D0 doesn't need latency check */
403 if ((endpoint->current_state != PCI_D0) &&
404 (endpoint->current_state != PCI_UNKNOWN))
405 return;
406
407 link = endpoint->bus->self->link_state;
408 acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];
409
410 while (link) {
411 /* Check upstream direction L0s latency */
412 if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&
413 (link->latency_up.l0s > acceptable->l0s))
414 link->aspm_capable &= ~ASPM_STATE_L0S_UP;
415
416 /* Check downstream direction L0s latency */
417 if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&
418 (link->latency_dw.l0s > acceptable->l0s))
419 link->aspm_capable &= ~ASPM_STATE_L0S_DW;
420 /*
421 * Check L1 latency.
422 * Every switch on the path to root complex need 1
423 * more microsecond for L1. Spec doesn't mention L0s.
424 *
425 * The exit latencies for L1 substates are not advertised
426 * by a device. Since the spec also doesn't mention a way
427 * to determine max latencies introduced by enabling L1
428 * substates on the components, it is not clear how to do
429 * a L1 substate exit latency check. We assume that the
430 * L1 exit latencies advertised by a device include L1
431 * substate latencies (and hence do not do any check).
432 */
433 latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1);
434 if ((link->aspm_capable & ASPM_STATE_L1) &&
435 (latency + l1_switch_latency > acceptable->l1))
436 link->aspm_capable &= ~ASPM_STATE_L1;
437 l1_switch_latency += 1000;
438
439 link = link->parent;
440 }
441 }
442
443 /*
444 * The L1 PM substate capability is only implemented in function 0 in a
445 * multi function device.
446 */
pci_function_0(struct pci_bus * linkbus)447 static struct pci_dev *pci_function_0(struct pci_bus *linkbus)
448 {
449 struct pci_dev *child;
450
451 list_for_each_entry(child, &linkbus->devices, bus_list)
452 if (PCI_FUNC(child->devfn) == 0)
453 return child;
454 return NULL;
455 }
456
457 /* Calculate L1.2 PM substate timing parameters */
aspm_calc_l1ss_info(struct pcie_link_state * link,struct aspm_register_info * upreg,struct aspm_register_info * dwreg)458 static void aspm_calc_l1ss_info(struct pcie_link_state *link,
459 struct aspm_register_info *upreg,
460 struct aspm_register_info *dwreg)
461 {
462 u32 val1, val2, scale1, scale2;
463
464 link->l1ss.up_cap_ptr = upreg->l1ss_cap_ptr;
465 link->l1ss.dw_cap_ptr = dwreg->l1ss_cap_ptr;
466 link->l1ss.ctl1 = link->l1ss.ctl2 = 0;
467
468 if (!(link->aspm_support & ASPM_STATE_L1_2_MASK))
469 return;
470
471 /* Choose the greater of the two T_cmn_mode_rstr_time */
472 val1 = (upreg->l1ss_cap >> 8) & 0xFF;
473 val2 = (dwreg->l1ss_cap >> 8) & 0xFF;
474 if (val1 > val2)
475 link->l1ss.ctl1 |= val1 << 8;
476 else
477 link->l1ss.ctl1 |= val2 << 8;
478 /*
479 * We currently use LTR L1.2 threshold to be fixed constant picked from
480 * Intel's coreboot.
481 */
482 link->l1ss.ctl1 |= LTR_L1_2_THRESHOLD_BITS;
483
484 /* Choose the greater of the two T_pwr_on */
485 val1 = (upreg->l1ss_cap >> 19) & 0x1F;
486 scale1 = (upreg->l1ss_cap >> 16) & 0x03;
487 val2 = (dwreg->l1ss_cap >> 19) & 0x1F;
488 scale2 = (dwreg->l1ss_cap >> 16) & 0x03;
489
490 if (calc_l1ss_pwron(link->pdev, scale1, val1) >
491 calc_l1ss_pwron(link->downstream, scale2, val2))
492 link->l1ss.ctl2 |= scale1 | (val1 << 3);
493 else
494 link->l1ss.ctl2 |= scale2 | (val2 << 3);
495 }
496
pcie_aspm_cap_init(struct pcie_link_state * link,int blacklist)497 static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
498 {
499 struct pci_dev *child = link->downstream, *parent = link->pdev;
500 struct pci_bus *linkbus = parent->subordinate;
501 struct aspm_register_info upreg, dwreg;
502
503 if (blacklist) {
504 /* Set enabled/disable so that we will disable ASPM later */
505 link->aspm_enabled = ASPM_STATE_ALL;
506 link->aspm_disable = ASPM_STATE_ALL;
507 return;
508 }
509
510 /* Get upstream/downstream components' register state */
511 pcie_get_aspm_reg(parent, &upreg);
512 pcie_get_aspm_reg(child, &dwreg);
513
514 /*
515 * If ASPM not supported, don't mess with the clocks and link,
516 * bail out now.
517 */
518 if (!(upreg.support & dwreg.support))
519 return;
520
521 /* Configure common clock before checking latencies */
522 pcie_aspm_configure_common_clock(link);
523
524 /*
525 * Re-read upstream/downstream components' register state
526 * after clock configuration
527 */
528 pcie_get_aspm_reg(parent, &upreg);
529 pcie_get_aspm_reg(child, &dwreg);
530
531 /*
532 * Setup L0s state
533 *
534 * Note that we must not enable L0s in either direction on a
535 * given link unless components on both sides of the link each
536 * support L0s.
537 */
538 if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S)
539 link->aspm_support |= ASPM_STATE_L0S;
540 if (dwreg.enabled & PCIE_LINK_STATE_L0S)
541 link->aspm_enabled |= ASPM_STATE_L0S_UP;
542 if (upreg.enabled & PCIE_LINK_STATE_L0S)
543 link->aspm_enabled |= ASPM_STATE_L0S_DW;
544 link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s);
545 link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s);
546
547 /* Setup L1 state */
548 if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1)
549 link->aspm_support |= ASPM_STATE_L1;
550 if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1)
551 link->aspm_enabled |= ASPM_STATE_L1;
552 link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1);
553 link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1);
554
555 /* Setup L1 substate */
556 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)
557 link->aspm_support |= ASPM_STATE_L1_1;
558 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)
559 link->aspm_support |= ASPM_STATE_L1_2;
560 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)
561 link->aspm_support |= ASPM_STATE_L1_1_PCIPM;
562 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)
563 link->aspm_support |= ASPM_STATE_L1_2_PCIPM;
564
565 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)
566 link->aspm_enabled |= ASPM_STATE_L1_1;
567 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)
568 link->aspm_enabled |= ASPM_STATE_L1_2;
569 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)
570 link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;
571 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)
572 link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;
573
574 if (link->aspm_support & ASPM_STATE_L1SS)
575 aspm_calc_l1ss_info(link, &upreg, &dwreg);
576
577 /* Save default state */
578 link->aspm_default = link->aspm_enabled;
579
580 /* Setup initial capable state. Will be updated later */
581 link->aspm_capable = link->aspm_support;
582 /*
583 * If the downstream component has pci bridge function, don't
584 * do ASPM for now.
585 */
586 list_for_each_entry(child, &linkbus->devices, bus_list) {
587 if (pci_pcie_type(child) == PCI_EXP_TYPE_PCI_BRIDGE) {
588 link->aspm_disable = ASPM_STATE_ALL;
589 break;
590 }
591 }
592
593 /* Get and check endpoint acceptable latencies */
594 list_for_each_entry(child, &linkbus->devices, bus_list) {
595 u32 reg32, encoding;
596 struct aspm_latency *acceptable =
597 &link->acceptable[PCI_FUNC(child->devfn)];
598
599 if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT &&
600 pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)
601 continue;
602
603 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32);
604 /* Calculate endpoint L0s acceptable latency */
605 encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
606 acceptable->l0s = calc_l0s_acceptable(encoding);
607 /* Calculate endpoint L1 acceptable latency */
608 encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
609 acceptable->l1 = calc_l1_acceptable(encoding);
610
611 pcie_aspm_check_latency(child);
612 }
613 }
614
pci_clear_and_set_dword(struct pci_dev * pdev,int pos,u32 clear,u32 set)615 static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos,
616 u32 clear, u32 set)
617 {
618 u32 val;
619
620 pci_read_config_dword(pdev, pos, &val);
621 val &= ~clear;
622 val |= set;
623 pci_write_config_dword(pdev, pos, val);
624 }
625
626 /* Configure the ASPM L1 substates */
pcie_config_aspm_l1ss(struct pcie_link_state * link,u32 state)627 static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)
628 {
629 u32 val, enable_req;
630 struct pci_dev *child = link->downstream, *parent = link->pdev;
631 u32 up_cap_ptr = link->l1ss.up_cap_ptr;
632 u32 dw_cap_ptr = link->l1ss.dw_cap_ptr;
633
634 enable_req = (link->aspm_enabled ^ state) & state;
635
636 /*
637 * Here are the rules specified in the PCIe spec for enabling L1SS:
638 * - When enabling L1.x, enable bit at parent first, then at child
639 * - When disabling L1.x, disable bit at child first, then at parent
640 * - When enabling ASPM L1.x, need to disable L1
641 * (at child followed by parent).
642 * - The ASPM/PCIPM L1.2 must be disabled while programming timing
643 * parameters
644 *
645 * To keep it simple, disable all L1SS bits first, and later enable
646 * what is needed.
647 */
648
649 /* Disable all L1 substates */
650 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1,
651 PCI_L1SS_CTL1_L1SS_MASK, 0);
652 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
653 PCI_L1SS_CTL1_L1SS_MASK, 0);
654 /*
655 * If needed, disable L1, and it gets enabled later
656 * in pcie_config_aspm_link().
657 */
658 if (enable_req & (ASPM_STATE_L1_1 | ASPM_STATE_L1_2)) {
659 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,
660 PCI_EXP_LNKCTL_ASPM_L1, 0);
661 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
662 PCI_EXP_LNKCTL_ASPM_L1, 0);
663 }
664
665 if (enable_req & ASPM_STATE_L1_2_MASK) {
666
667 /* Program T_pwr_on in both ports */
668 pci_write_config_dword(parent, up_cap_ptr + PCI_L1SS_CTL2,
669 link->l1ss.ctl2);
670 pci_write_config_dword(child, dw_cap_ptr + PCI_L1SS_CTL2,
671 link->l1ss.ctl2);
672
673 /* Program T_cmn_mode in parent */
674 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
675 0xFF00, link->l1ss.ctl1);
676
677 /* Program LTR L1.2 threshold in both ports */
678 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
679 0xE3FF0000, link->l1ss.ctl1);
680 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1,
681 0xE3FF0000, link->l1ss.ctl1);
682 }
683
684 val = 0;
685 if (state & ASPM_STATE_L1_1)
686 val |= PCI_L1SS_CTL1_ASPM_L1_1;
687 if (state & ASPM_STATE_L1_2)
688 val |= PCI_L1SS_CTL1_ASPM_L1_2;
689 if (state & ASPM_STATE_L1_1_PCIPM)
690 val |= PCI_L1SS_CTL1_PCIPM_L1_1;
691 if (state & ASPM_STATE_L1_2_PCIPM)
692 val |= PCI_L1SS_CTL1_PCIPM_L1_2;
693
694 /* Enable what we need to enable */
695 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
696 PCI_L1SS_CAP_L1_PM_SS, val);
697 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1,
698 PCI_L1SS_CAP_L1_PM_SS, val);
699 }
700
pcie_config_aspm_dev(struct pci_dev * pdev,u32 val)701 static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)
702 {
703 pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL,
704 PCI_EXP_LNKCTL_ASPMC, val);
705 }
706
pcie_config_aspm_link(struct pcie_link_state * link,u32 state)707 static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)
708 {
709 u32 upstream = 0, dwstream = 0;
710 struct pci_dev *child = link->downstream, *parent = link->pdev;
711 struct pci_bus *linkbus = parent->subordinate;
712
713 /* Enable only the states that were not explicitly disabled */
714 state &= (link->aspm_capable & ~link->aspm_disable);
715
716 /* Can't enable any substates if L1 is not enabled */
717 if (!(state & ASPM_STATE_L1))
718 state &= ~ASPM_STATE_L1SS;
719
720 /* Spec says both ports must be in D0 before enabling PCI PM substates*/
721 if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) {
722 state &= ~ASPM_STATE_L1_SS_PCIPM;
723 state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM);
724 }
725
726 /* Nothing to do if the link is already in the requested state */
727 if (link->aspm_enabled == state)
728 return;
729 /* Convert ASPM state to upstream/downstream ASPM register state */
730 if (state & ASPM_STATE_L0S_UP)
731 dwstream |= PCI_EXP_LNKCTL_ASPM_L0S;
732 if (state & ASPM_STATE_L0S_DW)
733 upstream |= PCI_EXP_LNKCTL_ASPM_L0S;
734 if (state & ASPM_STATE_L1) {
735 upstream |= PCI_EXP_LNKCTL_ASPM_L1;
736 dwstream |= PCI_EXP_LNKCTL_ASPM_L1;
737 }
738
739 if (link->aspm_capable & ASPM_STATE_L1SS)
740 pcie_config_aspm_l1ss(link, state);
741
742 /*
743 * Spec 2.0 suggests all functions should be configured the
744 * same setting for ASPM. Enabling ASPM L1 should be done in
745 * upstream component first and then downstream, and vice
746 * versa for disabling ASPM L1. Spec doesn't mention L0S.
747 */
748 if (state & ASPM_STATE_L1)
749 pcie_config_aspm_dev(parent, upstream);
750 list_for_each_entry(child, &linkbus->devices, bus_list)
751 pcie_config_aspm_dev(child, dwstream);
752 if (!(state & ASPM_STATE_L1))
753 pcie_config_aspm_dev(parent, upstream);
754
755 link->aspm_enabled = state;
756 }
757
pcie_config_aspm_path(struct pcie_link_state * link)758 static void pcie_config_aspm_path(struct pcie_link_state *link)
759 {
760 while (link) {
761 pcie_config_aspm_link(link, policy_to_aspm_state(link));
762 link = link->parent;
763 }
764 }
765
free_link_state(struct pcie_link_state * link)766 static void free_link_state(struct pcie_link_state *link)
767 {
768 link->pdev->link_state = NULL;
769 kfree(link);
770 }
771
pcie_aspm_sanity_check(struct pci_dev * pdev)772 static int pcie_aspm_sanity_check(struct pci_dev *pdev)
773 {
774 struct pci_dev *child;
775 u32 reg32;
776
777 /*
778 * Some functions in a slot might not all be PCIe functions,
779 * very strange. Disable ASPM for the whole slot
780 */
781 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {
782 if (!pci_is_pcie(child))
783 return -EINVAL;
784
785 /*
786 * If ASPM is disabled then we're not going to change
787 * the BIOS state. It's safe to continue even if it's a
788 * pre-1.1 device
789 */
790
791 if (aspm_disabled)
792 continue;
793
794 /*
795 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
796 * RBER bit to determine if a function is 1.1 version device
797 */
798 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32);
799 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
800 dev_info(&child->dev, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n");
801 return -EINVAL;
802 }
803 }
804 return 0;
805 }
806
alloc_pcie_link_state(struct pci_dev * pdev)807 static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev)
808 {
809 struct pcie_link_state *link;
810
811 link = kzalloc(sizeof(*link), GFP_KERNEL);
812 if (!link)
813 return NULL;
814
815 INIT_LIST_HEAD(&link->sibling);
816 INIT_LIST_HEAD(&link->children);
817 INIT_LIST_HEAD(&link->link);
818 link->pdev = pdev;
819 link->downstream = pci_function_0(pdev->subordinate);
820
821 /*
822 * Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe
823 * hierarchies. Note that some PCIe host implementations omit
824 * the root ports entirely, in which case a downstream port on
825 * a switch may become the root of the link state chain for all
826 * its subordinate endpoints.
827 */
828 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT ||
829 pci_pcie_type(pdev) == PCI_EXP_TYPE_PCIE_BRIDGE ||
830 !pdev->bus->parent->self) {
831 link->root = link;
832 } else {
833 struct pcie_link_state *parent;
834
835 parent = pdev->bus->parent->self->link_state;
836 if (!parent) {
837 kfree(link);
838 return NULL;
839 }
840
841 link->parent = parent;
842 link->root = link->parent->root;
843 list_add(&link->link, &parent->children);
844 }
845
846 list_add(&link->sibling, &link_list);
847 pdev->link_state = link;
848 return link;
849 }
850
851 /*
852 * pcie_aspm_init_link_state: Initiate PCI express link state.
853 * It is called after the pcie and its children devices are scanned.
854 * @pdev: the root port or switch downstream port
855 */
pcie_aspm_init_link_state(struct pci_dev * pdev)856 void pcie_aspm_init_link_state(struct pci_dev *pdev)
857 {
858 struct pcie_link_state *link;
859 int blacklist = !!pcie_aspm_sanity_check(pdev);
860
861 if (!aspm_support_enabled)
862 return;
863
864 if (pdev->link_state)
865 return;
866
867 /*
868 * We allocate pcie_link_state for the component on the upstream
869 * end of a Link, so there's nothing to do unless this device has a
870 * Link on its secondary side.
871 */
872 if (!pdev->has_secondary_link)
873 return;
874
875 /* VIA has a strange chipset, root port is under a bridge */
876 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT &&
877 pdev->bus->self)
878 return;
879
880 down_read(&pci_bus_sem);
881 if (list_empty(&pdev->subordinate->devices))
882 goto out;
883
884 mutex_lock(&aspm_lock);
885 link = alloc_pcie_link_state(pdev);
886 if (!link)
887 goto unlock;
888 /*
889 * Setup initial ASPM state. Note that we need to configure
890 * upstream links also because capable state of them can be
891 * update through pcie_aspm_cap_init().
892 */
893 pcie_aspm_cap_init(link, blacklist);
894
895 /* Setup initial Clock PM state */
896 pcie_clkpm_cap_init(link, blacklist);
897
898 /*
899 * At this stage drivers haven't had an opportunity to change the
900 * link policy setting. Enabling ASPM on broken hardware can cripple
901 * it even before the driver has had a chance to disable ASPM, so
902 * default to a safe level right now. If we're enabling ASPM beyond
903 * the BIOS's expectation, we'll do so once pci_enable_device() is
904 * called.
905 */
906 if (aspm_policy != POLICY_POWERSAVE &&
907 aspm_policy != POLICY_POWER_SUPERSAVE) {
908 pcie_config_aspm_path(link);
909 pcie_set_clkpm(link, policy_to_clkpm_state(link));
910 }
911
912 unlock:
913 mutex_unlock(&aspm_lock);
914 out:
915 up_read(&pci_bus_sem);
916 }
917
918 /* Recheck latencies and update aspm_capable for links under the root */
pcie_update_aspm_capable(struct pcie_link_state * root)919 static void pcie_update_aspm_capable(struct pcie_link_state *root)
920 {
921 struct pcie_link_state *link;
922 BUG_ON(root->parent);
923 list_for_each_entry(link, &link_list, sibling) {
924 if (link->root != root)
925 continue;
926 link->aspm_capable = link->aspm_support;
927 }
928 list_for_each_entry(link, &link_list, sibling) {
929 struct pci_dev *child;
930 struct pci_bus *linkbus = link->pdev->subordinate;
931 if (link->root != root)
932 continue;
933 list_for_each_entry(child, &linkbus->devices, bus_list) {
934 if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) &&
935 (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END))
936 continue;
937 pcie_aspm_check_latency(child);
938 }
939 }
940 }
941
942 /* @pdev: the endpoint device */
pcie_aspm_exit_link_state(struct pci_dev * pdev)943 void pcie_aspm_exit_link_state(struct pci_dev *pdev)
944 {
945 struct pci_dev *parent = pdev->bus->self;
946 struct pcie_link_state *link, *root, *parent_link;
947
948 if (!parent || !parent->link_state)
949 return;
950
951 down_read(&pci_bus_sem);
952 mutex_lock(&aspm_lock);
953 /*
954 * All PCIe functions are in one slot, remove one function will remove
955 * the whole slot, so just wait until we are the last function left.
956 */
957 if (!list_empty(&parent->subordinate->devices))
958 goto out;
959
960 link = parent->link_state;
961 root = link->root;
962 parent_link = link->parent;
963
964 /* All functions are removed, so just disable ASPM for the link */
965 pcie_config_aspm_link(link, 0);
966 list_del(&link->sibling);
967 list_del(&link->link);
968 /* Clock PM is for endpoint device */
969 free_link_state(link);
970
971 /* Recheck latencies and configure upstream links */
972 if (parent_link) {
973 pcie_update_aspm_capable(root);
974 pcie_config_aspm_path(parent_link);
975 }
976 out:
977 mutex_unlock(&aspm_lock);
978 up_read(&pci_bus_sem);
979 }
980
981 /* @pdev: the root port or switch downstream port */
pcie_aspm_pm_state_change(struct pci_dev * pdev)982 void pcie_aspm_pm_state_change(struct pci_dev *pdev)
983 {
984 struct pcie_link_state *link = pdev->link_state;
985
986 if (aspm_disabled || !link)
987 return;
988 /*
989 * Devices changed PM state, we should recheck if latency
990 * meets all functions' requirement
991 */
992 down_read(&pci_bus_sem);
993 mutex_lock(&aspm_lock);
994 pcie_update_aspm_capable(link->root);
995 pcie_config_aspm_path(link);
996 mutex_unlock(&aspm_lock);
997 up_read(&pci_bus_sem);
998 }
999
pcie_aspm_powersave_config_link(struct pci_dev * pdev)1000 void pcie_aspm_powersave_config_link(struct pci_dev *pdev)
1001 {
1002 struct pcie_link_state *link = pdev->link_state;
1003
1004 if (aspm_disabled || !link)
1005 return;
1006
1007 if (aspm_policy != POLICY_POWERSAVE &&
1008 aspm_policy != POLICY_POWER_SUPERSAVE)
1009 return;
1010
1011 down_read(&pci_bus_sem);
1012 mutex_lock(&aspm_lock);
1013 pcie_config_aspm_path(link);
1014 pcie_set_clkpm(link, policy_to_clkpm_state(link));
1015 mutex_unlock(&aspm_lock);
1016 up_read(&pci_bus_sem);
1017 }
1018
__pci_disable_link_state(struct pci_dev * pdev,int state,bool sem)1019 static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem)
1020 {
1021 struct pci_dev *parent = pdev->bus->self;
1022 struct pcie_link_state *link;
1023
1024 if (!pci_is_pcie(pdev))
1025 return;
1026
1027 if (pdev->has_secondary_link)
1028 parent = pdev;
1029 if (!parent || !parent->link_state)
1030 return;
1031
1032 /*
1033 * A driver requested that ASPM be disabled on this device, but
1034 * if we don't have permission to manage ASPM (e.g., on ACPI
1035 * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and
1036 * the _OSC method), we can't honor that request. Windows has
1037 * a similar mechanism using "PciASPMOptOut", which is also
1038 * ignored in this situation.
1039 */
1040 if (aspm_disabled) {
1041 dev_warn(&pdev->dev, "can't disable ASPM; OS doesn't have ASPM control\n");
1042 return;
1043 }
1044
1045 if (sem)
1046 down_read(&pci_bus_sem);
1047 mutex_lock(&aspm_lock);
1048 link = parent->link_state;
1049 if (state & PCIE_LINK_STATE_L0S)
1050 link->aspm_disable |= ASPM_STATE_L0S;
1051 if (state & PCIE_LINK_STATE_L1)
1052 link->aspm_disable |= ASPM_STATE_L1;
1053 pcie_config_aspm_link(link, policy_to_aspm_state(link));
1054
1055 if (state & PCIE_LINK_STATE_CLKPM) {
1056 link->clkpm_capable = 0;
1057 pcie_set_clkpm(link, 0);
1058 }
1059 mutex_unlock(&aspm_lock);
1060 if (sem)
1061 up_read(&pci_bus_sem);
1062 }
1063
pci_disable_link_state_locked(struct pci_dev * pdev,int state)1064 void pci_disable_link_state_locked(struct pci_dev *pdev, int state)
1065 {
1066 __pci_disable_link_state(pdev, state, false);
1067 }
1068 EXPORT_SYMBOL(pci_disable_link_state_locked);
1069
1070 /**
1071 * pci_disable_link_state - Disable device's link state, so the link will
1072 * never enter specific states. Note that if the BIOS didn't grant ASPM
1073 * control to the OS, this does nothing because we can't touch the LNKCTL
1074 * register.
1075 *
1076 * @pdev: PCI device
1077 * @state: ASPM link state to disable
1078 */
pci_disable_link_state(struct pci_dev * pdev,int state)1079 void pci_disable_link_state(struct pci_dev *pdev, int state)
1080 {
1081 __pci_disable_link_state(pdev, state, true);
1082 }
1083 EXPORT_SYMBOL(pci_disable_link_state);
1084
pcie_aspm_set_policy(const char * val,const struct kernel_param * kp)1085 static int pcie_aspm_set_policy(const char *val,
1086 const struct kernel_param *kp)
1087 {
1088 int i;
1089 struct pcie_link_state *link;
1090
1091 if (aspm_disabled)
1092 return -EPERM;
1093 for (i = 0; i < ARRAY_SIZE(policy_str); i++)
1094 if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
1095 break;
1096 if (i >= ARRAY_SIZE(policy_str))
1097 return -EINVAL;
1098 if (i == aspm_policy)
1099 return 0;
1100
1101 down_read(&pci_bus_sem);
1102 mutex_lock(&aspm_lock);
1103 aspm_policy = i;
1104 list_for_each_entry(link, &link_list, sibling) {
1105 pcie_config_aspm_link(link, policy_to_aspm_state(link));
1106 pcie_set_clkpm(link, policy_to_clkpm_state(link));
1107 }
1108 mutex_unlock(&aspm_lock);
1109 up_read(&pci_bus_sem);
1110 return 0;
1111 }
1112
pcie_aspm_get_policy(char * buffer,const struct kernel_param * kp)1113 static int pcie_aspm_get_policy(char *buffer, const struct kernel_param *kp)
1114 {
1115 int i, cnt = 0;
1116 for (i = 0; i < ARRAY_SIZE(policy_str); i++)
1117 if (i == aspm_policy)
1118 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
1119 else
1120 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
1121 return cnt;
1122 }
1123
1124 module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
1125 NULL, 0644);
1126
1127 #ifdef CONFIG_PCIEASPM_DEBUG
link_state_show(struct device * dev,struct device_attribute * attr,char * buf)1128 static ssize_t link_state_show(struct device *dev,
1129 struct device_attribute *attr,
1130 char *buf)
1131 {
1132 struct pci_dev *pci_device = to_pci_dev(dev);
1133 struct pcie_link_state *link_state = pci_device->link_state;
1134
1135 return sprintf(buf, "%d\n", link_state->aspm_enabled);
1136 }
1137
link_state_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t n)1138 static ssize_t link_state_store(struct device *dev,
1139 struct device_attribute *attr,
1140 const char *buf,
1141 size_t n)
1142 {
1143 struct pci_dev *pdev = to_pci_dev(dev);
1144 struct pcie_link_state *link, *root = pdev->link_state->root;
1145 u32 state;
1146
1147 if (aspm_disabled)
1148 return -EPERM;
1149
1150 if (kstrtouint(buf, 10, &state))
1151 return -EINVAL;
1152 if ((state & ~ASPM_STATE_ALL) != 0)
1153 return -EINVAL;
1154
1155 down_read(&pci_bus_sem);
1156 mutex_lock(&aspm_lock);
1157 list_for_each_entry(link, &link_list, sibling) {
1158 if (link->root != root)
1159 continue;
1160 pcie_config_aspm_link(link, state);
1161 }
1162 mutex_unlock(&aspm_lock);
1163 up_read(&pci_bus_sem);
1164 return n;
1165 }
1166
clk_ctl_show(struct device * dev,struct device_attribute * attr,char * buf)1167 static ssize_t clk_ctl_show(struct device *dev,
1168 struct device_attribute *attr,
1169 char *buf)
1170 {
1171 struct pci_dev *pci_device = to_pci_dev(dev);
1172 struct pcie_link_state *link_state = pci_device->link_state;
1173
1174 return sprintf(buf, "%d\n", link_state->clkpm_enabled);
1175 }
1176
clk_ctl_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t n)1177 static ssize_t clk_ctl_store(struct device *dev,
1178 struct device_attribute *attr,
1179 const char *buf,
1180 size_t n)
1181 {
1182 struct pci_dev *pdev = to_pci_dev(dev);
1183 bool state;
1184
1185 if (strtobool(buf, &state))
1186 return -EINVAL;
1187
1188 down_read(&pci_bus_sem);
1189 mutex_lock(&aspm_lock);
1190 pcie_set_clkpm_nocheck(pdev->link_state, state);
1191 mutex_unlock(&aspm_lock);
1192 up_read(&pci_bus_sem);
1193
1194 return n;
1195 }
1196
1197 static DEVICE_ATTR_RW(link_state);
1198 static DEVICE_ATTR_RW(clk_ctl);
1199
1200 static char power_group[] = "power";
pcie_aspm_create_sysfs_dev_files(struct pci_dev * pdev)1201 void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
1202 {
1203 struct pcie_link_state *link_state = pdev->link_state;
1204
1205 if (!link_state)
1206 return;
1207
1208 if (link_state->aspm_support)
1209 sysfs_add_file_to_group(&pdev->dev.kobj,
1210 &dev_attr_link_state.attr, power_group);
1211 if (link_state->clkpm_capable)
1212 sysfs_add_file_to_group(&pdev->dev.kobj,
1213 &dev_attr_clk_ctl.attr, power_group);
1214 }
1215
pcie_aspm_remove_sysfs_dev_files(struct pci_dev * pdev)1216 void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
1217 {
1218 struct pcie_link_state *link_state = pdev->link_state;
1219
1220 if (!link_state)
1221 return;
1222
1223 if (link_state->aspm_support)
1224 sysfs_remove_file_from_group(&pdev->dev.kobj,
1225 &dev_attr_link_state.attr, power_group);
1226 if (link_state->clkpm_capable)
1227 sysfs_remove_file_from_group(&pdev->dev.kobj,
1228 &dev_attr_clk_ctl.attr, power_group);
1229 }
1230 #endif
1231
pcie_aspm_disable(char * str)1232 static int __init pcie_aspm_disable(char *str)
1233 {
1234 if (!strcmp(str, "off")) {
1235 aspm_policy = POLICY_DEFAULT;
1236 aspm_disabled = 1;
1237 aspm_support_enabled = false;
1238 printk(KERN_INFO "PCIe ASPM is disabled\n");
1239 } else if (!strcmp(str, "force")) {
1240 aspm_force = 1;
1241 printk(KERN_INFO "PCIe ASPM is forcibly enabled\n");
1242 }
1243 return 1;
1244 }
1245
1246 __setup("pcie_aspm=", pcie_aspm_disable);
1247
pcie_no_aspm(void)1248 void pcie_no_aspm(void)
1249 {
1250 /*
1251 * Disabling ASPM is intended to prevent the kernel from modifying
1252 * existing hardware state, not to clear existing state. To that end:
1253 * (a) set policy to POLICY_DEFAULT in order to avoid changing state
1254 * (b) prevent userspace from changing policy
1255 */
1256 if (!aspm_force) {
1257 aspm_policy = POLICY_DEFAULT;
1258 aspm_disabled = 1;
1259 }
1260 }
1261
pcie_aspm_support_enabled(void)1262 bool pcie_aspm_support_enabled(void)
1263 {
1264 return aspm_support_enabled;
1265 }
1266 EXPORT_SYMBOL(pcie_aspm_support_enabled);
1267