1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
2 /*
3 * Copyright(c) 2019 Intel Corporation.
4 *
5 */
6
7 #include "aspm.h"
8
9 /* Time after which the timer interrupt will re-enable ASPM */
10 #define ASPM_TIMER_MS 1000
11 /* Time for which interrupts are ignored after a timer has been scheduled */
12 #define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2)
13 /* Two interrupts within this time trigger ASPM disable */
14 #define ASPM_TRIGGER_MS 1
15 #define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull)
16 #define ASPM_L1_SUPPORTED(reg) \
17 ((((reg) & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2)
18
19 uint aspm_mode = ASPM_MODE_DISABLED;
20 module_param_named(aspm, aspm_mode, uint, 0444);
21 MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic");
22
aspm_hw_l1_supported(struct hfi1_devdata * dd)23 static bool aspm_hw_l1_supported(struct hfi1_devdata *dd)
24 {
25 struct pci_dev *parent = dd->pcidev->bus->self;
26 u32 up, dn;
27
28 /*
29 * If the driver does not have access to the upstream component,
30 * it cannot support ASPM L1 at all.
31 */
32 if (!parent)
33 return false;
34
35 pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn);
36 dn = ASPM_L1_SUPPORTED(dn);
37
38 pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up);
39 up = ASPM_L1_SUPPORTED(up);
40
41 /* ASPM works on A-step but is reported as not supported */
42 return (!!dn || is_ax(dd)) && !!up;
43 }
44
45 /* Set L1 entrance latency for slower entry to L1 */
aspm_hw_set_l1_ent_latency(struct hfi1_devdata * dd)46 static void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd)
47 {
48 u32 l1_ent_lat = 0x4u;
49 u32 reg32;
50
51 pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, ®32);
52 reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK;
53 reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT;
54 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32);
55 }
56
aspm_hw_enable_l1(struct hfi1_devdata * dd)57 static void aspm_hw_enable_l1(struct hfi1_devdata *dd)
58 {
59 struct pci_dev *parent = dd->pcidev->bus->self;
60
61 /*
62 * If the driver does not have access to the upstream component,
63 * it cannot support ASPM L1 at all.
64 */
65 if (!parent)
66 return;
67
68 /* Enable ASPM L1 first in upstream component and then downstream */
69 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
70 PCI_EXP_LNKCTL_ASPMC,
71 PCI_EXP_LNKCTL_ASPM_L1);
72 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
73 PCI_EXP_LNKCTL_ASPMC,
74 PCI_EXP_LNKCTL_ASPM_L1);
75 }
76
aspm_hw_disable_l1(struct hfi1_devdata * dd)77 void aspm_hw_disable_l1(struct hfi1_devdata *dd)
78 {
79 struct pci_dev *parent = dd->pcidev->bus->self;
80
81 /* Disable ASPM L1 first in downstream component and then upstream */
82 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
83 PCI_EXP_LNKCTL_ASPMC, 0x0);
84 if (parent)
85 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
86 PCI_EXP_LNKCTL_ASPMC, 0x0);
87 }
88
aspm_enable(struct hfi1_devdata * dd)89 static void aspm_enable(struct hfi1_devdata *dd)
90 {
91 if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED ||
92 !dd->aspm_supported)
93 return;
94
95 aspm_hw_enable_l1(dd);
96 dd->aspm_enabled = true;
97 }
98
aspm_disable(struct hfi1_devdata * dd)99 static void aspm_disable(struct hfi1_devdata *dd)
100 {
101 if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED)
102 return;
103
104 aspm_hw_disable_l1(dd);
105 dd->aspm_enabled = false;
106 }
107
aspm_disable_inc(struct hfi1_devdata * dd)108 static void aspm_disable_inc(struct hfi1_devdata *dd)
109 {
110 unsigned long flags;
111
112 spin_lock_irqsave(&dd->aspm_lock, flags);
113 aspm_disable(dd);
114 atomic_inc(&dd->aspm_disabled_cnt);
115 spin_unlock_irqrestore(&dd->aspm_lock, flags);
116 }
117
aspm_enable_dec(struct hfi1_devdata * dd)118 static void aspm_enable_dec(struct hfi1_devdata *dd)
119 {
120 unsigned long flags;
121
122 spin_lock_irqsave(&dd->aspm_lock, flags);
123 if (atomic_dec_and_test(&dd->aspm_disabled_cnt))
124 aspm_enable(dd);
125 spin_unlock_irqrestore(&dd->aspm_lock, flags);
126 }
127
128 /* ASPM processing for each receive context interrupt */
__aspm_ctx_disable(struct hfi1_ctxtdata * rcd)129 void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd)
130 {
131 bool restart_timer;
132 bool close_interrupts;
133 unsigned long flags;
134 ktime_t now, prev;
135
136 spin_lock_irqsave(&rcd->aspm_lock, flags);
137 /* PSM contexts are open */
138 if (!rcd->aspm_intr_enable)
139 goto unlock;
140
141 prev = rcd->aspm_ts_last_intr;
142 now = ktime_get();
143 rcd->aspm_ts_last_intr = now;
144
145 /* An interrupt pair close together in time */
146 close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS;
147
148 /* Don't push out our timer till this much time has elapsed */
149 restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) >
150 ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC;
151 restart_timer = restart_timer && close_interrupts;
152
153 /* Disable ASPM and schedule timer */
154 if (rcd->aspm_enabled && close_interrupts) {
155 aspm_disable_inc(rcd->dd);
156 rcd->aspm_enabled = false;
157 restart_timer = true;
158 }
159
160 if (restart_timer) {
161 mod_timer(&rcd->aspm_timer,
162 jiffies + msecs_to_jiffies(ASPM_TIMER_MS));
163 rcd->aspm_ts_timer_sched = now;
164 }
165 unlock:
166 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
167 }
168
169 /* Timer function for re-enabling ASPM in the absence of interrupt activity */
aspm_ctx_timer_function(struct timer_list * t)170 static void aspm_ctx_timer_function(struct timer_list *t)
171 {
172 struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer);
173 unsigned long flags;
174
175 spin_lock_irqsave(&rcd->aspm_lock, flags);
176 aspm_enable_dec(rcd->dd);
177 rcd->aspm_enabled = true;
178 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
179 }
180
181 /*
182 * Disable interrupt processing for verbs contexts when PSM or VNIC contexts
183 * are open.
184 */
aspm_disable_all(struct hfi1_devdata * dd)185 void aspm_disable_all(struct hfi1_devdata *dd)
186 {
187 struct hfi1_ctxtdata *rcd;
188 unsigned long flags;
189 u16 i;
190
191 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
192 rcd = hfi1_rcd_get_by_index(dd, i);
193 if (rcd) {
194 del_timer_sync(&rcd->aspm_timer);
195 spin_lock_irqsave(&rcd->aspm_lock, flags);
196 rcd->aspm_intr_enable = false;
197 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
198 hfi1_rcd_put(rcd);
199 }
200 }
201
202 aspm_disable(dd);
203 atomic_set(&dd->aspm_disabled_cnt, 0);
204 }
205
206 /* Re-enable interrupt processing for verbs contexts */
aspm_enable_all(struct hfi1_devdata * dd)207 void aspm_enable_all(struct hfi1_devdata *dd)
208 {
209 struct hfi1_ctxtdata *rcd;
210 unsigned long flags;
211 u16 i;
212
213 aspm_enable(dd);
214
215 if (aspm_mode != ASPM_MODE_DYNAMIC)
216 return;
217
218 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
219 rcd = hfi1_rcd_get_by_index(dd, i);
220 if (rcd) {
221 spin_lock_irqsave(&rcd->aspm_lock, flags);
222 rcd->aspm_intr_enable = true;
223 rcd->aspm_enabled = true;
224 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
225 hfi1_rcd_put(rcd);
226 }
227 }
228 }
229
aspm_ctx_init(struct hfi1_ctxtdata * rcd)230 static void aspm_ctx_init(struct hfi1_ctxtdata *rcd)
231 {
232 spin_lock_init(&rcd->aspm_lock);
233 timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0);
234 rcd->aspm_intr_supported = rcd->dd->aspm_supported &&
235 aspm_mode == ASPM_MODE_DYNAMIC &&
236 rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt;
237 }
238
aspm_init(struct hfi1_devdata * dd)239 void aspm_init(struct hfi1_devdata *dd)
240 {
241 struct hfi1_ctxtdata *rcd;
242 u16 i;
243
244 spin_lock_init(&dd->aspm_lock);
245 dd->aspm_supported = aspm_hw_l1_supported(dd);
246
247 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
248 rcd = hfi1_rcd_get_by_index(dd, i);
249 if (rcd)
250 aspm_ctx_init(rcd);
251 hfi1_rcd_put(rcd);
252 }
253
254 /* Start with ASPM disabled */
255 aspm_hw_set_l1_ent_latency(dd);
256 dd->aspm_enabled = false;
257 aspm_hw_disable_l1(dd);
258
259 /* Now turn on ASPM if configured */
260 aspm_enable_all(dd);
261 }
262
aspm_exit(struct hfi1_devdata * dd)263 void aspm_exit(struct hfi1_devdata *dd)
264 {
265 aspm_disable_all(dd);
266
267 /* Turn on ASPM on exit to conserve power */
268 aspm_enable(dd);
269 }
270
271