1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/kernel.h>
4 #include <linux/init.h>
5 #include <linux/types.h>
6 #include <linux/pci.h>
7 #include <linux/slab.h>
8 #include <linux/errno.h>
9 #include <linux/interrupt.h>
10 #include "adf_accel_devices.h"
11 #include "adf_common_drv.h"
12 #include "adf_cfg.h"
13 #include "adf_cfg_strings.h"
14 #include "adf_cfg_common.h"
15 #include "adf_transport_access_macros.h"
16 #include "adf_transport_internal.h"
17
18 #define ADF_MAX_NUM_VFS 32
19
adf_enable_msix(struct adf_accel_dev * accel_dev)20 static int adf_enable_msix(struct adf_accel_dev *accel_dev)
21 {
22 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
23 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
24 u32 msix_num_entries = 1;
25
26 /* If SR-IOV is disabled, add entries for each bank */
27 if (!accel_dev->pf.vf_info) {
28 int i;
29
30 msix_num_entries += hw_data->num_banks;
31 for (i = 0; i < msix_num_entries; i++)
32 pci_dev_info->msix_entries.entries[i].entry = i;
33 } else {
34 pci_dev_info->msix_entries.entries[0].entry =
35 hw_data->num_banks;
36 }
37
38 if (pci_enable_msix_exact(pci_dev_info->pci_dev,
39 pci_dev_info->msix_entries.entries,
40 msix_num_entries)) {
41 dev_err(&GET_DEV(accel_dev), "Failed to enable MSI-X IRQ(s)\n");
42 return -EFAULT;
43 }
44 return 0;
45 }
46
adf_disable_msix(struct adf_accel_pci * pci_dev_info)47 static void adf_disable_msix(struct adf_accel_pci *pci_dev_info)
48 {
49 pci_disable_msix(pci_dev_info->pci_dev);
50 }
51
adf_msix_isr_bundle(int irq,void * bank_ptr)52 static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr)
53 {
54 struct adf_etr_bank_data *bank = bank_ptr;
55
56 WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, 0);
57 tasklet_hi_schedule(&bank->resp_handler);
58 return IRQ_HANDLED;
59 }
60
adf_msix_isr_ae(int irq,void * dev_ptr)61 static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr)
62 {
63 struct adf_accel_dev *accel_dev = dev_ptr;
64
65 #ifdef CONFIG_PCI_IOV
66 /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */
67 if (accel_dev->pf.vf_info) {
68 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
69 struct adf_bar *pmisc =
70 &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
71 void __iomem *pmisc_bar_addr = pmisc->virt_addr;
72 unsigned long vf_mask;
73
74 /* Get the interrupt sources triggered by VFs */
75 vf_mask = ((ADF_CSR_RD(pmisc_bar_addr, ADF_ERRSOU5) &
76 0x0000FFFF) << 16) |
77 ((ADF_CSR_RD(pmisc_bar_addr, ADF_ERRSOU3) &
78 0x01FFFE00) >> 9);
79
80 if (vf_mask) {
81 struct adf_accel_vf_info *vf_info;
82 bool irq_handled = false;
83 int i;
84
85 /* Disable VF2PF interrupts for VFs with pending ints */
86 adf_disable_vf2pf_interrupts(accel_dev, vf_mask);
87
88 /*
89 * Schedule tasklets to handle VF2PF interrupt BHs
90 * unless the VF is malicious and is attempting to
91 * flood the host OS with VF2PF interrupts.
92 */
93 for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) {
94 vf_info = accel_dev->pf.vf_info + i;
95
96 if (!__ratelimit(&vf_info->vf2pf_ratelimit)) {
97 dev_info(&GET_DEV(accel_dev),
98 "Too many ints from VF%d\n",
99 vf_info->vf_nr + 1);
100 continue;
101 }
102
103 /* Tasklet will re-enable ints from this VF */
104 tasklet_hi_schedule(&vf_info->vf2pf_bh_tasklet);
105 irq_handled = true;
106 }
107
108 if (irq_handled)
109 return IRQ_HANDLED;
110 }
111 }
112 #endif /* CONFIG_PCI_IOV */
113
114 dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n",
115 accel_dev->accel_id);
116
117 return IRQ_NONE;
118 }
119
adf_request_irqs(struct adf_accel_dev * accel_dev)120 static int adf_request_irqs(struct adf_accel_dev *accel_dev)
121 {
122 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
123 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
124 struct msix_entry *msixe = pci_dev_info->msix_entries.entries;
125 struct adf_etr_data *etr_data = accel_dev->transport;
126 int ret, i = 0;
127 char *name;
128
129 /* Request msix irq for all banks unless SR-IOV enabled */
130 if (!accel_dev->pf.vf_info) {
131 for (i = 0; i < hw_data->num_banks; i++) {
132 struct adf_etr_bank_data *bank = &etr_data->banks[i];
133 unsigned int cpu, cpus = num_online_cpus();
134
135 name = *(pci_dev_info->msix_entries.names + i);
136 snprintf(name, ADF_MAX_MSIX_VECTOR_NAME,
137 "qat%d-bundle%d", accel_dev->accel_id, i);
138 ret = request_irq(msixe[i].vector,
139 adf_msix_isr_bundle, 0, name, bank);
140 if (ret) {
141 dev_err(&GET_DEV(accel_dev),
142 "failed to enable irq %d for %s\n",
143 msixe[i].vector, name);
144 return ret;
145 }
146
147 cpu = ((accel_dev->accel_id * hw_data->num_banks) +
148 i) % cpus;
149 irq_set_affinity_hint(msixe[i].vector,
150 get_cpu_mask(cpu));
151 }
152 }
153
154 /* Request msix irq for AE */
155 name = *(pci_dev_info->msix_entries.names + i);
156 snprintf(name, ADF_MAX_MSIX_VECTOR_NAME,
157 "qat%d-ae-cluster", accel_dev->accel_id);
158 ret = request_irq(msixe[i].vector, adf_msix_isr_ae, 0, name, accel_dev);
159 if (ret) {
160 dev_err(&GET_DEV(accel_dev),
161 "failed to enable irq %d, for %s\n",
162 msixe[i].vector, name);
163 return ret;
164 }
165 return ret;
166 }
167
adf_free_irqs(struct adf_accel_dev * accel_dev)168 static void adf_free_irqs(struct adf_accel_dev *accel_dev)
169 {
170 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
171 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
172 struct msix_entry *msixe = pci_dev_info->msix_entries.entries;
173 struct adf_etr_data *etr_data = accel_dev->transport;
174 int i = 0;
175
176 if (pci_dev_info->msix_entries.num_entries > 1) {
177 for (i = 0; i < hw_data->num_banks; i++) {
178 irq_set_affinity_hint(msixe[i].vector, NULL);
179 free_irq(msixe[i].vector, &etr_data->banks[i]);
180 }
181 }
182 irq_set_affinity_hint(msixe[i].vector, NULL);
183 free_irq(msixe[i].vector, accel_dev);
184 }
185
adf_isr_alloc_msix_entry_table(struct adf_accel_dev * accel_dev)186 static int adf_isr_alloc_msix_entry_table(struct adf_accel_dev *accel_dev)
187 {
188 int i;
189 char **names;
190 struct msix_entry *entries;
191 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
192 u32 msix_num_entries = 1;
193
194 /* If SR-IOV is disabled (vf_info is NULL), add entries for each bank */
195 if (!accel_dev->pf.vf_info)
196 msix_num_entries += hw_data->num_banks;
197
198 entries = kcalloc_node(msix_num_entries, sizeof(*entries),
199 GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev)));
200 if (!entries)
201 return -ENOMEM;
202
203 names = kcalloc(msix_num_entries, sizeof(char *), GFP_KERNEL);
204 if (!names) {
205 kfree(entries);
206 return -ENOMEM;
207 }
208 for (i = 0; i < msix_num_entries; i++) {
209 *(names + i) = kzalloc(ADF_MAX_MSIX_VECTOR_NAME, GFP_KERNEL);
210 if (!(*(names + i)))
211 goto err;
212 }
213 accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries;
214 accel_dev->accel_pci_dev.msix_entries.entries = entries;
215 accel_dev->accel_pci_dev.msix_entries.names = names;
216 return 0;
217 err:
218 for (i = 0; i < msix_num_entries; i++)
219 kfree(*(names + i));
220 kfree(entries);
221 kfree(names);
222 return -ENOMEM;
223 }
224
adf_isr_free_msix_entry_table(struct adf_accel_dev * accel_dev)225 static void adf_isr_free_msix_entry_table(struct adf_accel_dev *accel_dev)
226 {
227 char **names = accel_dev->accel_pci_dev.msix_entries.names;
228 int i;
229
230 kfree(accel_dev->accel_pci_dev.msix_entries.entries);
231 for (i = 0; i < accel_dev->accel_pci_dev.msix_entries.num_entries; i++)
232 kfree(*(names + i));
233 kfree(names);
234 }
235
adf_setup_bh(struct adf_accel_dev * accel_dev)236 static int adf_setup_bh(struct adf_accel_dev *accel_dev)
237 {
238 struct adf_etr_data *priv_data = accel_dev->transport;
239 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
240 int i;
241
242 for (i = 0; i < hw_data->num_banks; i++)
243 tasklet_init(&priv_data->banks[i].resp_handler,
244 adf_response_handler,
245 (unsigned long)&priv_data->banks[i]);
246 return 0;
247 }
248
adf_cleanup_bh(struct adf_accel_dev * accel_dev)249 static void adf_cleanup_bh(struct adf_accel_dev *accel_dev)
250 {
251 struct adf_etr_data *priv_data = accel_dev->transport;
252 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
253 int i;
254
255 for (i = 0; i < hw_data->num_banks; i++) {
256 tasklet_disable(&priv_data->banks[i].resp_handler);
257 tasklet_kill(&priv_data->banks[i].resp_handler);
258 }
259 }
260
261 /**
262 * adf_isr_resource_free() - Free IRQ for acceleration device
263 * @accel_dev: Pointer to acceleration device.
264 *
265 * Function frees interrupts for acceleration device.
266 */
adf_isr_resource_free(struct adf_accel_dev * accel_dev)267 void adf_isr_resource_free(struct adf_accel_dev *accel_dev)
268 {
269 adf_free_irqs(accel_dev);
270 adf_cleanup_bh(accel_dev);
271 adf_disable_msix(&accel_dev->accel_pci_dev);
272 adf_isr_free_msix_entry_table(accel_dev);
273 }
274 EXPORT_SYMBOL_GPL(adf_isr_resource_free);
275
276 /**
277 * adf_isr_resource_alloc() - Allocate IRQ for acceleration device
278 * @accel_dev: Pointer to acceleration device.
279 *
280 * Function allocates interrupts for acceleration device.
281 *
282 * Return: 0 on success, error code otherwise.
283 */
adf_isr_resource_alloc(struct adf_accel_dev * accel_dev)284 int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev)
285 {
286 int ret;
287
288 ret = adf_isr_alloc_msix_entry_table(accel_dev);
289 if (ret)
290 goto err_out;
291
292 ret = adf_enable_msix(accel_dev);
293 if (ret)
294 goto err_free_msix_table;
295
296 ret = adf_setup_bh(accel_dev);
297 if (ret)
298 goto err_disable_msix;
299
300 ret = adf_request_irqs(accel_dev);
301 if (ret)
302 goto err_cleanup_bh;
303
304 return 0;
305
306 err_cleanup_bh:
307 adf_cleanup_bh(accel_dev);
308
309 err_disable_msix:
310 adf_disable_msix(&accel_dev->accel_pci_dev);
311
312 err_free_msix_table:
313 adf_isr_free_msix_entry_table(accel_dev);
314
315 err_out:
316 return ret;
317 }
318 EXPORT_SYMBOL_GPL(adf_isr_resource_alloc);
319