1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/kernel.h>
4 #include <linux/init.h>
5 #include <linux/types.h>
6 #include <linux/pci.h>
7 #include <linux/slab.h>
8 #include <linux/errno.h>
9 #include <linux/interrupt.h>
10 #include <linux/workqueue.h>
11 #include "adf_accel_devices.h"
12 #include "adf_common_drv.h"
13 #include "adf_cfg.h"
14 #include "adf_cfg_strings.h"
15 #include "adf_cfg_common.h"
16 #include "adf_transport_access_macros.h"
17 #include "adf_transport_internal.h"
18 #include "adf_pf2vf_msg.h"
19
20 #define ADF_VINTSOU_OFFSET 0x204
21 #define ADF_VINTMSK_OFFSET 0x208
22 #define ADF_VINTSOU_BUN BIT(0)
23 #define ADF_VINTSOU_PF2VF BIT(1)
24
25 static struct workqueue_struct *adf_vf_stop_wq;
26
27 struct adf_vf_stop_data {
28 struct adf_accel_dev *accel_dev;
29 struct work_struct work;
30 };
31
adf_enable_pf2vf_interrupts(struct adf_accel_dev * accel_dev)32 void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev)
33 {
34 struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
35 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
36 void __iomem *pmisc_bar_addr =
37 pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr;
38
39 ADF_CSR_WR(pmisc_bar_addr, ADF_VINTMSK_OFFSET, 0x0);
40 }
41
adf_disable_pf2vf_interrupts(struct adf_accel_dev * accel_dev)42 void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev)
43 {
44 struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
45 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
46 void __iomem *pmisc_bar_addr =
47 pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr;
48
49 ADF_CSR_WR(pmisc_bar_addr, ADF_VINTMSK_OFFSET, 0x2);
50 }
51 EXPORT_SYMBOL_GPL(adf_disable_pf2vf_interrupts);
52
adf_enable_msi(struct adf_accel_dev * accel_dev)53 static int adf_enable_msi(struct adf_accel_dev *accel_dev)
54 {
55 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
56 int stat = pci_enable_msi(pci_dev_info->pci_dev);
57
58 if (stat) {
59 dev_err(&GET_DEV(accel_dev),
60 "Failed to enable MSI interrupts\n");
61 return stat;
62 }
63
64 accel_dev->vf.irq_name = kzalloc(ADF_MAX_MSIX_VECTOR_NAME, GFP_KERNEL);
65 if (!accel_dev->vf.irq_name)
66 return -ENOMEM;
67
68 return stat;
69 }
70
adf_disable_msi(struct adf_accel_dev * accel_dev)71 static void adf_disable_msi(struct adf_accel_dev *accel_dev)
72 {
73 struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
74
75 kfree(accel_dev->vf.irq_name);
76 pci_disable_msi(pdev);
77 }
78
adf_dev_stop_async(struct work_struct * work)79 static void adf_dev_stop_async(struct work_struct *work)
80 {
81 struct adf_vf_stop_data *stop_data =
82 container_of(work, struct adf_vf_stop_data, work);
83 struct adf_accel_dev *accel_dev = stop_data->accel_dev;
84
85 adf_dev_stop(accel_dev);
86 adf_dev_shutdown(accel_dev);
87
88 /* Re-enable PF2VF interrupts */
89 adf_enable_pf2vf_interrupts(accel_dev);
90 kfree(stop_data);
91 }
92
adf_pf2vf_bh_handler(void * data)93 static void adf_pf2vf_bh_handler(void *data)
94 {
95 struct adf_accel_dev *accel_dev = data;
96 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
97 struct adf_bar *pmisc =
98 &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
99 void __iomem *pmisc_bar_addr = pmisc->virt_addr;
100 u32 msg;
101
102 /* Read the message from PF */
103 msg = ADF_CSR_RD(pmisc_bar_addr, hw_data->get_pf2vf_offset(0));
104 if (!(msg & ADF_PF2VF_INT)) {
105 dev_info(&GET_DEV(accel_dev),
106 "Spurious PF2VF interrupt, msg %X. Ignored\n", msg);
107 goto out;
108 }
109
110 if (!(msg & ADF_PF2VF_MSGORIGIN_SYSTEM))
111 /* Ignore legacy non-system (non-kernel) PF2VF messages */
112 goto err;
113
114 switch ((msg & ADF_PF2VF_MSGTYPE_MASK) >> ADF_PF2VF_MSGTYPE_SHIFT) {
115 case ADF_PF2VF_MSGTYPE_RESTARTING: {
116 struct adf_vf_stop_data *stop_data;
117
118 dev_dbg(&GET_DEV(accel_dev),
119 "Restarting msg received from PF 0x%x\n", msg);
120
121 clear_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status);
122
123 stop_data = kzalloc(sizeof(*stop_data), GFP_ATOMIC);
124 if (!stop_data) {
125 dev_err(&GET_DEV(accel_dev),
126 "Couldn't schedule stop for vf_%d\n",
127 accel_dev->accel_id);
128 return;
129 }
130 stop_data->accel_dev = accel_dev;
131 INIT_WORK(&stop_data->work, adf_dev_stop_async);
132 queue_work(adf_vf_stop_wq, &stop_data->work);
133 /* To ack, clear the PF2VFINT bit */
134 msg &= ~ADF_PF2VF_INT;
135 ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg);
136 return;
137 }
138 case ADF_PF2VF_MSGTYPE_VERSION_RESP:
139 dev_dbg(&GET_DEV(accel_dev),
140 "Version resp received from PF 0x%x\n", msg);
141 accel_dev->vf.pf_version =
142 (msg & ADF_PF2VF_VERSION_RESP_VERS_MASK) >>
143 ADF_PF2VF_VERSION_RESP_VERS_SHIFT;
144 accel_dev->vf.compatible =
145 (msg & ADF_PF2VF_VERSION_RESP_RESULT_MASK) >>
146 ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
147 complete(&accel_dev->vf.iov_msg_completion);
148 break;
149 default:
150 goto err;
151 }
152
153 /* To ack, clear the PF2VFINT bit */
154 msg &= ~ADF_PF2VF_INT;
155 ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg);
156
157 out:
158 /* Re-enable PF2VF interrupts */
159 adf_enable_pf2vf_interrupts(accel_dev);
160 return;
161 err:
162 dev_err(&GET_DEV(accel_dev),
163 "Unknown message from PF (0x%x); leaving PF2VF ints disabled\n",
164 msg);
165 }
166
adf_setup_pf2vf_bh(struct adf_accel_dev * accel_dev)167 static int adf_setup_pf2vf_bh(struct adf_accel_dev *accel_dev)
168 {
169 tasklet_init(&accel_dev->vf.pf2vf_bh_tasklet,
170 (void *)adf_pf2vf_bh_handler, (unsigned long)accel_dev);
171
172 mutex_init(&accel_dev->vf.vf2pf_lock);
173 return 0;
174 }
175
adf_cleanup_pf2vf_bh(struct adf_accel_dev * accel_dev)176 static void adf_cleanup_pf2vf_bh(struct adf_accel_dev *accel_dev)
177 {
178 tasklet_disable(&accel_dev->vf.pf2vf_bh_tasklet);
179 tasklet_kill(&accel_dev->vf.pf2vf_bh_tasklet);
180 mutex_destroy(&accel_dev->vf.vf2pf_lock);
181 }
182
adf_isr(int irq,void * privdata)183 static irqreturn_t adf_isr(int irq, void *privdata)
184 {
185 struct adf_accel_dev *accel_dev = privdata;
186 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
187 struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops;
188 struct adf_bar *pmisc =
189 &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
190 void __iomem *pmisc_bar_addr = pmisc->virt_addr;
191 bool handled = false;
192 u32 v_int, v_mask;
193
194 /* Read VF INT source CSR to determine the source of VF interrupt */
195 v_int = ADF_CSR_RD(pmisc_bar_addr, ADF_VINTSOU_OFFSET);
196
197 /* Read VF INT mask CSR to determine which sources are masked */
198 v_mask = ADF_CSR_RD(pmisc_bar_addr, ADF_VINTMSK_OFFSET);
199
200 /*
201 * Recompute v_int ignoring sources that are masked. This is to
202 * avoid rescheduling the tasklet for interrupts already handled
203 */
204 v_int &= ~v_mask;
205
206 /* Check for PF2VF interrupt */
207 if (v_int & ADF_VINTSOU_PF2VF) {
208 /* Disable PF to VF interrupt */
209 adf_disable_pf2vf_interrupts(accel_dev);
210
211 /* Schedule tasklet to handle interrupt BH */
212 tasklet_hi_schedule(&accel_dev->vf.pf2vf_bh_tasklet);
213 handled = true;
214 }
215
216 /* Check bundle interrupt */
217 if (v_int & ADF_VINTSOU_BUN) {
218 struct adf_etr_data *etr_data = accel_dev->transport;
219 struct adf_etr_bank_data *bank = &etr_data->banks[0];
220
221 /* Disable Flag and Coalesce Ring Interrupts */
222 csr_ops->write_csr_int_flag_and_col(bank->csr_addr,
223 bank->bank_number, 0);
224 tasklet_hi_schedule(&bank->resp_handler);
225 handled = true;
226 }
227
228 return handled ? IRQ_HANDLED : IRQ_NONE;
229 }
230
adf_request_msi_irq(struct adf_accel_dev * accel_dev)231 static int adf_request_msi_irq(struct adf_accel_dev *accel_dev)
232 {
233 struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
234 unsigned int cpu;
235 int ret;
236
237 snprintf(accel_dev->vf.irq_name, ADF_MAX_MSIX_VECTOR_NAME,
238 "qat_%02x:%02d.%02d", pdev->bus->number, PCI_SLOT(pdev->devfn),
239 PCI_FUNC(pdev->devfn));
240 ret = request_irq(pdev->irq, adf_isr, 0, accel_dev->vf.irq_name,
241 (void *)accel_dev);
242 if (ret) {
243 dev_err(&GET_DEV(accel_dev), "failed to enable irq for %s\n",
244 accel_dev->vf.irq_name);
245 return ret;
246 }
247 cpu = accel_dev->accel_id % num_online_cpus();
248 irq_set_affinity_hint(pdev->irq, get_cpu_mask(cpu));
249
250 return ret;
251 }
252
adf_setup_bh(struct adf_accel_dev * accel_dev)253 static int adf_setup_bh(struct adf_accel_dev *accel_dev)
254 {
255 struct adf_etr_data *priv_data = accel_dev->transport;
256
257 tasklet_init(&priv_data->banks[0].resp_handler, adf_response_handler,
258 (unsigned long)priv_data->banks);
259 return 0;
260 }
261
adf_cleanup_bh(struct adf_accel_dev * accel_dev)262 static void adf_cleanup_bh(struct adf_accel_dev *accel_dev)
263 {
264 struct adf_etr_data *priv_data = accel_dev->transport;
265
266 tasklet_disable(&priv_data->banks[0].resp_handler);
267 tasklet_kill(&priv_data->banks[0].resp_handler);
268 }
269
270 /**
271 * adf_vf_isr_resource_free() - Free IRQ for acceleration device
272 * @accel_dev: Pointer to acceleration device.
273 *
274 * Function frees interrupts for acceleration device virtual function.
275 */
adf_vf_isr_resource_free(struct adf_accel_dev * accel_dev)276 void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev)
277 {
278 struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
279
280 irq_set_affinity_hint(pdev->irq, NULL);
281 free_irq(pdev->irq, (void *)accel_dev);
282 adf_cleanup_bh(accel_dev);
283 adf_cleanup_pf2vf_bh(accel_dev);
284 adf_disable_msi(accel_dev);
285 }
286 EXPORT_SYMBOL_GPL(adf_vf_isr_resource_free);
287
288 /**
289 * adf_vf_isr_resource_alloc() - Allocate IRQ for acceleration device
290 * @accel_dev: Pointer to acceleration device.
291 *
292 * Function allocates interrupts for acceleration device virtual function.
293 *
294 * Return: 0 on success, error code otherwise.
295 */
adf_vf_isr_resource_alloc(struct adf_accel_dev * accel_dev)296 int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev)
297 {
298 if (adf_enable_msi(accel_dev))
299 goto err_out;
300
301 if (adf_setup_pf2vf_bh(accel_dev))
302 goto err_disable_msi;
303
304 if (adf_setup_bh(accel_dev))
305 goto err_cleanup_pf2vf_bh;
306
307 if (adf_request_msi_irq(accel_dev))
308 goto err_cleanup_bh;
309
310 return 0;
311
312 err_cleanup_bh:
313 adf_cleanup_bh(accel_dev);
314
315 err_cleanup_pf2vf_bh:
316 adf_cleanup_pf2vf_bh(accel_dev);
317
318 err_disable_msi:
319 adf_disable_msi(accel_dev);
320
321 err_out:
322 return -EFAULT;
323 }
324 EXPORT_SYMBOL_GPL(adf_vf_isr_resource_alloc);
325
326 /**
327 * adf_flush_vf_wq() - Flush workqueue for VF
328 * @accel_dev: Pointer to acceleration device.
329 *
330 * Function disables the PF/VF interrupts on the VF so that no new messages
331 * are received and flushes the workqueue 'adf_vf_stop_wq'.
332 *
333 * Return: void.
334 */
adf_flush_vf_wq(struct adf_accel_dev * accel_dev)335 void adf_flush_vf_wq(struct adf_accel_dev *accel_dev)
336 {
337 adf_disable_pf2vf_interrupts(accel_dev);
338
339 flush_workqueue(adf_vf_stop_wq);
340 }
341 EXPORT_SYMBOL_GPL(adf_flush_vf_wq);
342
343 /**
344 * adf_init_vf_wq() - Init workqueue for VF
345 *
346 * Function init workqueue 'adf_vf_stop_wq' for VF.
347 *
348 * Return: 0 on success, error code otherwise.
349 */
adf_init_vf_wq(void)350 int __init adf_init_vf_wq(void)
351 {
352 adf_vf_stop_wq = alloc_workqueue("adf_vf_stop_wq", WQ_MEM_RECLAIM, 0);
353
354 return !adf_vf_stop_wq ? -EFAULT : 0;
355 }
356
adf_exit_vf_wq(void)357 void adf_exit_vf_wq(void)
358 {
359 if (adf_vf_stop_wq)
360 destroy_workqueue(adf_vf_stop_wq);
361
362 adf_vf_stop_wq = NULL;
363 }
364