1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/kernel.h>
4 #include <linux/init.h>
5 #include <linux/types.h>
6 #include <linux/pci.h>
7 #include <linux/slab.h>
8 #include <linux/errno.h>
9 #include <linux/interrupt.h>
10 #include <linux/workqueue.h>
11 #include "adf_accel_devices.h"
12 #include "adf_common_drv.h"
13 #include "adf_cfg.h"
14 #include "adf_cfg_strings.h"
15 #include "adf_cfg_common.h"
16 #include "adf_transport_access_macros.h"
17 #include "adf_transport_internal.h"
18 #include "adf_pf2vf_msg.h"
19
20 #define ADF_VINTSOU_OFFSET 0x204
21 #define ADF_VINTSOU_BUN BIT(0)
22 #define ADF_VINTSOU_PF2VF BIT(1)
23
24 static struct workqueue_struct *adf_vf_stop_wq;
25
26 struct adf_vf_stop_data {
27 struct adf_accel_dev *accel_dev;
28 struct work_struct work;
29 };
30
adf_enable_msi(struct adf_accel_dev * accel_dev)31 static int adf_enable_msi(struct adf_accel_dev *accel_dev)
32 {
33 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;
34 int stat = pci_enable_msi(pci_dev_info->pci_dev);
35
36 if (stat) {
37 dev_err(&GET_DEV(accel_dev),
38 "Failed to enable MSI interrupts\n");
39 return stat;
40 }
41
42 accel_dev->vf.irq_name = kzalloc(ADF_MAX_MSIX_VECTOR_NAME, GFP_KERNEL);
43 if (!accel_dev->vf.irq_name)
44 return -ENOMEM;
45
46 return stat;
47 }
48
adf_disable_msi(struct adf_accel_dev * accel_dev)49 static void adf_disable_msi(struct adf_accel_dev *accel_dev)
50 {
51 struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
52
53 kfree(accel_dev->vf.irq_name);
54 pci_disable_msi(pdev);
55 }
56
adf_dev_stop_async(struct work_struct * work)57 static void adf_dev_stop_async(struct work_struct *work)
58 {
59 struct adf_vf_stop_data *stop_data =
60 container_of(work, struct adf_vf_stop_data, work);
61 struct adf_accel_dev *accel_dev = stop_data->accel_dev;
62
63 adf_dev_stop(accel_dev);
64 adf_dev_shutdown(accel_dev);
65
66 /* Re-enable PF2VF interrupts */
67 adf_enable_pf2vf_interrupts(accel_dev);
68 kfree(stop_data);
69 }
70
adf_pf2vf_bh_handler(void * data)71 static void adf_pf2vf_bh_handler(void *data)
72 {
73 struct adf_accel_dev *accel_dev = data;
74 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
75 struct adf_bar *pmisc =
76 &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
77 void __iomem *pmisc_bar_addr = pmisc->virt_addr;
78 u32 msg;
79
80 /* Read the message from PF */
81 msg = ADF_CSR_RD(pmisc_bar_addr, hw_data->get_pf2vf_offset(0));
82 if (!(msg & ADF_PF2VF_INT)) {
83 dev_info(&GET_DEV(accel_dev),
84 "Spurious PF2VF interrupt, msg %X. Ignored\n", msg);
85 goto out;
86 }
87
88 if (!(msg & ADF_PF2VF_MSGORIGIN_SYSTEM))
89 /* Ignore legacy non-system (non-kernel) PF2VF messages */
90 goto err;
91
92 switch ((msg & ADF_PF2VF_MSGTYPE_MASK) >> ADF_PF2VF_MSGTYPE_SHIFT) {
93 case ADF_PF2VF_MSGTYPE_RESTARTING: {
94 struct adf_vf_stop_data *stop_data;
95
96 dev_dbg(&GET_DEV(accel_dev),
97 "Restarting msg received from PF 0x%x\n", msg);
98
99 clear_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status);
100
101 stop_data = kzalloc(sizeof(*stop_data), GFP_ATOMIC);
102 if (!stop_data) {
103 dev_err(&GET_DEV(accel_dev),
104 "Couldn't schedule stop for vf_%d\n",
105 accel_dev->accel_id);
106 return;
107 }
108 stop_data->accel_dev = accel_dev;
109 INIT_WORK(&stop_data->work, adf_dev_stop_async);
110 queue_work(adf_vf_stop_wq, &stop_data->work);
111 /* To ack, clear the PF2VFINT bit */
112 msg &= ~ADF_PF2VF_INT;
113 ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg);
114 return;
115 }
116 case ADF_PF2VF_MSGTYPE_VERSION_RESP:
117 dev_dbg(&GET_DEV(accel_dev),
118 "Version resp received from PF 0x%x\n", msg);
119 accel_dev->vf.pf_version =
120 (msg & ADF_PF2VF_VERSION_RESP_VERS_MASK) >>
121 ADF_PF2VF_VERSION_RESP_VERS_SHIFT;
122 accel_dev->vf.compatible =
123 (msg & ADF_PF2VF_VERSION_RESP_RESULT_MASK) >>
124 ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
125 complete(&accel_dev->vf.iov_msg_completion);
126 break;
127 default:
128 goto err;
129 }
130
131 /* To ack, clear the PF2VFINT bit */
132 msg &= ~ADF_PF2VF_INT;
133 ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg);
134
135 out:
136 /* Re-enable PF2VF interrupts */
137 adf_enable_pf2vf_interrupts(accel_dev);
138 return;
139 err:
140 dev_err(&GET_DEV(accel_dev),
141 "Unknown message from PF (0x%x); leaving PF2VF ints disabled\n",
142 msg);
143 }
144
adf_setup_pf2vf_bh(struct adf_accel_dev * accel_dev)145 static int adf_setup_pf2vf_bh(struct adf_accel_dev *accel_dev)
146 {
147 tasklet_init(&accel_dev->vf.pf2vf_bh_tasklet,
148 (void *)adf_pf2vf_bh_handler, (unsigned long)accel_dev);
149
150 mutex_init(&accel_dev->vf.vf2pf_lock);
151 return 0;
152 }
153
adf_cleanup_pf2vf_bh(struct adf_accel_dev * accel_dev)154 static void adf_cleanup_pf2vf_bh(struct adf_accel_dev *accel_dev)
155 {
156 tasklet_disable(&accel_dev->vf.pf2vf_bh_tasklet);
157 tasklet_kill(&accel_dev->vf.pf2vf_bh_tasklet);
158 mutex_destroy(&accel_dev->vf.vf2pf_lock);
159 }
160
adf_isr(int irq,void * privdata)161 static irqreturn_t adf_isr(int irq, void *privdata)
162 {
163 struct adf_accel_dev *accel_dev = privdata;
164 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
165 struct adf_bar *pmisc =
166 &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
167 void __iomem *pmisc_bar_addr = pmisc->virt_addr;
168 bool handled = false;
169 u32 v_int;
170
171 /* Read VF INT source CSR to determine the source of VF interrupt */
172 v_int = ADF_CSR_RD(pmisc_bar_addr, ADF_VINTSOU_OFFSET);
173
174 /* Check for PF2VF interrupt */
175 if (v_int & ADF_VINTSOU_PF2VF) {
176 /* Disable PF to VF interrupt */
177 adf_disable_pf2vf_interrupts(accel_dev);
178
179 /* Schedule tasklet to handle interrupt BH */
180 tasklet_hi_schedule(&accel_dev->vf.pf2vf_bh_tasklet);
181 handled = true;
182 }
183
184 /* Check bundle interrupt */
185 if (v_int & ADF_VINTSOU_BUN) {
186 struct adf_etr_data *etr_data = accel_dev->transport;
187 struct adf_etr_bank_data *bank = &etr_data->banks[0];
188
189 /* Disable Flag and Coalesce Ring Interrupts */
190 WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number,
191 0);
192 tasklet_hi_schedule(&bank->resp_handler);
193 handled = true;
194 }
195
196 return handled ? IRQ_HANDLED : IRQ_NONE;
197 }
198
adf_request_msi_irq(struct adf_accel_dev * accel_dev)199 static int adf_request_msi_irq(struct adf_accel_dev *accel_dev)
200 {
201 struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
202 unsigned int cpu;
203 int ret;
204
205 snprintf(accel_dev->vf.irq_name, ADF_MAX_MSIX_VECTOR_NAME,
206 "qat_%02x:%02d.%02d", pdev->bus->number, PCI_SLOT(pdev->devfn),
207 PCI_FUNC(pdev->devfn));
208 ret = request_irq(pdev->irq, adf_isr, 0, accel_dev->vf.irq_name,
209 (void *)accel_dev);
210 if (ret) {
211 dev_err(&GET_DEV(accel_dev), "failed to enable irq for %s\n",
212 accel_dev->vf.irq_name);
213 return ret;
214 }
215 cpu = accel_dev->accel_id % num_online_cpus();
216 irq_set_affinity_hint(pdev->irq, get_cpu_mask(cpu));
217
218 return ret;
219 }
220
adf_setup_bh(struct adf_accel_dev * accel_dev)221 static int adf_setup_bh(struct adf_accel_dev *accel_dev)
222 {
223 struct adf_etr_data *priv_data = accel_dev->transport;
224
225 tasklet_init(&priv_data->banks[0].resp_handler, adf_response_handler,
226 (unsigned long)priv_data->banks);
227 return 0;
228 }
229
adf_cleanup_bh(struct adf_accel_dev * accel_dev)230 static void adf_cleanup_bh(struct adf_accel_dev *accel_dev)
231 {
232 struct adf_etr_data *priv_data = accel_dev->transport;
233
234 tasklet_disable(&priv_data->banks[0].resp_handler);
235 tasklet_kill(&priv_data->banks[0].resp_handler);
236 }
237
238 /**
239 * adf_vf_isr_resource_free() - Free IRQ for acceleration device
240 * @accel_dev: Pointer to acceleration device.
241 *
242 * Function frees interrupts for acceleration device virtual function.
243 */
adf_vf_isr_resource_free(struct adf_accel_dev * accel_dev)244 void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev)
245 {
246 struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
247
248 irq_set_affinity_hint(pdev->irq, NULL);
249 free_irq(pdev->irq, (void *)accel_dev);
250 adf_cleanup_bh(accel_dev);
251 adf_cleanup_pf2vf_bh(accel_dev);
252 adf_disable_msi(accel_dev);
253 }
254 EXPORT_SYMBOL_GPL(adf_vf_isr_resource_free);
255
256 /**
257 * adf_vf_isr_resource_alloc() - Allocate IRQ for acceleration device
258 * @accel_dev: Pointer to acceleration device.
259 *
260 * Function allocates interrupts for acceleration device virtual function.
261 *
262 * Return: 0 on success, error code otherwise.
263 */
adf_vf_isr_resource_alloc(struct adf_accel_dev * accel_dev)264 int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev)
265 {
266 if (adf_enable_msi(accel_dev))
267 goto err_out;
268
269 if (adf_setup_pf2vf_bh(accel_dev))
270 goto err_disable_msi;
271
272 if (adf_setup_bh(accel_dev))
273 goto err_cleanup_pf2vf_bh;
274
275 if (adf_request_msi_irq(accel_dev))
276 goto err_cleanup_bh;
277
278 return 0;
279
280 err_cleanup_bh:
281 adf_cleanup_bh(accel_dev);
282
283 err_cleanup_pf2vf_bh:
284 adf_cleanup_pf2vf_bh(accel_dev);
285
286 err_disable_msi:
287 adf_disable_msi(accel_dev);
288
289 err_out:
290 return -EFAULT;
291 }
292 EXPORT_SYMBOL_GPL(adf_vf_isr_resource_alloc);
293
adf_init_vf_wq(void)294 int __init adf_init_vf_wq(void)
295 {
296 adf_vf_stop_wq = alloc_workqueue("adf_vf_stop_wq", WQ_MEM_RECLAIM, 0);
297
298 return !adf_vf_stop_wq ? -EFAULT : 0;
299 }
300
adf_exit_vf_wq(void)301 void adf_exit_vf_wq(void)
302 {
303 if (adf_vf_stop_wq)
304 destroy_workqueue(adf_vf_stop_wq);
305
306 adf_vf_stop_wq = NULL;
307 }
308