1 /*
2 This file is provided under a dual BSD/GPLv2 license. When using or
3 redistributing this file, you may do so under either license.
4
5 GPL LICENSE SUMMARY
6 Copyright(c) 2014 Intel Corporation.
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of version 2 of the GNU General Public License as
9 published by the Free Software Foundation.
10
11 This program is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 General Public License for more details.
15
16 Contact Information:
17 qat-linux@intel.com
18
19 BSD LICENSE
20 Copyright(c) 2014 Intel Corporation.
21 Redistribution and use in source and binary forms, with or without
22 modification, are permitted provided that the following conditions
23 are met:
24
25 * Redistributions of source code must retain the above copyright
26 notice, this list of conditions and the following disclaimer.
27 * Redistributions in binary form must reproduce the above copyright
28 notice, this list of conditions and the following disclaimer in
29 the documentation and/or other materials provided with the
30 distribution.
31 * Neither the name of Intel Corporation nor the names of its
32 contributors may be used to endorse or promote products derived
33 from this software without specific prior written permission.
34
35 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46 */
47 #include <linux/kernel.h>
48 #include <linux/module.h>
49 #include <linux/pci.h>
50 #include <linux/init.h>
51 #include <linux/types.h>
52 #include <linux/fs.h>
53 #include <linux/slab.h>
54 #include <linux/errno.h>
55 #include <linux/device.h>
56 #include <linux/dma-mapping.h>
57 #include <linux/platform_device.h>
58 #include <linux/workqueue.h>
59 #include <linux/io.h>
60 #include <adf_accel_devices.h>
61 #include <adf_common_drv.h>
62 #include <adf_cfg.h>
63 #include <adf_transport_access_macros.h>
64 #include "adf_dh895xccvf_hw_data.h"
65 #include "adf_drv.h"
66
67 static const char adf_driver_name[] = ADF_DH895XCCVF_DEVICE_NAME;
68
69 #define ADF_SYSTEM_DEVICE(device_id) \
70 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
71
72 static const struct pci_device_id adf_pci_tbl[] = {
73 ADF_SYSTEM_DEVICE(ADF_DH895XCCIOV_PCI_DEVICE_ID),
74 {0,}
75 };
76 MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
77
78 static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent);
79 static void adf_remove(struct pci_dev *dev);
80
81 static struct pci_driver adf_driver = {
82 .id_table = adf_pci_tbl,
83 .name = adf_driver_name,
84 .probe = adf_probe,
85 .remove = adf_remove,
86 };
87
adf_cleanup_pci_dev(struct adf_accel_dev * accel_dev)88 static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev)
89 {
90 pci_release_regions(accel_dev->accel_pci_dev.pci_dev);
91 pci_disable_device(accel_dev->accel_pci_dev.pci_dev);
92 }
93
adf_cleanup_accel(struct adf_accel_dev * accel_dev)94 static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
95 {
96 struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev;
97 struct adf_accel_dev *pf;
98 int i;
99
100 for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
101 struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
102
103 if (bar->virt_addr)
104 pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr);
105 }
106
107 if (accel_dev->hw_device) {
108 switch (accel_pci_dev->pci_dev->device) {
109 case ADF_DH895XCCIOV_PCI_DEVICE_ID:
110 adf_clean_hw_data_dh895xcciov(accel_dev->hw_device);
111 break;
112 default:
113 break;
114 }
115 kfree(accel_dev->hw_device);
116 accel_dev->hw_device = NULL;
117 }
118 adf_cfg_dev_remove(accel_dev);
119 debugfs_remove(accel_dev->debugfs_dir);
120 pf = adf_devmgr_pci_to_accel_dev(accel_pci_dev->pci_dev->physfn);
121 adf_devmgr_rm_dev(accel_dev, pf);
122 }
123
adf_dev_configure(struct adf_accel_dev * accel_dev)124 static int adf_dev_configure(struct adf_accel_dev *accel_dev)
125 {
126 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
127 unsigned long val, bank = 0;
128
129 if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC))
130 goto err;
131 if (adf_cfg_section_add(accel_dev, "Accelerator0"))
132 goto err;
133
134 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, 0);
135 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key,
136 (void *)&bank, ADF_DEC))
137 goto err;
138
139 val = bank;
140 snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, 0);
141 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key,
142 (void *)&val, ADF_DEC))
143 goto err;
144
145 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, 0);
146
147 val = 128;
148 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key,
149 (void *)&val, ADF_DEC))
150 goto err;
151
152 val = 512;
153 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, 0);
154 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
155 key, (void *)&val, ADF_DEC))
156 goto err;
157
158 val = 0;
159 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, 0);
160 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
161 key, (void *)&val, ADF_DEC))
162 goto err;
163
164 val = 2;
165 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, 0);
166 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
167 key, (void *)&val, ADF_DEC))
168 goto err;
169
170 val = 8;
171 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, 0);
172 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
173 key, (void *)&val, ADF_DEC))
174 goto err;
175
176 val = 10;
177 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, 0);
178 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
179 key, (void *)&val, ADF_DEC))
180 goto err;
181
182 val = ADF_COALESCING_DEF_TIME;
183 snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT,
184 (int)bank);
185 if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0",
186 key, (void *)&val, ADF_DEC))
187 goto err;
188
189 val = 1;
190 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
191 ADF_NUM_CY, (void *)&val, ADF_DEC))
192 goto err;
193
194 set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
195 return 0;
196 err:
197 dev_err(&GET_DEV(accel_dev), "Failed to configure QAT accel dev\n");
198 return -EINVAL;
199 }
200
adf_probe(struct pci_dev * pdev,const struct pci_device_id * ent)201 static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
202 {
203 struct adf_accel_dev *accel_dev;
204 struct adf_accel_dev *pf;
205 struct adf_accel_pci *accel_pci_dev;
206 struct adf_hw_device_data *hw_data;
207 char name[ADF_DEVICE_NAME_LENGTH];
208 unsigned int i, bar_nr;
209 int ret, bar_mask;
210
211 switch (ent->device) {
212 case ADF_DH895XCCIOV_PCI_DEVICE_ID:
213 break;
214 default:
215 dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
216 return -ENODEV;
217 }
218
219 accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL,
220 dev_to_node(&pdev->dev));
221 if (!accel_dev)
222 return -ENOMEM;
223
224 accel_dev->is_vf = true;
225 pf = adf_devmgr_pci_to_accel_dev(pdev->physfn);
226 accel_pci_dev = &accel_dev->accel_pci_dev;
227 accel_pci_dev->pci_dev = pdev;
228
229 /* Add accel device to accel table */
230 if (adf_devmgr_add_dev(accel_dev, pf)) {
231 dev_err(&pdev->dev, "Failed to add new accelerator device.\n");
232 kfree(accel_dev);
233 return -EFAULT;
234 }
235 INIT_LIST_HEAD(&accel_dev->crypto_list);
236
237 accel_dev->owner = THIS_MODULE;
238 /* Allocate and configure device configuration structure */
239 hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL,
240 dev_to_node(&pdev->dev));
241 if (!hw_data) {
242 ret = -ENOMEM;
243 goto out_err;
244 }
245 accel_dev->hw_device = hw_data;
246 switch (ent->device) {
247 case ADF_DH895XCCIOV_PCI_DEVICE_ID:
248 adf_init_hw_data_dh895xcciov(accel_dev->hw_device);
249 break;
250 default:
251 ret = -ENODEV;
252 goto out_err;
253 }
254
255 /* Get Accelerators and Accelerators Engines masks */
256 hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses);
257 hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses);
258 accel_pci_dev->sku = hw_data->get_sku(hw_data);
259
260 /* Create dev top level debugfs entry */
261 snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d",
262 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
263 pdev->bus->number, PCI_SLOT(pdev->devfn),
264 PCI_FUNC(pdev->devfn));
265
266 accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
267 if (!accel_dev->debugfs_dir) {
268 dev_err(&pdev->dev, "Could not create debugfs dir %s\n", name);
269 ret = -EINVAL;
270 goto out_err;
271 }
272
273 /* Create device configuration table */
274 ret = adf_cfg_dev_add(accel_dev);
275 if (ret)
276 goto out_err;
277
278 /* enable PCI device */
279 if (pci_enable_device(pdev)) {
280 ret = -EFAULT;
281 goto out_err;
282 }
283
284 /* set dma identifier */
285 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
286 if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) {
287 dev_err(&pdev->dev, "No usable DMA configuration\n");
288 ret = -EFAULT;
289 goto out_err_disable;
290 } else {
291 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
292 }
293
294 } else {
295 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
296 }
297
298 if (pci_request_regions(pdev, adf_driver_name)) {
299 ret = -EFAULT;
300 goto out_err_disable;
301 }
302
303 /* Find and map all the device's BARS */
304 i = 0;
305 bar_mask = pci_select_bars(pdev, IORESOURCE_MEM);
306 for_each_set_bit(bar_nr, (const unsigned long *)&bar_mask,
307 ADF_PCI_MAX_BARS * 2) {
308 struct adf_bar *bar = &accel_pci_dev->pci_bars[i++];
309
310 bar->base_addr = pci_resource_start(pdev, bar_nr);
311 if (!bar->base_addr)
312 break;
313 bar->size = pci_resource_len(pdev, bar_nr);
314 bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0);
315 if (!bar->virt_addr) {
316 dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr);
317 ret = -EFAULT;
318 goto out_err_free_reg;
319 }
320 }
321 pci_set_master(pdev);
322 /* Completion for VF2PF request/response message exchange */
323 init_completion(&accel_dev->vf.iov_msg_completion);
324
325 ret = adf_dev_configure(accel_dev);
326 if (ret)
327 goto out_err_free_reg;
328
329 ret = adf_dev_init(accel_dev);
330 if (ret)
331 goto out_err_dev_shutdown;
332
333 ret = adf_dev_start(accel_dev);
334 if (ret)
335 goto out_err_dev_stop;
336
337 return ret;
338
339 out_err_dev_stop:
340 adf_dev_stop(accel_dev);
341 out_err_dev_shutdown:
342 adf_dev_shutdown(accel_dev);
343 out_err_free_reg:
344 pci_release_regions(accel_pci_dev->pci_dev);
345 out_err_disable:
346 pci_disable_device(accel_pci_dev->pci_dev);
347 out_err:
348 adf_cleanup_accel(accel_dev);
349 kfree(accel_dev);
350 return ret;
351 }
352
adf_remove(struct pci_dev * pdev)353 static void adf_remove(struct pci_dev *pdev)
354 {
355 struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);
356
357 if (!accel_dev) {
358 pr_err("QAT: Driver removal failed\n");
359 return;
360 }
361 if (adf_dev_stop(accel_dev))
362 dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n");
363
364 adf_dev_shutdown(accel_dev);
365 adf_cleanup_accel(accel_dev);
366 adf_cleanup_pci_dev(accel_dev);
367 kfree(accel_dev);
368 }
369
adfdrv_init(void)370 static int __init adfdrv_init(void)
371 {
372 request_module("intel_qat");
373
374 if (pci_register_driver(&adf_driver)) {
375 pr_err("QAT: Driver initialization failed\n");
376 return -EFAULT;
377 }
378 return 0;
379 }
380
adfdrv_release(void)381 static void __exit adfdrv_release(void)
382 {
383 pci_unregister_driver(&adf_driver);
384 adf_clean_vf_map(true);
385 }
386
387 module_init(adfdrv_init);
388 module_exit(adfdrv_release);
389
390 MODULE_LICENSE("Dual BSD/GPL");
391 MODULE_AUTHOR("Intel");
392 MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
393 MODULE_VERSION(ADF_DRV_VERSION);
394