1 /*
2 This file is provided under a dual BSD/GPLv2 license. When using or
3 redistributing this file, you may do so under either license.
4
5 GPL LICENSE SUMMARY
6 Copyright(c) 2014 Intel Corporation.
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of version 2 of the GNU General Public License as
9 published by the Free Software Foundation.
10
11 This program is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 General Public License for more details.
15
16 Contact Information:
17 qat-linux@intel.com
18
19 BSD LICENSE
20 Copyright(c) 2014 Intel Corporation.
21 Redistribution and use in source and binary forms, with or without
22 modification, are permitted provided that the following conditions
23 are met:
24
25 * Redistributions of source code must retain the above copyright
26 notice, this list of conditions and the following disclaimer.
27 * Redistributions in binary form must reproduce the above copyright
28 notice, this list of conditions and the following disclaimer in
29 the documentation and/or other materials provided with the
30 distribution.
31 * Neither the name of Intel Corporation nor the names of its
32 contributors may be used to endorse or promote products derived
33 from this software without specific prior written permission.
34
35 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46 */
47 #include <linux/kernel.h>
48 #include <linux/module.h>
49 #include <linux/pci.h>
50 #include <linux/init.h>
51 #include <linux/types.h>
52 #include <linux/fs.h>
53 #include <linux/slab.h>
54 #include <linux/errno.h>
55 #include <linux/device.h>
56 #include <linux/dma-mapping.h>
57 #include <linux/platform_device.h>
58 #include <linux/workqueue.h>
59 #include <linux/io.h>
60 #include <adf_accel_devices.h>
61 #include <adf_common_drv.h>
62 #include <adf_cfg.h>
63 #include "adf_c3xxx_hw_data.h"
64
65 #define ADF_SYSTEM_DEVICE(device_id) \
66 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
67
68 static const struct pci_device_id adf_pci_tbl[] = {
69 ADF_SYSTEM_DEVICE(ADF_C3XXX_PCI_DEVICE_ID),
70 {0,}
71 };
72 MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
73
74 static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent);
75 static void adf_remove(struct pci_dev *dev);
76
77 static struct pci_driver adf_driver = {
78 .id_table = adf_pci_tbl,
79 .name = ADF_C3XXX_DEVICE_NAME,
80 .probe = adf_probe,
81 .remove = adf_remove,
82 .sriov_configure = adf_sriov_configure,
83 };
84
adf_cleanup_pci_dev(struct adf_accel_dev * accel_dev)85 static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev)
86 {
87 pci_release_regions(accel_dev->accel_pci_dev.pci_dev);
88 pci_disable_device(accel_dev->accel_pci_dev.pci_dev);
89 }
90
adf_cleanup_accel(struct adf_accel_dev * accel_dev)91 static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
92 {
93 struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev;
94 int i;
95
96 for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
97 struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
98
99 if (bar->virt_addr)
100 pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr);
101 }
102
103 if (accel_dev->hw_device) {
104 switch (accel_pci_dev->pci_dev->device) {
105 case ADF_C3XXX_PCI_DEVICE_ID:
106 adf_clean_hw_data_c3xxx(accel_dev->hw_device);
107 break;
108 default:
109 break;
110 }
111 kfree(accel_dev->hw_device);
112 accel_dev->hw_device = NULL;
113 }
114 adf_cfg_dev_remove(accel_dev);
115 debugfs_remove(accel_dev->debugfs_dir);
116 adf_devmgr_rm_dev(accel_dev, NULL);
117 }
118
adf_probe(struct pci_dev * pdev,const struct pci_device_id * ent)119 static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
120 {
121 struct adf_accel_dev *accel_dev;
122 struct adf_accel_pci *accel_pci_dev;
123 struct adf_hw_device_data *hw_data;
124 char name[ADF_DEVICE_NAME_LENGTH];
125 unsigned int i, bar_nr;
126 unsigned long bar_mask;
127 int ret;
128
129 switch (ent->device) {
130 case ADF_C3XXX_PCI_DEVICE_ID:
131 break;
132 default:
133 dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
134 return -ENODEV;
135 }
136
137 if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) {
138 /* If the accelerator is connected to a node with no memory
139 * there is no point in using the accelerator since the remote
140 * memory transaction will be very slow. */
141 dev_err(&pdev->dev, "Invalid NUMA configuration.\n");
142 return -EINVAL;
143 }
144
145 accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL,
146 dev_to_node(&pdev->dev));
147 if (!accel_dev)
148 return -ENOMEM;
149
150 INIT_LIST_HEAD(&accel_dev->crypto_list);
151 accel_pci_dev = &accel_dev->accel_pci_dev;
152 accel_pci_dev->pci_dev = pdev;
153
154 /* Add accel device to accel table.
155 * This should be called before adf_cleanup_accel is called */
156 if (adf_devmgr_add_dev(accel_dev, NULL)) {
157 dev_err(&pdev->dev, "Failed to add new accelerator device.\n");
158 kfree(accel_dev);
159 return -EFAULT;
160 }
161
162 accel_dev->owner = THIS_MODULE;
163 /* Allocate and configure device configuration structure */
164 hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL,
165 dev_to_node(&pdev->dev));
166 if (!hw_data) {
167 ret = -ENOMEM;
168 goto out_err;
169 }
170
171 accel_dev->hw_device = hw_data;
172 adf_init_hw_data_c3xxx(accel_dev->hw_device);
173 pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid);
174 pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET,
175 &hw_data->fuses);
176
177 /* Get Accelerators and Accelerators Engines masks */
178 hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses);
179 hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses);
180 accel_pci_dev->sku = hw_data->get_sku(hw_data);
181 /* If the device has no acceleration engines then ignore it. */
182 if (!hw_data->accel_mask || !hw_data->ae_mask ||
183 ((~hw_data->ae_mask) & 0x01)) {
184 dev_err(&pdev->dev, "No acceleration units found");
185 ret = -EFAULT;
186 goto out_err;
187 }
188
189 /* Create dev top level debugfs entry */
190 snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d",
191 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
192 pdev->bus->number, PCI_SLOT(pdev->devfn),
193 PCI_FUNC(pdev->devfn));
194
195 accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
196 if (!accel_dev->debugfs_dir) {
197 dev_err(&pdev->dev, "Could not create debugfs dir %s\n", name);
198 ret = -EINVAL;
199 goto out_err;
200 }
201
202 /* Create device configuration table */
203 ret = adf_cfg_dev_add(accel_dev);
204 if (ret)
205 goto out_err;
206
207 /* enable PCI device */
208 if (pci_enable_device(pdev)) {
209 ret = -EFAULT;
210 goto out_err;
211 }
212
213 /* set dma identifier */
214 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
215 if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) {
216 dev_err(&pdev->dev, "No usable DMA configuration\n");
217 ret = -EFAULT;
218 goto out_err_disable;
219 } else {
220 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
221 }
222
223 } else {
224 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
225 }
226
227 if (pci_request_regions(pdev, ADF_C3XXX_DEVICE_NAME)) {
228 ret = -EFAULT;
229 goto out_err_disable;
230 }
231
232 /* Read accelerator capabilities mask */
233 pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET,
234 &hw_data->accel_capabilities_mask);
235
236 /* Find and map all the device's BARS */
237 i = 0;
238 bar_mask = pci_select_bars(pdev, IORESOURCE_MEM);
239 for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) {
240 struct adf_bar *bar = &accel_pci_dev->pci_bars[i++];
241
242 bar->base_addr = pci_resource_start(pdev, bar_nr);
243 if (!bar->base_addr)
244 break;
245 bar->size = pci_resource_len(pdev, bar_nr);
246 bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0);
247 if (!bar->virt_addr) {
248 dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr);
249 ret = -EFAULT;
250 goto out_err_free_reg;
251 }
252 }
253 pci_set_master(pdev);
254
255 if (adf_enable_aer(accel_dev, &adf_driver)) {
256 dev_err(&pdev->dev, "Failed to enable aer\n");
257 ret = -EFAULT;
258 goto out_err_free_reg;
259 }
260
261 if (pci_save_state(pdev)) {
262 dev_err(&pdev->dev, "Failed to save pci state\n");
263 ret = -ENOMEM;
264 goto out_err_free_reg;
265 }
266
267 ret = qat_crypto_dev_config(accel_dev);
268 if (ret)
269 goto out_err_free_reg;
270
271 ret = adf_dev_init(accel_dev);
272 if (ret)
273 goto out_err_dev_shutdown;
274
275 ret = adf_dev_start(accel_dev);
276 if (ret)
277 goto out_err_dev_stop;
278
279 return ret;
280
281 out_err_dev_stop:
282 adf_dev_stop(accel_dev);
283 out_err_dev_shutdown:
284 adf_dev_shutdown(accel_dev);
285 out_err_free_reg:
286 pci_release_regions(accel_pci_dev->pci_dev);
287 out_err_disable:
288 pci_disable_device(accel_pci_dev->pci_dev);
289 out_err:
290 adf_cleanup_accel(accel_dev);
291 kfree(accel_dev);
292 return ret;
293 }
294
adf_remove(struct pci_dev * pdev)295 static void adf_remove(struct pci_dev *pdev)
296 {
297 struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);
298
299 if (!accel_dev) {
300 pr_err("QAT: Driver removal failed\n");
301 return;
302 }
303 adf_dev_stop(accel_dev);
304 adf_dev_shutdown(accel_dev);
305 adf_disable_aer(accel_dev);
306 adf_cleanup_accel(accel_dev);
307 adf_cleanup_pci_dev(accel_dev);
308 kfree(accel_dev);
309 }
310
adfdrv_init(void)311 static int __init adfdrv_init(void)
312 {
313 request_module("intel_qat");
314
315 if (pci_register_driver(&adf_driver)) {
316 pr_err("QAT: Driver initialization failed\n");
317 return -EFAULT;
318 }
319 return 0;
320 }
321
adfdrv_release(void)322 static void __exit adfdrv_release(void)
323 {
324 pci_unregister_driver(&adf_driver);
325 }
326
327 module_init(adfdrv_init);
328 module_exit(adfdrv_release);
329
330 MODULE_LICENSE("Dual BSD/GPL");
331 MODULE_AUTHOR("Intel");
332 MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
333 MODULE_VERSION(ADF_DRV_VERSION);
334