1 /*
2 This file is provided under a dual BSD/GPLv2 license. When using or
3 redistributing this file, you may do so under either license.
4
5 GPL LICENSE SUMMARY
6 Copyright(c) 2014 Intel Corporation.
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of version 2 of the GNU General Public License as
9 published by the Free Software Foundation.
10
11 This program is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 General Public License for more details.
15
16 Contact Information:
17 qat-linux@intel.com
18
19 BSD LICENSE
20 Copyright(c) 2014 Intel Corporation.
21 Redistribution and use in source and binary forms, with or without
22 modification, are permitted provided that the following conditions
23 are met:
24
25 * Redistributions of source code must retain the above copyright
26 notice, this list of conditions and the following disclaimer.
27 * Redistributions in binary form must reproduce the above copyright
28 notice, this list of conditions and the following disclaimer in
29 the documentation and/or other materials provided with the
30 distribution.
31 * Neither the name of Intel Corporation nor the names of its
32 contributors may be used to endorse or promote products derived
33 from this software without specific prior written permission.
34
35 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46 */
47 #include <linux/mutex.h>
48 #include <linux/list.h>
49 #include <linux/bitops.h>
50 #include <linux/delay.h>
51 #include "adf_accel_devices.h"
52 #include "adf_cfg.h"
53 #include "adf_common_drv.h"
54
55 static LIST_HEAD(service_table);
56 static DEFINE_MUTEX(service_lock);
57
adf_service_add(struct service_hndl * service)58 static void adf_service_add(struct service_hndl *service)
59 {
60 mutex_lock(&service_lock);
61 list_add(&service->list, &service_table);
62 mutex_unlock(&service_lock);
63 }
64
adf_service_register(struct service_hndl * service)65 int adf_service_register(struct service_hndl *service)
66 {
67 service->init_status = 0;
68 service->start_status = 0;
69 adf_service_add(service);
70 return 0;
71 }
72
adf_service_remove(struct service_hndl * service)73 static void adf_service_remove(struct service_hndl *service)
74 {
75 mutex_lock(&service_lock);
76 list_del(&service->list);
77 mutex_unlock(&service_lock);
78 }
79
adf_service_unregister(struct service_hndl * service)80 int adf_service_unregister(struct service_hndl *service)
81 {
82 if (service->init_status || service->start_status) {
83 pr_err("QAT: Could not remove active service\n");
84 return -EFAULT;
85 }
86 adf_service_remove(service);
87 return 0;
88 }
89
90 /**
91 * adf_dev_init() - Init data structures and services for the given accel device
92 * @accel_dev: Pointer to acceleration device.
93 *
94 * Initialize the ring data structures and the admin comms and arbitration
95 * services.
96 *
97 * Return: 0 on success, error code otherwise.
98 */
adf_dev_init(struct adf_accel_dev * accel_dev)99 int adf_dev_init(struct adf_accel_dev *accel_dev)
100 {
101 struct service_hndl *service;
102 struct list_head *list_itr;
103 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
104
105 if (!hw_data) {
106 dev_err(&GET_DEV(accel_dev),
107 "Failed to init device - hw_data not set\n");
108 return -EFAULT;
109 }
110
111 if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status)) {
112 dev_err(&GET_DEV(accel_dev), "Device not configured\n");
113 return -EFAULT;
114 }
115
116 if (adf_init_etr_data(accel_dev)) {
117 dev_err(&GET_DEV(accel_dev), "Failed initialize etr\n");
118 return -EFAULT;
119 }
120
121 if (hw_data->init_admin_comms && hw_data->init_admin_comms(accel_dev)) {
122 dev_err(&GET_DEV(accel_dev), "Failed initialize admin comms\n");
123 return -EFAULT;
124 }
125
126 if (hw_data->init_arb && hw_data->init_arb(accel_dev)) {
127 dev_err(&GET_DEV(accel_dev), "Failed initialize hw arbiter\n");
128 return -EFAULT;
129 }
130
131 hw_data->enable_ints(accel_dev);
132
133 if (adf_ae_init(accel_dev)) {
134 dev_err(&GET_DEV(accel_dev),
135 "Failed to initialise Acceleration Engine\n");
136 return -EFAULT;
137 }
138 set_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status);
139
140 if (adf_ae_fw_load(accel_dev)) {
141 dev_err(&GET_DEV(accel_dev),
142 "Failed to load acceleration FW\n");
143 return -EFAULT;
144 }
145 set_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status);
146
147 if (hw_data->alloc_irq(accel_dev)) {
148 dev_err(&GET_DEV(accel_dev), "Failed to allocate interrupts\n");
149 return -EFAULT;
150 }
151 set_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status);
152
153 /*
154 * Subservice initialisation is divided into two stages: init and start.
155 * This is to facilitate any ordering dependencies between services
156 * prior to starting any of the accelerators.
157 */
158 list_for_each(list_itr, &service_table) {
159 service = list_entry(list_itr, struct service_hndl, list);
160 if (service->event_hld(accel_dev, ADF_EVENT_INIT)) {
161 dev_err(&GET_DEV(accel_dev),
162 "Failed to initialise service %s\n",
163 service->name);
164 return -EFAULT;
165 }
166 set_bit(accel_dev->accel_id, &service->init_status);
167 }
168
169 hw_data->enable_error_correction(accel_dev);
170 hw_data->enable_vf2pf_comms(accel_dev);
171
172 return 0;
173 }
174 EXPORT_SYMBOL_GPL(adf_dev_init);
175
176 /**
177 * adf_dev_start() - Start acceleration service for the given accel device
178 * @accel_dev: Pointer to acceleration device.
179 *
180 * Function notifies all the registered services that the acceleration device
181 * is ready to be used.
182 * To be used by QAT device specific drivers.
183 *
184 * Return: 0 on success, error code otherwise.
185 */
adf_dev_start(struct adf_accel_dev * accel_dev)186 int adf_dev_start(struct adf_accel_dev *accel_dev)
187 {
188 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
189 struct service_hndl *service;
190 struct list_head *list_itr;
191
192 set_bit(ADF_STATUS_STARTING, &accel_dev->status);
193
194 if (adf_ae_start(accel_dev)) {
195 dev_err(&GET_DEV(accel_dev), "AE Start Failed\n");
196 return -EFAULT;
197 }
198 set_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
199
200 if (hw_data->send_admin_init(accel_dev)) {
201 dev_err(&GET_DEV(accel_dev), "Failed to send init message\n");
202 return -EFAULT;
203 }
204
205 list_for_each(list_itr, &service_table) {
206 service = list_entry(list_itr, struct service_hndl, list);
207 if (service->event_hld(accel_dev, ADF_EVENT_START)) {
208 dev_err(&GET_DEV(accel_dev),
209 "Failed to start service %s\n",
210 service->name);
211 return -EFAULT;
212 }
213 set_bit(accel_dev->accel_id, &service->start_status);
214 }
215
216 clear_bit(ADF_STATUS_STARTING, &accel_dev->status);
217 set_bit(ADF_STATUS_STARTED, &accel_dev->status);
218
219 if (!list_empty(&accel_dev->crypto_list) &&
220 (qat_algs_register() || qat_asym_algs_register())) {
221 dev_err(&GET_DEV(accel_dev),
222 "Failed to register crypto algs\n");
223 set_bit(ADF_STATUS_STARTING, &accel_dev->status);
224 clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
225 return -EFAULT;
226 }
227 return 0;
228 }
229 EXPORT_SYMBOL_GPL(adf_dev_start);
230
231 /**
232 * adf_dev_stop() - Stop acceleration service for the given accel device
233 * @accel_dev: Pointer to acceleration device.
234 *
235 * Function notifies all the registered services that the acceleration device
236 * is shuting down.
237 * To be used by QAT device specific drivers.
238 *
239 * Return: void
240 */
adf_dev_stop(struct adf_accel_dev * accel_dev)241 void adf_dev_stop(struct adf_accel_dev *accel_dev)
242 {
243 struct service_hndl *service;
244 struct list_head *list_itr;
245 bool wait = false;
246 int ret;
247
248 if (!adf_dev_started(accel_dev) &&
249 !test_bit(ADF_STATUS_STARTING, &accel_dev->status))
250 return;
251
252 clear_bit(ADF_STATUS_STARTING, &accel_dev->status);
253 clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
254
255 if (!list_empty(&accel_dev->crypto_list)) {
256 qat_algs_unregister();
257 qat_asym_algs_unregister();
258 }
259
260 list_for_each(list_itr, &service_table) {
261 service = list_entry(list_itr, struct service_hndl, list);
262 if (!test_bit(accel_dev->accel_id, &service->start_status))
263 continue;
264 ret = service->event_hld(accel_dev, ADF_EVENT_STOP);
265 if (!ret) {
266 clear_bit(accel_dev->accel_id, &service->start_status);
267 } else if (ret == -EAGAIN) {
268 wait = true;
269 clear_bit(accel_dev->accel_id, &service->start_status);
270 }
271 }
272
273 if (wait)
274 msleep(100);
275
276 if (test_bit(ADF_STATUS_AE_STARTED, &accel_dev->status)) {
277 if (adf_ae_stop(accel_dev))
278 dev_err(&GET_DEV(accel_dev), "failed to stop AE\n");
279 else
280 clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
281 }
282 }
283 EXPORT_SYMBOL_GPL(adf_dev_stop);
284
285 /**
286 * adf_dev_shutdown() - shutdown acceleration services and data strucutures
287 * @accel_dev: Pointer to acceleration device
288 *
289 * Cleanup the ring data structures and the admin comms and arbitration
290 * services.
291 */
adf_dev_shutdown(struct adf_accel_dev * accel_dev)292 void adf_dev_shutdown(struct adf_accel_dev *accel_dev)
293 {
294 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
295 struct service_hndl *service;
296 struct list_head *list_itr;
297
298 if (!hw_data) {
299 dev_err(&GET_DEV(accel_dev),
300 "QAT: Failed to shutdown device - hw_data not set\n");
301 return;
302 }
303
304 if (test_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status)) {
305 adf_ae_fw_release(accel_dev);
306 clear_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status);
307 }
308
309 if (test_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status)) {
310 if (adf_ae_shutdown(accel_dev))
311 dev_err(&GET_DEV(accel_dev),
312 "Failed to shutdown Accel Engine\n");
313 else
314 clear_bit(ADF_STATUS_AE_INITIALISED,
315 &accel_dev->status);
316 }
317
318 list_for_each(list_itr, &service_table) {
319 service = list_entry(list_itr, struct service_hndl, list);
320 if (!test_bit(accel_dev->accel_id, &service->init_status))
321 continue;
322 if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN))
323 dev_err(&GET_DEV(accel_dev),
324 "Failed to shutdown service %s\n",
325 service->name);
326 else
327 clear_bit(accel_dev->accel_id, &service->init_status);
328 }
329
330 hw_data->disable_iov(accel_dev);
331
332 if (test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status)) {
333 hw_data->free_irq(accel_dev);
334 clear_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status);
335 }
336
337 /* Delete configuration only if not restarting */
338 if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status))
339 adf_cfg_del_all(accel_dev);
340
341 if (hw_data->exit_arb)
342 hw_data->exit_arb(accel_dev);
343
344 if (hw_data->exit_admin_comms)
345 hw_data->exit_admin_comms(accel_dev);
346
347 adf_cleanup_etr_data(accel_dev);
348 adf_dev_restore(accel_dev);
349 }
350 EXPORT_SYMBOL_GPL(adf_dev_shutdown);
351
adf_dev_restarting_notify(struct adf_accel_dev * accel_dev)352 int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev)
353 {
354 struct service_hndl *service;
355 struct list_head *list_itr;
356
357 list_for_each(list_itr, &service_table) {
358 service = list_entry(list_itr, struct service_hndl, list);
359 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING))
360 dev_err(&GET_DEV(accel_dev),
361 "Failed to restart service %s.\n",
362 service->name);
363 }
364 return 0;
365 }
366
adf_dev_restarted_notify(struct adf_accel_dev * accel_dev)367 int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev)
368 {
369 struct service_hndl *service;
370 struct list_head *list_itr;
371
372 list_for_each(list_itr, &service_table) {
373 service = list_entry(list_itr, struct service_hndl, list);
374 if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED))
375 dev_err(&GET_DEV(accel_dev),
376 "Failed to restart service %s.\n",
377 service->name);
378 }
379 return 0;
380 }
381