1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/slab.h>
4 #include <linux/delay.h>
5 #include <linux/pci_ids.h>
6
7 #include "adf_accel_devices.h"
8 #include "adf_common_drv.h"
9 #include "icp_qat_hal.h"
10 #include "icp_qat_uclo.h"
11
12 #define BAD_REGADDR 0xffff
13 #define MAX_RETRY_TIMES 10000
14 #define INIT_CTX_ARB_VALUE 0x0
15 #define INIT_CTX_ENABLE_VALUE 0x0
16 #define INIT_PC_VALUE 0x0
17 #define INIT_WAKEUP_EVENTS_VALUE 0x1
18 #define INIT_SIG_EVENTS_VALUE 0x1
19 #define INIT_CCENABLE_VALUE 0x2000
20 #define RST_CSR_QAT_LSB 20
21 #define RST_CSR_AE_LSB 0
22 #define MC_TIMESTAMP_ENABLE (0x1 << 7)
23
24 #define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \
25 (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \
26 (~(1 << CE_REG_PAR_ERR_BITPOS)))
27 #define INSERT_IMMED_GPRA_CONST(inst, const_val) \
28 (inst = ((inst & 0xFFFF00C03FFull) | \
29 ((((const_val) << 12) & 0x0FF00000ull) | \
30 (((const_val) << 10) & 0x0003FC00ull))))
31 #define INSERT_IMMED_GPRB_CONST(inst, const_val) \
32 (inst = ((inst & 0xFFFF00FFF00ull) | \
33 ((((const_val) << 12) & 0x0FF00000ull) | \
34 (((const_val) << 0) & 0x000000FFull))))
35
36 #define AE(handle, ae) handle->hal_handle->aes[ae]
37
38 static const u64 inst_4b[] = {
39 0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull,
40 0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
41 0x0A021000000ull
42 };
43
44 static const u64 inst[] = {
45 0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull,
46 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
47 0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull,
48 0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
49 0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull,
50 0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull,
51 0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull,
52 0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull,
53 0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull,
54 0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull,
55 0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull,
56 0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull,
57 0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull,
58 0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull,
59 0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull,
60 0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull,
61 0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull,
62 0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull,
63 0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull,
64 0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull,
65 0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull,
66 0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull,
67 };
68
qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)69 void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
70 unsigned char ae, unsigned int ctx_mask)
71 {
72 AE(handle, ae).live_ctx_mask = ctx_mask;
73 }
74
75 #define CSR_RETRY_TIMES 500
qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int csr)76 static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,
77 unsigned char ae, unsigned int csr)
78 {
79 unsigned int iterations = CSR_RETRY_TIMES;
80 int value;
81
82 do {
83 value = GET_AE_CSR(handle, ae, csr);
84 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
85 return value;
86 } while (iterations--);
87
88 pr_err("QAT: Read CSR timeout\n");
89 return 0;
90 }
91
qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int csr,unsigned int value)92 static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,
93 unsigned char ae, unsigned int csr,
94 unsigned int value)
95 {
96 unsigned int iterations = CSR_RETRY_TIMES;
97
98 do {
99 SET_AE_CSR(handle, ae, csr, value);
100 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
101 return 0;
102 } while (iterations--);
103
104 pr_err("QAT: Write CSR Timeout\n");
105 return -EFAULT;
106 }
107
qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,unsigned int * events)108 static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,
109 unsigned char ae, unsigned char ctx,
110 unsigned int *events)
111 {
112 unsigned int cur_ctx;
113
114 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
115 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
116 *events = qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT);
117 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
118 }
119
qat_hal_wait_cycles(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int cycles,int chk_inactive)120 static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,
121 unsigned char ae, unsigned int cycles,
122 int chk_inactive)
123 {
124 unsigned int base_cnt = 0, cur_cnt = 0;
125 unsigned int csr = (1 << ACS_ABO_BITPOS);
126 int times = MAX_RETRY_TIMES;
127 int elapsed_cycles = 0;
128
129 base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
130 base_cnt &= 0xffff;
131 while ((int)cycles > elapsed_cycles && times--) {
132 if (chk_inactive)
133 csr = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
134
135 cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
136 cur_cnt &= 0xffff;
137 elapsed_cycles = cur_cnt - base_cnt;
138
139 if (elapsed_cycles < 0)
140 elapsed_cycles += 0x10000;
141
142 /* ensure at least 8 time cycles elapsed in wait_cycles */
143 if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS)))
144 return 0;
145 }
146 if (times < 0) {
147 pr_err("QAT: wait_num_cycles time out\n");
148 return -EFAULT;
149 }
150 return 0;
151 }
152
153 #define CLR_BIT(wrd, bit) (wrd & ~(1 << bit))
154 #define SET_BIT(wrd, bit) (wrd | 1 << bit)
155
qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char mode)156 int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,
157 unsigned char ae, unsigned char mode)
158 {
159 unsigned int csr, new_csr;
160
161 if ((mode != 4) && (mode != 8)) {
162 pr_err("QAT: bad ctx mode=%d\n", mode);
163 return -EINVAL;
164 }
165
166 /* Sets the accelaration engine context mode to either four or eight */
167 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
168 csr = IGNORE_W1C_MASK & csr;
169 new_csr = (mode == 4) ?
170 SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) :
171 CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS);
172 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
173 return 0;
174 }
175
qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char mode)176 int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,
177 unsigned char ae, unsigned char mode)
178 {
179 unsigned int csr, new_csr;
180
181 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
182 csr &= IGNORE_W1C_MASK;
183
184 new_csr = (mode) ?
185 SET_BIT(csr, CE_NN_MODE_BITPOS) :
186 CLR_BIT(csr, CE_NN_MODE_BITPOS);
187
188 if (new_csr != csr)
189 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
190
191 return 0;
192 }
193
qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle * handle,unsigned char ae,enum icp_qat_uof_regtype lm_type,unsigned char mode)194 int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,
195 unsigned char ae, enum icp_qat_uof_regtype lm_type,
196 unsigned char mode)
197 {
198 unsigned int csr, new_csr;
199
200 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
201 csr &= IGNORE_W1C_MASK;
202 switch (lm_type) {
203 case ICP_LMEM0:
204 new_csr = (mode) ?
205 SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) :
206 CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS);
207 break;
208 case ICP_LMEM1:
209 new_csr = (mode) ?
210 SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) :
211 CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS);
212 break;
213 default:
214 pr_err("QAT: lmType = 0x%x\n", lm_type);
215 return -EINVAL;
216 }
217
218 if (new_csr != csr)
219 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
220 return 0;
221 }
222
qat_hal_get_reg_addr(unsigned int type,unsigned short reg_num)223 static unsigned short qat_hal_get_reg_addr(unsigned int type,
224 unsigned short reg_num)
225 {
226 unsigned short reg_addr;
227
228 switch (type) {
229 case ICP_GPA_ABS:
230 case ICP_GPB_ABS:
231 reg_addr = 0x80 | (reg_num & 0x7f);
232 break;
233 case ICP_GPA_REL:
234 case ICP_GPB_REL:
235 reg_addr = reg_num & 0x1f;
236 break;
237 case ICP_SR_RD_REL:
238 case ICP_SR_WR_REL:
239 case ICP_SR_REL:
240 reg_addr = 0x180 | (reg_num & 0x1f);
241 break;
242 case ICP_SR_ABS:
243 reg_addr = 0x140 | ((reg_num & 0x3) << 1);
244 break;
245 case ICP_DR_RD_REL:
246 case ICP_DR_WR_REL:
247 case ICP_DR_REL:
248 reg_addr = 0x1c0 | (reg_num & 0x1f);
249 break;
250 case ICP_DR_ABS:
251 reg_addr = 0x100 | ((reg_num & 0x3) << 1);
252 break;
253 case ICP_NEIGH_REL:
254 reg_addr = 0x280 | (reg_num & 0x1f);
255 break;
256 case ICP_LMEM0:
257 reg_addr = 0x200;
258 break;
259 case ICP_LMEM1:
260 reg_addr = 0x220;
261 break;
262 case ICP_NO_DEST:
263 reg_addr = 0x300 | (reg_num & 0xff);
264 break;
265 default:
266 reg_addr = BAD_REGADDR;
267 break;
268 }
269 return reg_addr;
270 }
271
qat_hal_reset(struct icp_qat_fw_loader_handle * handle)272 void qat_hal_reset(struct icp_qat_fw_loader_handle *handle)
273 {
274 unsigned int ae_reset_csr;
275
276 ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);
277 ae_reset_csr |= handle->hal_handle->ae_mask << RST_CSR_AE_LSB;
278 ae_reset_csr |= handle->hal_handle->slice_mask << RST_CSR_QAT_LSB;
279 SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);
280 }
281
qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask,unsigned int ae_csr,unsigned int csr_val)282 static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,
283 unsigned char ae, unsigned int ctx_mask,
284 unsigned int ae_csr, unsigned int csr_val)
285 {
286 unsigned int ctx, cur_ctx;
287
288 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
289
290 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
291 if (!(ctx_mask & (1 << ctx)))
292 continue;
293 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
294 qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val);
295 }
296
297 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
298 }
299
qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,unsigned int ae_csr)300 static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,
301 unsigned char ae, unsigned char ctx,
302 unsigned int ae_csr)
303 {
304 unsigned int cur_ctx, csr_val;
305
306 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
307 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
308 csr_val = qat_hal_rd_ae_csr(handle, ae, ae_csr);
309 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
310
311 return csr_val;
312 }
313
qat_hal_put_sig_event(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask,unsigned int events)314 static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,
315 unsigned char ae, unsigned int ctx_mask,
316 unsigned int events)
317 {
318 unsigned int ctx, cur_ctx;
319
320 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
321 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
322 if (!(ctx_mask & (1 << ctx)))
323 continue;
324 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
325 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events);
326 }
327 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
328 }
329
qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask,unsigned int events)330 static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,
331 unsigned char ae, unsigned int ctx_mask,
332 unsigned int events)
333 {
334 unsigned int ctx, cur_ctx;
335
336 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
337 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
338 if (!(ctx_mask & (1 << ctx)))
339 continue;
340 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
341 qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT,
342 events);
343 }
344 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
345 }
346
qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle * handle)347 static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
348 {
349 unsigned int base_cnt, cur_cnt;
350 unsigned char ae;
351 int times = MAX_RETRY_TIMES;
352
353 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
354 base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
355 base_cnt &= 0xffff;
356
357 do {
358 cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
359 cur_cnt &= 0xffff;
360 } while (times-- && (cur_cnt == base_cnt));
361
362 if (times < 0) {
363 pr_err("QAT: AE%d is inactive!!\n", ae);
364 return -EFAULT;
365 }
366 }
367
368 return 0;
369 }
370
qat_hal_check_ae_active(struct icp_qat_fw_loader_handle * handle,unsigned int ae)371 int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle,
372 unsigned int ae)
373 {
374 unsigned int enable = 0, active = 0;
375
376 enable = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
377 active = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
378 if ((enable & (0xff << CE_ENABLE_BITPOS)) ||
379 (active & (1 << ACS_ABO_BITPOS)))
380 return 1;
381 else
382 return 0;
383 }
384
qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle * handle)385 static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)
386 {
387 unsigned int misc_ctl;
388 unsigned char ae;
389
390 /* stop the timestamp timers */
391 misc_ctl = GET_GLB_CSR(handle, MISC_CONTROL);
392 if (misc_ctl & MC_TIMESTAMP_ENABLE)
393 SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl &
394 (~MC_TIMESTAMP_ENABLE));
395
396 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
397 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0);
398 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0);
399 }
400 /* start timestamp timers */
401 SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE);
402 }
403
404 #define ESRAM_AUTO_TINIT BIT(2)
405 #define ESRAM_AUTO_TINIT_DONE BIT(3)
406 #define ESRAM_AUTO_INIT_USED_CYCLES (1640)
407 #define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C
qat_hal_init_esram(struct icp_qat_fw_loader_handle * handle)408 static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
409 {
410 void __iomem *csr_addr =
411 (void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v +
412 ESRAM_AUTO_INIT_CSR_OFFSET);
413 unsigned int csr_val;
414 int times = 30;
415
416 if (handle->pci_dev->device != PCI_DEVICE_ID_INTEL_QAT_DH895XCC)
417 return 0;
418
419 csr_val = ADF_CSR_RD(csr_addr, 0);
420 if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE))
421 return 0;
422
423 csr_val = ADF_CSR_RD(csr_addr, 0);
424 csr_val |= ESRAM_AUTO_TINIT;
425 ADF_CSR_WR(csr_addr, 0, csr_val);
426
427 do {
428 qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0);
429 csr_val = ADF_CSR_RD(csr_addr, 0);
430 } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--);
431 if ((times < 0)) {
432 pr_err("QAT: Fail to init eSram!\n");
433 return -EFAULT;
434 }
435 return 0;
436 }
437
438 #define SHRAM_INIT_CYCLES 2060
qat_hal_clr_reset(struct icp_qat_fw_loader_handle * handle)439 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle)
440 {
441 unsigned int ae_reset_csr;
442 unsigned char ae;
443 unsigned int clk_csr;
444 unsigned int times = 100;
445 unsigned int csr;
446
447 /* write to the reset csr */
448 ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);
449 ae_reset_csr &= ~(handle->hal_handle->ae_mask << RST_CSR_AE_LSB);
450 ae_reset_csr &= ~(handle->hal_handle->slice_mask << RST_CSR_QAT_LSB);
451 do {
452 SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);
453 if (!(times--))
454 goto out_err;
455 csr = GET_GLB_CSR(handle, ICP_RESET);
456 } while ((handle->hal_handle->ae_mask |
457 (handle->hal_handle->slice_mask << RST_CSR_QAT_LSB)) & csr);
458 /* enable clock */
459 clk_csr = GET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE);
460 clk_csr |= handle->hal_handle->ae_mask << 0;
461 clk_csr |= handle->hal_handle->slice_mask << 20;
462 SET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr);
463 if (qat_hal_check_ae_alive(handle))
464 goto out_err;
465
466 /* Set undefined power-up/reset states to reasonable default values */
467 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
468 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
469 INIT_CTX_ENABLE_VALUE);
470 qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX,
471 CTX_STS_INDIRECT,
472 handle->hal_handle->upc_mask &
473 INIT_PC_VALUE);
474 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
475 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
476 qat_hal_put_wakeup_event(handle, ae,
477 ICP_QAT_UCLO_AE_ALL_CTX,
478 INIT_WAKEUP_EVENTS_VALUE);
479 qat_hal_put_sig_event(handle, ae,
480 ICP_QAT_UCLO_AE_ALL_CTX,
481 INIT_SIG_EVENTS_VALUE);
482 }
483 if (qat_hal_init_esram(handle))
484 goto out_err;
485 if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0))
486 goto out_err;
487 qat_hal_reset_timestamp(handle);
488
489 return 0;
490 out_err:
491 pr_err("QAT: failed to get device out of reset\n");
492 return -EFAULT;
493 }
494
qat_hal_disable_ctx(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)495 static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,
496 unsigned char ae, unsigned int ctx_mask)
497 {
498 unsigned int ctx;
499
500 ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
501 ctx &= IGNORE_W1C_MASK &
502 (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS));
503 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
504 }
505
qat_hal_parity_64bit(u64 word)506 static u64 qat_hal_parity_64bit(u64 word)
507 {
508 word ^= word >> 1;
509 word ^= word >> 2;
510 word ^= word >> 4;
511 word ^= word >> 8;
512 word ^= word >> 16;
513 word ^= word >> 32;
514 return word & 1;
515 }
516
qat_hal_set_uword_ecc(u64 uword)517 static u64 qat_hal_set_uword_ecc(u64 uword)
518 {
519 u64 bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL,
520 bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL,
521 bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL,
522 bit6_mask = 0xdaf69a46910ULL;
523
524 /* clear the ecc bits */
525 uword &= ~(0x7fULL << 0x2C);
526 uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C;
527 uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D;
528 uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E;
529 uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F;
530 uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30;
531 uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31;
532 uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32;
533 return uword;
534 }
535
qat_hal_wr_uwords(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int uaddr,unsigned int words_num,u64 * uword)536 void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,
537 unsigned char ae, unsigned int uaddr,
538 unsigned int words_num, u64 *uword)
539 {
540 unsigned int ustore_addr;
541 unsigned int i;
542
543 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
544 uaddr |= UA_ECS;
545 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
546 for (i = 0; i < words_num; i++) {
547 unsigned int uwrd_lo, uwrd_hi;
548 u64 tmp;
549
550 tmp = qat_hal_set_uword_ecc(uword[i]);
551 uwrd_lo = (unsigned int)(tmp & 0xffffffff);
552 uwrd_hi = (unsigned int)(tmp >> 0x20);
553 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
554 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
555 }
556 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
557 }
558
qat_hal_enable_ctx(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)559 static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,
560 unsigned char ae, unsigned int ctx_mask)
561 {
562 unsigned int ctx;
563
564 ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
565 ctx &= IGNORE_W1C_MASK;
566 ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF;
567 ctx |= (ctx_mask << CE_ENABLE_BITPOS);
568 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
569 }
570
qat_hal_clear_xfer(struct icp_qat_fw_loader_handle * handle)571 static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle)
572 {
573 unsigned char ae;
574 unsigned short reg;
575
576 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
577 for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) {
578 qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS,
579 reg, 0);
580 qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS,
581 reg, 0);
582 }
583 }
584 }
585
qat_hal_clear_gpr(struct icp_qat_fw_loader_handle * handle)586 static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
587 {
588 unsigned char ae;
589 unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX;
590 int times = MAX_RETRY_TIMES;
591 unsigned int csr_val = 0;
592 unsigned int savctx = 0;
593 int ret = 0;
594
595 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
596 csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
597 csr_val &= ~(1 << MMC_SHARE_CS_BITPOS);
598 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);
599 csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
600 csr_val &= IGNORE_W1C_MASK;
601 csr_val |= CE_NN_MODE;
602 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);
603 qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst),
604 (u64 *)inst);
605 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
606 handle->hal_handle->upc_mask &
607 INIT_PC_VALUE);
608 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
609 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0);
610 qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);
611 qat_hal_wr_indr_csr(handle, ae, ctx_mask,
612 CTX_SIG_EVENTS_INDIRECT, 0);
613 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
614 qat_hal_enable_ctx(handle, ae, ctx_mask);
615 }
616 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
617 /* wait for AE to finish */
618 do {
619 ret = qat_hal_wait_cycles(handle, ae, 20, 1);
620 } while (ret && times--);
621
622 if (times < 0) {
623 pr_err("QAT: clear GPR of AE %d failed", ae);
624 return -EINVAL;
625 }
626 qat_hal_disable_ctx(handle, ae, ctx_mask);
627 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
628 savctx & ACS_ACNO);
629 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
630 INIT_CTX_ENABLE_VALUE);
631 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
632 handle->hal_handle->upc_mask &
633 INIT_PC_VALUE);
634 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
635 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
636 qat_hal_put_wakeup_event(handle, ae, ctx_mask,
637 INIT_WAKEUP_EVENTS_VALUE);
638 qat_hal_put_sig_event(handle, ae, ctx_mask,
639 INIT_SIG_EVENTS_VALUE);
640 }
641 return 0;
642 }
643
644 #define ICP_QAT_AE_OFFSET 0x20000
645 #define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000)
646 #define LOCAL_TO_XFER_REG_OFFSET 0x800
647 #define ICP_QAT_EP_OFFSET 0x3a000
qat_hal_init(struct adf_accel_dev * accel_dev)648 int qat_hal_init(struct adf_accel_dev *accel_dev)
649 {
650 unsigned char ae;
651 unsigned int max_en_ae_id = 0;
652 struct icp_qat_fw_loader_handle *handle;
653 struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
654 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
655 struct adf_bar *misc_bar =
656 &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)];
657 struct adf_bar *sram_bar;
658
659 handle = kzalloc(sizeof(*handle), GFP_KERNEL);
660 if (!handle)
661 return -ENOMEM;
662
663 handle->hal_cap_g_ctl_csr_addr_v =
664 (void __iomem *)((uintptr_t)misc_bar->virt_addr +
665 ICP_QAT_CAP_OFFSET);
666 handle->hal_cap_ae_xfer_csr_addr_v =
667 (void __iomem *)((uintptr_t)misc_bar->virt_addr +
668 ICP_QAT_AE_OFFSET);
669 handle->hal_ep_csr_addr_v =
670 (void __iomem *)((uintptr_t)misc_bar->virt_addr +
671 ICP_QAT_EP_OFFSET);
672 handle->hal_cap_ae_local_csr_addr_v =
673 (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v +
674 LOCAL_TO_XFER_REG_OFFSET);
675 handle->pci_dev = pci_info->pci_dev;
676 if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_DH895XCC) {
677 sram_bar =
678 &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)];
679 handle->hal_sram_addr_v = sram_bar->virt_addr;
680 }
681 handle->fw_auth = (handle->pci_dev->device ==
682 PCI_DEVICE_ID_INTEL_QAT_DH895XCC) ? false : true;
683 handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);
684 if (!handle->hal_handle)
685 goto out_hal_handle;
686 handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;
687 handle->hal_handle->ae_mask = hw_data->ae_mask;
688 handle->hal_handle->slice_mask = hw_data->accel_mask;
689 /* create AE objects */
690 handle->hal_handle->upc_mask = 0x1ffff;
691 handle->hal_handle->max_ustore = 0x4000;
692 for (ae = 0; ae < ICP_QAT_UCLO_MAX_AE; ae++) {
693 if (!(hw_data->ae_mask & (1 << ae)))
694 continue;
695 handle->hal_handle->aes[ae].free_addr = 0;
696 handle->hal_handle->aes[ae].free_size =
697 handle->hal_handle->max_ustore;
698 handle->hal_handle->aes[ae].ustore_size =
699 handle->hal_handle->max_ustore;
700 handle->hal_handle->aes[ae].live_ctx_mask =
701 ICP_QAT_UCLO_AE_ALL_CTX;
702 max_en_ae_id = ae;
703 }
704 handle->hal_handle->ae_max_num = max_en_ae_id + 1;
705 /* take all AEs out of reset */
706 if (qat_hal_clr_reset(handle)) {
707 dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n");
708 goto out_err;
709 }
710 qat_hal_clear_xfer(handle);
711 if (!handle->fw_auth) {
712 if (qat_hal_clear_gpr(handle))
713 goto out_err;
714 }
715
716 /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
717 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
718 unsigned int csr_val = 0;
719
720 csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE);
721 csr_val |= 0x1;
722 qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val);
723 }
724 accel_dev->fw_loader->fw_loader = handle;
725 return 0;
726
727 out_err:
728 kfree(handle->hal_handle);
729 out_hal_handle:
730 kfree(handle);
731 return -EFAULT;
732 }
733
qat_hal_deinit(struct icp_qat_fw_loader_handle * handle)734 void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle)
735 {
736 if (!handle)
737 return;
738 kfree(handle->hal_handle);
739 kfree(handle);
740 }
741
qat_hal_start(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)742 void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
743 unsigned int ctx_mask)
744 {
745 int retry = 0;
746 unsigned int fcu_sts = 0;
747
748 if (handle->fw_auth) {
749 SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_START);
750 do {
751 msleep(FW_AUTH_WAIT_PERIOD);
752 fcu_sts = GET_CAP_CSR(handle, FCU_STATUS);
753 if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1))
754 return;
755 } while (retry++ < FW_AUTH_MAX_RETRY);
756 pr_err("QAT: start error (AE 0x%x FCU_STS = 0x%x)\n", ae,
757 fcu_sts);
758 } else {
759 qat_hal_put_wakeup_event(handle, ae, (~ctx_mask) &
760 ICP_QAT_UCLO_AE_ALL_CTX, 0x10000);
761 qat_hal_enable_ctx(handle, ae, ctx_mask);
762 }
763 }
764
qat_hal_stop(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask)765 void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
766 unsigned int ctx_mask)
767 {
768 if (!handle->fw_auth)
769 qat_hal_disable_ctx(handle, ae, ctx_mask);
770 }
771
qat_hal_set_pc(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int ctx_mask,unsigned int upc)772 void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,
773 unsigned char ae, unsigned int ctx_mask, unsigned int upc)
774 {
775 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
776 handle->hal_handle->upc_mask & upc);
777 }
778
qat_hal_get_uwords(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int uaddr,unsigned int words_num,u64 * uword)779 static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,
780 unsigned char ae, unsigned int uaddr,
781 unsigned int words_num, u64 *uword)
782 {
783 unsigned int i, uwrd_lo, uwrd_hi;
784 unsigned int ustore_addr, misc_control;
785
786 misc_control = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
787 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL,
788 misc_control & 0xfffffffb);
789 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
790 uaddr |= UA_ECS;
791 for (i = 0; i < words_num; i++) {
792 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
793 uaddr++;
794 uwrd_lo = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER);
795 uwrd_hi = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER);
796 uword[i] = uwrd_hi;
797 uword[i] = (uword[i] << 0x20) | uwrd_lo;
798 }
799 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control);
800 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
801 }
802
qat_hal_wr_umem(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned int uaddr,unsigned int words_num,unsigned int * data)803 void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,
804 unsigned char ae, unsigned int uaddr,
805 unsigned int words_num, unsigned int *data)
806 {
807 unsigned int i, ustore_addr;
808
809 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
810 uaddr |= UA_ECS;
811 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
812 for (i = 0; i < words_num; i++) {
813 unsigned int uwrd_lo, uwrd_hi, tmp;
814
815 uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) |
816 ((data[i] & 0xff00) << 2) |
817 (0x3 << 8) | (data[i] & 0xff);
818 uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28);
819 uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8;
820 tmp = ((data[i] >> 0x10) & 0xffff);
821 uwrd_hi |= (hweight32(tmp) & 0x1) << 9;
822 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
823 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
824 }
825 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
826 }
827
828 #define MAX_EXEC_INST 100
qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,u64 * micro_inst,unsigned int inst_num,int code_off,unsigned int max_cycle,unsigned int * endpc)829 static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
830 unsigned char ae, unsigned char ctx,
831 u64 *micro_inst, unsigned int inst_num,
832 int code_off, unsigned int max_cycle,
833 unsigned int *endpc)
834 {
835 u64 savuwords[MAX_EXEC_INST];
836 unsigned int ind_lm_addr0, ind_lm_addr1;
837 unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1;
838 unsigned int ind_cnt_sig;
839 unsigned int ind_sig, act_sig;
840 unsigned int csr_val = 0, newcsr_val;
841 unsigned int savctx;
842 unsigned int savcc, wakeup_events, savpc;
843 unsigned int ctxarb_ctl, ctx_enables;
844
845 if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) {
846 pr_err("QAT: invalid instruction num %d\n", inst_num);
847 return -EINVAL;
848 }
849 /* save current context */
850 ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT);
851 ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT);
852 ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx,
853 INDIRECT_LM_ADDR_0_BYTE_INDEX);
854 ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx,
855 INDIRECT_LM_ADDR_1_BYTE_INDEX);
856 if (inst_num <= MAX_EXEC_INST)
857 qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords);
858 qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events);
859 savpc = qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT);
860 savpc = (savpc & handle->hal_handle->upc_mask) >> 0;
861 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
862 ctx_enables &= IGNORE_W1C_MASK;
863 savcc = qat_hal_rd_ae_csr(handle, ae, CC_ENABLE);
864 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
865 ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
866 ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
867 FUTURE_COUNT_SIGNAL_INDIRECT);
868 ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
869 CTX_SIG_EVENTS_INDIRECT);
870 act_sig = qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE);
871 /* execute micro codes */
872 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
873 qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst);
874 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0);
875 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO);
876 if (code_off)
877 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff);
878 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY);
879 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0);
880 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
881 qat_hal_enable_ctx(handle, ae, (1 << ctx));
882 /* wait for micro codes to finish */
883 if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0)
884 return -EFAULT;
885 if (endpc) {
886 unsigned int ctx_status;
887
888 ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx,
889 CTX_STS_INDIRECT);
890 *endpc = ctx_status & handle->hal_handle->upc_mask;
891 }
892 /* retore to saved context */
893 qat_hal_disable_ctx(handle, ae, (1 << ctx));
894 if (inst_num <= MAX_EXEC_INST)
895 qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords);
896 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events);
897 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT,
898 handle->hal_handle->upc_mask & savpc);
899 csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
900 newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
901 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
902 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc);
903 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO);
904 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl);
905 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
906 LM_ADDR_0_INDIRECT, ind_lm_addr0);
907 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
908 LM_ADDR_1_INDIRECT, ind_lm_addr1);
909 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
910 INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0);
911 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
912 INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1);
913 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
914 FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig);
915 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
916 CTX_SIG_EVENTS_INDIRECT, ind_sig);
917 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig);
918 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
919
920 return 0;
921 }
922
qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int * data)923 static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,
924 unsigned char ae, unsigned char ctx,
925 enum icp_qat_uof_regtype reg_type,
926 unsigned short reg_num, unsigned int *data)
927 {
928 unsigned int savctx, uaddr, uwrd_lo, uwrd_hi;
929 unsigned int ctxarb_cntl, ustore_addr, ctx_enables;
930 unsigned short reg_addr;
931 int status = 0;
932 u64 insts, savuword;
933
934 reg_addr = qat_hal_get_reg_addr(reg_type, reg_num);
935 if (reg_addr == BAD_REGADDR) {
936 pr_err("QAT: bad regaddr=0x%x\n", reg_addr);
937 return -EINVAL;
938 }
939 switch (reg_type) {
940 case ICP_GPA_REL:
941 insts = 0xA070000000ull | (reg_addr & 0x3ff);
942 break;
943 default:
944 insts = (u64)0xA030000000ull | ((reg_addr & 0x3ff) << 10);
945 break;
946 }
947 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
948 ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
949 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
950 ctx_enables &= IGNORE_W1C_MASK;
951 if (ctx != (savctx & ACS_ACNO))
952 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
953 ctx & ACS_ACNO);
954 qat_hal_get_uwords(handle, ae, 0, 1, &savuword);
955 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
956 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
957 uaddr = UA_ECS;
958 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
959 insts = qat_hal_set_uword_ecc(insts);
960 uwrd_lo = (unsigned int)(insts & 0xffffffff);
961 uwrd_hi = (unsigned int)(insts >> 0x20);
962 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
963 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
964 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
965 /* delay for at least 8 cycles */
966 qat_hal_wait_cycles(handle, ae, 0x8, 0);
967 /*
968 * read ALU output
969 * the instruction should have been executed
970 * prior to clearing the ECS in putUwords
971 */
972 *data = qat_hal_rd_ae_csr(handle, ae, ALU_OUT);
973 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
974 qat_hal_wr_uwords(handle, ae, 0, 1, &savuword);
975 if (ctx != (savctx & ACS_ACNO))
976 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
977 savctx & ACS_ACNO);
978 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl);
979 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
980
981 return status;
982 }
983
qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int data)984 static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle,
985 unsigned char ae, unsigned char ctx,
986 enum icp_qat_uof_regtype reg_type,
987 unsigned short reg_num, unsigned int data)
988 {
989 unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo;
990 u64 insts[] = {
991 0x0F440000000ull,
992 0x0F040000000ull,
993 0x0F0000C0300ull,
994 0x0E000010000ull
995 };
996 const int num_inst = ARRAY_SIZE(insts), code_off = 1;
997 const int imm_w1 = 0, imm_w0 = 1;
998
999 dest_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1000 if (dest_addr == BAD_REGADDR) {
1001 pr_err("QAT: bad destAddr=0x%x\n", dest_addr);
1002 return -EINVAL;
1003 }
1004
1005 data16lo = 0xffff & data;
1006 data16hi = 0xffff & (data >> 0x10);
1007 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1008 (0xff & data16hi));
1009 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1010 (0xff & data16lo));
1011 switch (reg_type) {
1012 case ICP_GPA_REL:
1013 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1014 ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1015 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1016 ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1017 break;
1018 default:
1019 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1020 ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1021
1022 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1023 ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1024 break;
1025 }
1026
1027 return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst,
1028 code_off, num_inst * 0x5, NULL);
1029 }
1030
qat_hal_get_ins_num(void)1031 int qat_hal_get_ins_num(void)
1032 {
1033 return ARRAY_SIZE(inst_4b);
1034 }
1035
qat_hal_concat_micro_code(u64 * micro_inst,unsigned int inst_num,unsigned int size,unsigned int addr,unsigned int * value)1036 static int qat_hal_concat_micro_code(u64 *micro_inst,
1037 unsigned int inst_num, unsigned int size,
1038 unsigned int addr, unsigned int *value)
1039 {
1040 int i;
1041 unsigned int cur_value;
1042 const u64 *inst_arr;
1043 int fixup_offset;
1044 int usize = 0;
1045 int orig_num;
1046
1047 orig_num = inst_num;
1048 cur_value = value[0];
1049 inst_arr = inst_4b;
1050 usize = ARRAY_SIZE(inst_4b);
1051 fixup_offset = inst_num;
1052 for (i = 0; i < usize; i++)
1053 micro_inst[inst_num++] = inst_arr[i];
1054 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr));
1055 fixup_offset++;
1056 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0);
1057 fixup_offset++;
1058 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0));
1059 fixup_offset++;
1060 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10));
1061
1062 return inst_num - orig_num;
1063 }
1064
qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,int * pfirst_exec,u64 * micro_inst,unsigned int inst_num)1065 static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle,
1066 unsigned char ae, unsigned char ctx,
1067 int *pfirst_exec, u64 *micro_inst,
1068 unsigned int inst_num)
1069 {
1070 int stat = 0;
1071 unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0;
1072 unsigned int gprb0 = 0, gprb1 = 0;
1073
1074 if (*pfirst_exec) {
1075 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0);
1076 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1);
1077 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2);
1078 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0);
1079 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1);
1080 *pfirst_exec = 0;
1081 }
1082 stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1,
1083 inst_num * 0x5, NULL);
1084 if (stat != 0)
1085 return -EFAULT;
1086 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0);
1087 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1);
1088 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2);
1089 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0);
1090 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1);
1091
1092 return 0;
1093 }
1094
qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle * handle,unsigned char ae,struct icp_qat_uof_batch_init * lm_init_header)1095 int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,
1096 unsigned char ae,
1097 struct icp_qat_uof_batch_init *lm_init_header)
1098 {
1099 struct icp_qat_uof_batch_init *plm_init;
1100 u64 *micro_inst_arry;
1101 int micro_inst_num;
1102 int alloc_inst_size;
1103 int first_exec = 1;
1104 int stat = 0;
1105
1106 plm_init = lm_init_header->next;
1107 alloc_inst_size = lm_init_header->size;
1108 if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore)
1109 alloc_inst_size = handle->hal_handle->max_ustore;
1110 micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(u64),
1111 GFP_KERNEL);
1112 if (!micro_inst_arry)
1113 return -ENOMEM;
1114 micro_inst_num = 0;
1115 while (plm_init) {
1116 unsigned int addr, *value, size;
1117
1118 ae = plm_init->ae;
1119 addr = plm_init->addr;
1120 value = plm_init->value;
1121 size = plm_init->size;
1122 micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry,
1123 micro_inst_num,
1124 size, addr, value);
1125 plm_init = plm_init->next;
1126 }
1127 /* exec micro codes */
1128 if (micro_inst_arry && (micro_inst_num > 0)) {
1129 micro_inst_arry[micro_inst_num++] = 0x0E000010000ull;
1130 stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec,
1131 micro_inst_arry,
1132 micro_inst_num);
1133 }
1134 kfree(micro_inst_arry);
1135 return stat;
1136 }
1137
qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int val)1138 static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1139 unsigned char ae, unsigned char ctx,
1140 enum icp_qat_uof_regtype reg_type,
1141 unsigned short reg_num, unsigned int val)
1142 {
1143 int status = 0;
1144 unsigned int reg_addr;
1145 unsigned int ctx_enables;
1146 unsigned short mask;
1147 unsigned short dr_offset = 0x10;
1148
1149 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1150 if (CE_INUSE_CONTEXTS & ctx_enables) {
1151 if (ctx & 0x1) {
1152 pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx);
1153 return -EINVAL;
1154 }
1155 mask = 0x1f;
1156 dr_offset = 0x20;
1157 } else {
1158 mask = 0x0f;
1159 }
1160 if (reg_num & ~mask)
1161 return -EINVAL;
1162 reg_addr = reg_num + (ctx << 0x5);
1163 switch (reg_type) {
1164 case ICP_SR_RD_REL:
1165 case ICP_SR_REL:
1166 SET_AE_XFER(handle, ae, reg_addr, val);
1167 break;
1168 case ICP_DR_RD_REL:
1169 case ICP_DR_REL:
1170 SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val);
1171 break;
1172 default:
1173 status = -EINVAL;
1174 break;
1175 }
1176 return status;
1177 }
1178
qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int data)1179 static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1180 unsigned char ae, unsigned char ctx,
1181 enum icp_qat_uof_regtype reg_type,
1182 unsigned short reg_num, unsigned int data)
1183 {
1184 unsigned int gprval, ctx_enables;
1185 unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi,
1186 data16low;
1187 unsigned short reg_mask;
1188 int status = 0;
1189 u64 micro_inst[] = {
1190 0x0F440000000ull,
1191 0x0F040000000ull,
1192 0x0A000000000ull,
1193 0x0F0000C0300ull,
1194 0x0E000010000ull
1195 };
1196 const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1;
1197 const unsigned short gprnum = 0, dly = num_inst * 0x5;
1198
1199 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1200 if (CE_INUSE_CONTEXTS & ctx_enables) {
1201 if (ctx & 0x1) {
1202 pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx);
1203 return -EINVAL;
1204 }
1205 reg_mask = (unsigned short)~0x1f;
1206 } else {
1207 reg_mask = (unsigned short)~0xf;
1208 }
1209 if (reg_num & reg_mask)
1210 return -EINVAL;
1211 xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1212 if (xfr_addr == BAD_REGADDR) {
1213 pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr);
1214 return -EINVAL;
1215 }
1216 status = qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);
1217 if (status) {
1218 pr_err("QAT: failed to read register");
1219 return status;
1220 }
1221 gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum);
1222 data16low = 0xffff & data;
1223 data16hi = 0xffff & (data >> 0x10);
1224 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1225 (unsigned short)(0xff & data16hi));
1226 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1227 (unsigned short)(0xff & data16low));
1228 micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) |
1229 ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1230 micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) |
1231 ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1232 micro_inst[0x2] = micro_inst[0x2] |
1233 ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10);
1234 status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst,
1235 code_off, dly, NULL);
1236 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval);
1237 return status;
1238 }
1239
qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx,unsigned short nn,unsigned int val)1240 static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,
1241 unsigned char ae, unsigned char ctx,
1242 unsigned short nn, unsigned int val)
1243 {
1244 unsigned int ctx_enables;
1245 int stat = 0;
1246
1247 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1248 ctx_enables &= IGNORE_W1C_MASK;
1249 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE);
1250
1251 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val);
1252 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1253 return stat;
1254 }
1255
qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned short absreg_num,unsigned short * relreg,unsigned char * ctx)1256 static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle
1257 *handle, unsigned char ae,
1258 unsigned short absreg_num,
1259 unsigned short *relreg,
1260 unsigned char *ctx)
1261 {
1262 unsigned int ctx_enables;
1263
1264 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1265 if (ctx_enables & CE_INUSE_CONTEXTS) {
1266 /* 4-ctx mode */
1267 *relreg = absreg_num & 0x1F;
1268 *ctx = (absreg_num >> 0x4) & 0x6;
1269 } else {
1270 /* 8-ctx mode */
1271 *relreg = absreg_num & 0x0F;
1272 *ctx = (absreg_num >> 0x4) & 0x7;
1273 }
1274 return 0;
1275 }
1276
qat_hal_init_gpr(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx_mask,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int regdata)1277 int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,
1278 unsigned char ae, unsigned char ctx_mask,
1279 enum icp_qat_uof_regtype reg_type,
1280 unsigned short reg_num, unsigned int regdata)
1281 {
1282 int stat = 0;
1283 unsigned short reg;
1284 unsigned char ctx = 0;
1285 enum icp_qat_uof_regtype type;
1286
1287 if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG)
1288 return -EINVAL;
1289
1290 do {
1291 if (ctx_mask == 0) {
1292 qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®,
1293 &ctx);
1294 type = reg_type - 1;
1295 } else {
1296 reg = reg_num;
1297 type = reg_type;
1298 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1299 continue;
1300 }
1301 stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata);
1302 if (stat) {
1303 pr_err("QAT: write gpr fail\n");
1304 return -EINVAL;
1305 }
1306 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1307
1308 return 0;
1309 }
1310
qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx_mask,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int regdata)1311 int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1312 unsigned char ae, unsigned char ctx_mask,
1313 enum icp_qat_uof_regtype reg_type,
1314 unsigned short reg_num, unsigned int regdata)
1315 {
1316 int stat = 0;
1317 unsigned short reg;
1318 unsigned char ctx = 0;
1319 enum icp_qat_uof_regtype type;
1320
1321 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1322 return -EINVAL;
1323
1324 do {
1325 if (ctx_mask == 0) {
1326 qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®,
1327 &ctx);
1328 type = reg_type - 3;
1329 } else {
1330 reg = reg_num;
1331 type = reg_type;
1332 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1333 continue;
1334 }
1335 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg,
1336 regdata);
1337 if (stat) {
1338 pr_err("QAT: write wr xfer fail\n");
1339 return -EINVAL;
1340 }
1341 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1342
1343 return 0;
1344 }
1345
qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx_mask,enum icp_qat_uof_regtype reg_type,unsigned short reg_num,unsigned int regdata)1346 int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1347 unsigned char ae, unsigned char ctx_mask,
1348 enum icp_qat_uof_regtype reg_type,
1349 unsigned short reg_num, unsigned int regdata)
1350 {
1351 int stat = 0;
1352 unsigned short reg;
1353 unsigned char ctx = 0;
1354 enum icp_qat_uof_regtype type;
1355
1356 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1357 return -EINVAL;
1358
1359 do {
1360 if (ctx_mask == 0) {
1361 qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®,
1362 &ctx);
1363 type = reg_type - 3;
1364 } else {
1365 reg = reg_num;
1366 type = reg_type;
1367 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1368 continue;
1369 }
1370 stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg,
1371 regdata);
1372 if (stat) {
1373 pr_err("QAT: write rd xfer fail\n");
1374 return -EINVAL;
1375 }
1376 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1377
1378 return 0;
1379 }
1380
qat_hal_init_nn(struct icp_qat_fw_loader_handle * handle,unsigned char ae,unsigned char ctx_mask,unsigned short reg_num,unsigned int regdata)1381 int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,
1382 unsigned char ae, unsigned char ctx_mask,
1383 unsigned short reg_num, unsigned int regdata)
1384 {
1385 int stat = 0;
1386 unsigned char ctx;
1387
1388 if (ctx_mask == 0)
1389 return -EINVAL;
1390
1391 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
1392 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1393 continue;
1394 stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata);
1395 if (stat) {
1396 pr_err("QAT: write neigh error\n");
1397 return -EINVAL;
1398 }
1399 }
1400
1401 return 0;
1402 }
1403