1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * Shared descriptors for aead, skcipher algorithms
4 *
5 * Copyright 2016-2019 NXP
6 */
7
8 #include "compat.h"
9 #include "desc_constr.h"
10 #include "caamalg_desc.h"
11
12 /*
13 * For aead functions, read payload and write payload,
14 * both of which are specified in req->src and req->dst
15 */
aead_append_src_dst(u32 * desc,u32 msg_type)16 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
17 {
18 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
19 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
20 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
21 }
22
23 /* Set DK bit in class 1 operation if shared */
append_dec_op1(u32 * desc,u32 type)24 static inline void append_dec_op1(u32 *desc, u32 type)
25 {
26 u32 *jump_cmd, *uncond_jump_cmd;
27
28 /* DK bit is valid only for AES */
29 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
30 append_operation(desc, type | OP_ALG_AS_INITFINAL |
31 OP_ALG_DECRYPT);
32 return;
33 }
34
35 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
36 append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT);
37 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
38 set_jump_tgt_here(desc, jump_cmd);
39 append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT |
40 OP_ALG_AAI_DK);
41 set_jump_tgt_here(desc, uncond_jump_cmd);
42 }
43
44 /**
45 * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
46 * (non-protocol) with no (null) encryption.
47 * @desc: pointer to buffer used for descriptor construction
48 * @adata: pointer to authentication transform definitions.
49 * A split key is required for SEC Era < 6; the size of the split key
50 * is specified in this case. Valid algorithm values - one of
51 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
52 * with OP_ALG_AAI_HMAC_PRECOMP.
53 * @icvsize: integrity check value (ICV) size (truncated or full)
54 * @era: SEC Era
55 */
cnstr_shdsc_aead_null_encap(u32 * const desc,struct alginfo * adata,unsigned int icvsize,int era)56 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
57 unsigned int icvsize, int era)
58 {
59 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
60
61 init_sh_desc(desc, HDR_SHARE_SERIAL);
62
63 /* Skip if already shared */
64 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
65 JUMP_COND_SHRD);
66 if (era < 6) {
67 if (adata->key_inline)
68 append_key_as_imm(desc, adata->key_virt,
69 adata->keylen_pad, adata->keylen,
70 CLASS_2 | KEY_DEST_MDHA_SPLIT |
71 KEY_ENC);
72 else
73 append_key(desc, adata->key_dma, adata->keylen,
74 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
75 } else {
76 append_proto_dkp(desc, adata);
77 }
78 set_jump_tgt_here(desc, key_jump_cmd);
79
80 /* assoclen + cryptlen = seqinlen */
81 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
82
83 /* Prepare to read and write cryptlen + assoclen bytes */
84 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
85 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
86
87 /*
88 * MOVE_LEN opcode is not available in all SEC HW revisions,
89 * thus need to do some magic, i.e. self-patch the descriptor
90 * buffer.
91 */
92 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
93 MOVE_DEST_MATH3 |
94 (0x6 << MOVE_LEN_SHIFT));
95 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
96 MOVE_DEST_DESCBUF |
97 MOVE_WAITCOMP |
98 (0x8 << MOVE_LEN_SHIFT));
99
100 /* Class 2 operation */
101 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
102 OP_ALG_ENCRYPT);
103
104 /* Read and write cryptlen bytes */
105 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
106
107 set_move_tgt_here(desc, read_move_cmd);
108 set_move_tgt_here(desc, write_move_cmd);
109 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
110 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
111 MOVE_AUX_LS);
112
113 /* Write ICV */
114 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
115 LDST_SRCDST_BYTE_CONTEXT);
116
117 print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ",
118 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
119 1);
120 }
121 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
122
123 /**
124 * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
125 * (non-protocol) with no (null) decryption.
126 * @desc: pointer to buffer used for descriptor construction
127 * @adata: pointer to authentication transform definitions.
128 * A split key is required for SEC Era < 6; the size of the split key
129 * is specified in this case. Valid algorithm values - one of
130 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
131 * with OP_ALG_AAI_HMAC_PRECOMP.
132 * @icvsize: integrity check value (ICV) size (truncated or full)
133 * @era: SEC Era
134 */
cnstr_shdsc_aead_null_decap(u32 * const desc,struct alginfo * adata,unsigned int icvsize,int era)135 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
136 unsigned int icvsize, int era)
137 {
138 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
139
140 init_sh_desc(desc, HDR_SHARE_SERIAL);
141
142 /* Skip if already shared */
143 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
144 JUMP_COND_SHRD);
145 if (era < 6) {
146 if (adata->key_inline)
147 append_key_as_imm(desc, adata->key_virt,
148 adata->keylen_pad, adata->keylen,
149 CLASS_2 | KEY_DEST_MDHA_SPLIT |
150 KEY_ENC);
151 else
152 append_key(desc, adata->key_dma, adata->keylen,
153 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
154 } else {
155 append_proto_dkp(desc, adata);
156 }
157 set_jump_tgt_here(desc, key_jump_cmd);
158
159 /* Class 2 operation */
160 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
161 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
162
163 /* assoclen + cryptlen = seqoutlen */
164 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
165
166 /* Prepare to read and write cryptlen + assoclen bytes */
167 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
168 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
169
170 /*
171 * MOVE_LEN opcode is not available in all SEC HW revisions,
172 * thus need to do some magic, i.e. self-patch the descriptor
173 * buffer.
174 */
175 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
176 MOVE_DEST_MATH2 |
177 (0x6 << MOVE_LEN_SHIFT));
178 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
179 MOVE_DEST_DESCBUF |
180 MOVE_WAITCOMP |
181 (0x8 << MOVE_LEN_SHIFT));
182
183 /* Read and write cryptlen bytes */
184 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
185
186 /*
187 * Insert a NOP here, since we need at least 4 instructions between
188 * code patching the descriptor buffer and the location being patched.
189 */
190 jump_cmd = append_jump(desc, JUMP_TEST_ALL);
191 set_jump_tgt_here(desc, jump_cmd);
192
193 set_move_tgt_here(desc, read_move_cmd);
194 set_move_tgt_here(desc, write_move_cmd);
195 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
196 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
197 MOVE_AUX_LS);
198 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
199
200 /* Load ICV */
201 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
202 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
203
204 print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ",
205 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
206 1);
207 }
208 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
209
init_sh_desc_key_aead(u32 * const desc,struct alginfo * const cdata,struct alginfo * const adata,const bool is_rfc3686,u32 * nonce,int era)210 static void init_sh_desc_key_aead(u32 * const desc,
211 struct alginfo * const cdata,
212 struct alginfo * const adata,
213 const bool is_rfc3686, u32 *nonce, int era)
214 {
215 u32 *key_jump_cmd;
216 unsigned int enckeylen = cdata->keylen;
217
218 /* Note: Context registers are saved. */
219 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
220
221 /* Skip if already shared */
222 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
223 JUMP_COND_SHRD);
224
225 /*
226 * RFC3686 specific:
227 * | key = {AUTH_KEY, ENC_KEY, NONCE}
228 * | enckeylen = encryption key size + nonce size
229 */
230 if (is_rfc3686)
231 enckeylen -= CTR_RFC3686_NONCE_SIZE;
232
233 if (era < 6) {
234 if (adata->key_inline)
235 append_key_as_imm(desc, adata->key_virt,
236 adata->keylen_pad, adata->keylen,
237 CLASS_2 | KEY_DEST_MDHA_SPLIT |
238 KEY_ENC);
239 else
240 append_key(desc, adata->key_dma, adata->keylen,
241 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
242 } else {
243 append_proto_dkp(desc, adata);
244 }
245
246 if (cdata->key_inline)
247 append_key_as_imm(desc, cdata->key_virt, enckeylen,
248 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
249 else
250 append_key(desc, cdata->key_dma, enckeylen, CLASS_1 |
251 KEY_DEST_CLASS_REG);
252
253 /* Load Counter into CONTEXT1 reg */
254 if (is_rfc3686) {
255 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
256 LDST_CLASS_IND_CCB |
257 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
258 append_move(desc,
259 MOVE_SRC_OUTFIFO |
260 MOVE_DEST_CLASS1CTX |
261 (16 << MOVE_OFFSET_SHIFT) |
262 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
263 }
264
265 set_jump_tgt_here(desc, key_jump_cmd);
266 }
267
268 /**
269 * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor
270 * (non-protocol).
271 * @desc: pointer to buffer used for descriptor construction
272 * @cdata: pointer to block cipher transform definitions
273 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
274 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
275 * @adata: pointer to authentication transform definitions.
276 * A split key is required for SEC Era < 6; the size of the split key
277 * is specified in this case. Valid algorithm values - one of
278 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
279 * with OP_ALG_AAI_HMAC_PRECOMP.
280 * @ivsize: initialization vector size
281 * @icvsize: integrity check value (ICV) size (truncated or full)
282 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
283 * @nonce: pointer to rfc3686 nonce
284 * @ctx1_iv_off: IV offset in CONTEXT1 register
285 * @is_qi: true when called from caam/qi
286 * @era: SEC Era
287 */
cnstr_shdsc_aead_encap(u32 * const desc,struct alginfo * cdata,struct alginfo * adata,unsigned int ivsize,unsigned int icvsize,const bool is_rfc3686,u32 * nonce,const u32 ctx1_iv_off,const bool is_qi,int era)288 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
289 struct alginfo *adata, unsigned int ivsize,
290 unsigned int icvsize, const bool is_rfc3686,
291 u32 *nonce, const u32 ctx1_iv_off, const bool is_qi,
292 int era)
293 {
294 /* Note: Context registers are saved. */
295 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
296
297 /* Class 2 operation */
298 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
299 OP_ALG_ENCRYPT);
300
301 if (is_qi) {
302 u32 *wait_load_cmd;
303
304 /* REG3 = assoclen */
305 append_seq_load(desc, 4, LDST_CLASS_DECO |
306 LDST_SRCDST_WORD_DECO_MATH3 |
307 (4 << LDST_OFFSET_SHIFT));
308
309 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
310 JUMP_COND_CALM | JUMP_COND_NCP |
311 JUMP_COND_NOP | JUMP_COND_NIP |
312 JUMP_COND_NIFP);
313 set_jump_tgt_here(desc, wait_load_cmd);
314
315 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
316 LDST_SRCDST_BYTE_CONTEXT |
317 (ctx1_iv_off << LDST_OFFSET_SHIFT));
318 }
319
320 /* Read and write assoclen bytes */
321 if (is_qi || era < 3) {
322 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
323 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
324 } else {
325 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
326 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
327 }
328
329 /* Skip assoc data */
330 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
331
332 /* read assoc before reading payload */
333 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
334 FIFOLDST_VLF);
335
336 /* Load Counter into CONTEXT1 reg */
337 if (is_rfc3686)
338 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
339 LDST_SRCDST_BYTE_CONTEXT |
340 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
341 LDST_OFFSET_SHIFT));
342
343 /* Class 1 operation */
344 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
345 OP_ALG_ENCRYPT);
346
347 /* Read and write cryptlen bytes */
348 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
349 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
350 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
351
352 /* Write ICV */
353 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
354 LDST_SRCDST_BYTE_CONTEXT);
355
356 print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ",
357 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
358 1);
359 }
360 EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
361
362 /**
363 * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor
364 * (non-protocol).
365 * @desc: pointer to buffer used for descriptor construction
366 * @cdata: pointer to block cipher transform definitions
367 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
368 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
369 * @adata: pointer to authentication transform definitions.
370 * A split key is required for SEC Era < 6; the size of the split key
371 * is specified in this case. Valid algorithm values - one of
372 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
373 * with OP_ALG_AAI_HMAC_PRECOMP.
374 * @ivsize: initialization vector size
375 * @icvsize: integrity check value (ICV) size (truncated or full)
376 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
377 * @nonce: pointer to rfc3686 nonce
378 * @ctx1_iv_off: IV offset in CONTEXT1 register
379 * @is_qi: true when called from caam/qi
380 * @era: SEC Era
381 */
cnstr_shdsc_aead_decap(u32 * const desc,struct alginfo * cdata,struct alginfo * adata,unsigned int ivsize,unsigned int icvsize,const bool geniv,const bool is_rfc3686,u32 * nonce,const u32 ctx1_iv_off,const bool is_qi,int era)382 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
383 struct alginfo *adata, unsigned int ivsize,
384 unsigned int icvsize, const bool geniv,
385 const bool is_rfc3686, u32 *nonce,
386 const u32 ctx1_iv_off, const bool is_qi, int era)
387 {
388 /* Note: Context registers are saved. */
389 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
390
391 /* Class 2 operation */
392 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
393 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
394
395 if (is_qi) {
396 u32 *wait_load_cmd;
397
398 /* REG3 = assoclen */
399 append_seq_load(desc, 4, LDST_CLASS_DECO |
400 LDST_SRCDST_WORD_DECO_MATH3 |
401 (4 << LDST_OFFSET_SHIFT));
402
403 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
404 JUMP_COND_CALM | JUMP_COND_NCP |
405 JUMP_COND_NOP | JUMP_COND_NIP |
406 JUMP_COND_NIFP);
407 set_jump_tgt_here(desc, wait_load_cmd);
408
409 if (!geniv)
410 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
411 LDST_SRCDST_BYTE_CONTEXT |
412 (ctx1_iv_off << LDST_OFFSET_SHIFT));
413 }
414
415 /* Read and write assoclen bytes */
416 if (is_qi || era < 3) {
417 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
418 if (geniv)
419 append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM,
420 ivsize);
421 else
422 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3,
423 CAAM_CMD_SZ);
424 } else {
425 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
426 if (geniv)
427 append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM,
428 ivsize);
429 else
430 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD,
431 CAAM_CMD_SZ);
432 }
433
434 /* Skip assoc data */
435 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
436
437 /* read assoc before reading payload */
438 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
439 KEY_VLF);
440
441 if (geniv) {
442 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
443 LDST_SRCDST_BYTE_CONTEXT |
444 (ctx1_iv_off << LDST_OFFSET_SHIFT));
445 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
446 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
447 }
448
449 /* Load Counter into CONTEXT1 reg */
450 if (is_rfc3686)
451 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
452 LDST_SRCDST_BYTE_CONTEXT |
453 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
454 LDST_OFFSET_SHIFT));
455
456 /* Choose operation */
457 if (ctx1_iv_off)
458 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
459 OP_ALG_DECRYPT);
460 else
461 append_dec_op1(desc, cdata->algtype);
462
463 /* Read and write cryptlen bytes */
464 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
465 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
466 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
467
468 /* Load ICV */
469 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
470 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
471
472 print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ",
473 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
474 1);
475 }
476 EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
477
478 /**
479 * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor
480 * (non-protocol) with HW-generated initialization
481 * vector.
482 * @desc: pointer to buffer used for descriptor construction
483 * @cdata: pointer to block cipher transform definitions
484 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
485 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
486 * @adata: pointer to authentication transform definitions.
487 * A split key is required for SEC Era < 6; the size of the split key
488 * is specified in this case. Valid algorithm values - one of
489 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
490 * with OP_ALG_AAI_HMAC_PRECOMP.
491 * @ivsize: initialization vector size
492 * @icvsize: integrity check value (ICV) size (truncated or full)
493 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
494 * @nonce: pointer to rfc3686 nonce
495 * @ctx1_iv_off: IV offset in CONTEXT1 register
496 * @is_qi: true when called from caam/qi
497 * @era: SEC Era
498 */
cnstr_shdsc_aead_givencap(u32 * const desc,struct alginfo * cdata,struct alginfo * adata,unsigned int ivsize,unsigned int icvsize,const bool is_rfc3686,u32 * nonce,const u32 ctx1_iv_off,const bool is_qi,int era)499 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
500 struct alginfo *adata, unsigned int ivsize,
501 unsigned int icvsize, const bool is_rfc3686,
502 u32 *nonce, const u32 ctx1_iv_off,
503 const bool is_qi, int era)
504 {
505 u32 geniv, moveiv;
506 u32 *wait_cmd;
507
508 /* Note: Context registers are saved. */
509 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
510
511 if (is_qi) {
512 u32 *wait_load_cmd;
513
514 /* REG3 = assoclen */
515 append_seq_load(desc, 4, LDST_CLASS_DECO |
516 LDST_SRCDST_WORD_DECO_MATH3 |
517 (4 << LDST_OFFSET_SHIFT));
518
519 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
520 JUMP_COND_CALM | JUMP_COND_NCP |
521 JUMP_COND_NOP | JUMP_COND_NIP |
522 JUMP_COND_NIFP);
523 set_jump_tgt_here(desc, wait_load_cmd);
524 }
525
526 if (is_rfc3686) {
527 if (is_qi)
528 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
529 LDST_SRCDST_BYTE_CONTEXT |
530 (ctx1_iv_off << LDST_OFFSET_SHIFT));
531
532 goto copy_iv;
533 }
534
535 /* Generate IV */
536 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
537 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
538 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
539 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
540 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
541 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
542 append_move(desc, MOVE_WAITCOMP |
543 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
544 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
545 (ivsize << MOVE_LEN_SHIFT));
546 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
547
548 copy_iv:
549 /* Copy IV to class 1 context */
550 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
551 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
552 (ivsize << MOVE_LEN_SHIFT));
553
554 /* Return to encryption */
555 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
556 OP_ALG_ENCRYPT);
557
558 /* Read and write assoclen bytes */
559 if (is_qi || era < 3) {
560 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
561 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
562 } else {
563 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
564 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
565 }
566
567 /* Skip assoc data */
568 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
569
570 /* read assoc before reading payload */
571 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
572 KEY_VLF);
573
574 /* Copy iv from outfifo to class 2 fifo */
575 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
576 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
577 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
578 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
579 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
580 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
581
582 /* Load Counter into CONTEXT1 reg */
583 if (is_rfc3686)
584 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
585 LDST_SRCDST_BYTE_CONTEXT |
586 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
587 LDST_OFFSET_SHIFT));
588
589 /* Class 1 operation */
590 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
591 OP_ALG_ENCRYPT);
592
593 /* Will write ivsize + cryptlen */
594 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
595
596 /* Not need to reload iv */
597 append_seq_fifo_load(desc, ivsize,
598 FIFOLD_CLASS_SKIP);
599
600 /* Will read cryptlen */
601 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
602
603 /*
604 * Wait for IV transfer (ofifo -> class2) to finish before starting
605 * ciphertext transfer (ofifo -> external memory).
606 */
607 wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP);
608 set_jump_tgt_here(desc, wait_cmd);
609
610 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
611 FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
612 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
613
614 /* Write ICV */
615 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
616 LDST_SRCDST_BYTE_CONTEXT);
617
618 print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ",
619 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
620 1);
621 }
622 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
623
624 /**
625 * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor
626 * @desc: pointer to buffer used for descriptor construction
627 * @cdata: pointer to block cipher transform definitions
628 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
629 * @ivsize: initialization vector size
630 * @icvsize: integrity check value (ICV) size (truncated or full)
631 * @is_qi: true when called from caam/qi
632 */
cnstr_shdsc_gcm_encap(u32 * const desc,struct alginfo * cdata,unsigned int ivsize,unsigned int icvsize,const bool is_qi)633 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
634 unsigned int ivsize, unsigned int icvsize,
635 const bool is_qi)
636 {
637 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
638 *zero_assoc_jump_cmd2;
639
640 init_sh_desc(desc, HDR_SHARE_SERIAL);
641
642 /* skip key loading if they are loaded due to sharing */
643 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
644 JUMP_COND_SHRD);
645 if (cdata->key_inline)
646 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
647 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
648 else
649 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
650 KEY_DEST_CLASS_REG);
651 set_jump_tgt_here(desc, key_jump_cmd);
652
653 /* class 1 operation */
654 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
655 OP_ALG_ENCRYPT);
656
657 if (is_qi) {
658 u32 *wait_load_cmd;
659
660 /* REG3 = assoclen */
661 append_seq_load(desc, 4, LDST_CLASS_DECO |
662 LDST_SRCDST_WORD_DECO_MATH3 |
663 (4 << LDST_OFFSET_SHIFT));
664
665 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
666 JUMP_COND_CALM | JUMP_COND_NCP |
667 JUMP_COND_NOP | JUMP_COND_NIP |
668 JUMP_COND_NIFP);
669 set_jump_tgt_here(desc, wait_load_cmd);
670
671 append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM,
672 ivsize);
673 } else {
674 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0,
675 CAAM_CMD_SZ);
676 }
677
678 /* if assoclen + cryptlen is ZERO, skip to ICV write */
679 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
680 JUMP_COND_MATH_Z);
681
682 if (is_qi)
683 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
684 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
685
686 /* if assoclen is ZERO, skip reading the assoc data */
687 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
688 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
689 JUMP_COND_MATH_Z);
690
691 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
692
693 /* skip assoc data */
694 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
695
696 /* cryptlen = seqinlen - assoclen */
697 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
698
699 /* if cryptlen is ZERO jump to zero-payload commands */
700 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
701 JUMP_COND_MATH_Z);
702
703 /* read assoc data */
704 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
705 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
706 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
707
708 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
709
710 /* write encrypted data */
711 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
712
713 /* read payload data */
714 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
715 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
716
717 /* jump to ICV writing */
718 if (is_qi)
719 append_jump(desc, JUMP_TEST_ALL | 4);
720 else
721 append_jump(desc, JUMP_TEST_ALL | 2);
722
723 /* zero-payload commands */
724 set_jump_tgt_here(desc, zero_payload_jump_cmd);
725
726 /* read assoc data */
727 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
728 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
729 if (is_qi)
730 /* jump to ICV writing */
731 append_jump(desc, JUMP_TEST_ALL | 2);
732
733 /* There is no input data */
734 set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
735
736 if (is_qi)
737 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
738 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
739 FIFOLD_TYPE_LAST1);
740
741 /* write ICV */
742 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
743 LDST_SRCDST_BYTE_CONTEXT);
744
745 print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ",
746 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
747 1);
748 }
749 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
750
751 /**
752 * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor
753 * @desc: pointer to buffer used for descriptor construction
754 * @cdata: pointer to block cipher transform definitions
755 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
756 * @ivsize: initialization vector size
757 * @icvsize: integrity check value (ICV) size (truncated or full)
758 * @is_qi: true when called from caam/qi
759 */
cnstr_shdsc_gcm_decap(u32 * const desc,struct alginfo * cdata,unsigned int ivsize,unsigned int icvsize,const bool is_qi)760 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
761 unsigned int ivsize, unsigned int icvsize,
762 const bool is_qi)
763 {
764 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
765
766 init_sh_desc(desc, HDR_SHARE_SERIAL);
767
768 /* skip key loading if they are loaded due to sharing */
769 key_jump_cmd = append_jump(desc, JUMP_JSL |
770 JUMP_TEST_ALL | JUMP_COND_SHRD);
771 if (cdata->key_inline)
772 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
773 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
774 else
775 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
776 KEY_DEST_CLASS_REG);
777 set_jump_tgt_here(desc, key_jump_cmd);
778
779 /* class 1 operation */
780 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
781 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
782
783 if (is_qi) {
784 u32 *wait_load_cmd;
785
786 /* REG3 = assoclen */
787 append_seq_load(desc, 4, LDST_CLASS_DECO |
788 LDST_SRCDST_WORD_DECO_MATH3 |
789 (4 << LDST_OFFSET_SHIFT));
790
791 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
792 JUMP_COND_CALM | JUMP_COND_NCP |
793 JUMP_COND_NOP | JUMP_COND_NIP |
794 JUMP_COND_NIFP);
795 set_jump_tgt_here(desc, wait_load_cmd);
796
797 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
798 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
799 }
800
801 /* if assoclen is ZERO, skip reading the assoc data */
802 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
803 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
804 JUMP_COND_MATH_Z);
805
806 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
807
808 /* skip assoc data */
809 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
810
811 /* read assoc data */
812 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
813 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
814
815 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
816
817 /* cryptlen = seqoutlen - assoclen */
818 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
819
820 /* jump to zero-payload command if cryptlen is zero */
821 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
822 JUMP_COND_MATH_Z);
823
824 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
825
826 /* store encrypted data */
827 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
828
829 /* read payload data */
830 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
831 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
832
833 /* zero-payload command */
834 set_jump_tgt_here(desc, zero_payload_jump_cmd);
835
836 /* read ICV */
837 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
838 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
839
840 print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ",
841 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
842 1);
843 }
844 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
845
846 /**
847 * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor
848 * (non-protocol).
849 * @desc: pointer to buffer used for descriptor construction
850 * @cdata: pointer to block cipher transform definitions
851 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
852 * @ivsize: initialization vector size
853 * @icvsize: integrity check value (ICV) size (truncated or full)
854 * @is_qi: true when called from caam/qi
855 *
856 * Input sequence: AAD | PTXT
857 * Output sequence: AAD | CTXT | ICV
858 * AAD length (assoclen), which includes the IV length, is available in Math3.
859 */
cnstr_shdsc_rfc4106_encap(u32 * const desc,struct alginfo * cdata,unsigned int ivsize,unsigned int icvsize,const bool is_qi)860 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
861 unsigned int ivsize, unsigned int icvsize,
862 const bool is_qi)
863 {
864 u32 *key_jump_cmd, *zero_cryptlen_jump_cmd, *skip_instructions;
865 init_sh_desc(desc, HDR_SHARE_SERIAL);
866
867 /* Skip key loading if it is loaded due to sharing */
868 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
869 JUMP_COND_SHRD);
870 if (cdata->key_inline)
871 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
872 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
873 else
874 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
875 KEY_DEST_CLASS_REG);
876 set_jump_tgt_here(desc, key_jump_cmd);
877
878 /* Class 1 operation */
879 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
880 OP_ALG_ENCRYPT);
881
882 if (is_qi) {
883 u32 *wait_load_cmd;
884
885 /* REG3 = assoclen */
886 append_seq_load(desc, 4, LDST_CLASS_DECO |
887 LDST_SRCDST_WORD_DECO_MATH3 |
888 (4 << LDST_OFFSET_SHIFT));
889
890 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
891 JUMP_COND_CALM | JUMP_COND_NCP |
892 JUMP_COND_NOP | JUMP_COND_NIP |
893 JUMP_COND_NIFP);
894 set_jump_tgt_here(desc, wait_load_cmd);
895
896 /* Read salt and IV */
897 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
898 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
899 FIFOLD_TYPE_IV);
900 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
901 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
902 }
903
904 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
905 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
906
907 /* Skip AAD */
908 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
909
910 /* Read cryptlen and set this value into VARSEQOUTLEN */
911 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
912
913 /* If cryptlen is ZERO jump to AAD command */
914 zero_cryptlen_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
915 JUMP_COND_MATH_Z);
916
917 /* Read AAD data */
918 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
919 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
920
921 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
922 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA);
923
924 /* Skip IV */
925 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
926 append_math_add(desc, VARSEQINLEN, VARSEQOUTLEN, REG0, CAAM_CMD_SZ);
927
928 /* Write encrypted data */
929 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
930
931 /* Read payload data */
932 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
933 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
934
935 /* Jump instructions to avoid double reading of AAD */
936 skip_instructions = append_jump(desc, JUMP_TEST_ALL);
937
938 /* There is no input data, cryptlen = 0 */
939 set_jump_tgt_here(desc, zero_cryptlen_jump_cmd);
940
941 /* Read AAD */
942 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
943 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
944
945 set_jump_tgt_here(desc, skip_instructions);
946
947 /* Write ICV */
948 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
949 LDST_SRCDST_BYTE_CONTEXT);
950
951 print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ",
952 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
953 1);
954 }
955 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
956
957 /**
958 * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor
959 * (non-protocol).
960 * @desc: pointer to buffer used for descriptor construction
961 * @cdata: pointer to block cipher transform definitions
962 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
963 * @ivsize: initialization vector size
964 * @icvsize: integrity check value (ICV) size (truncated or full)
965 * @is_qi: true when called from caam/qi
966 */
cnstr_shdsc_rfc4106_decap(u32 * const desc,struct alginfo * cdata,unsigned int ivsize,unsigned int icvsize,const bool is_qi)967 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
968 unsigned int ivsize, unsigned int icvsize,
969 const bool is_qi)
970 {
971 u32 *key_jump_cmd;
972
973 init_sh_desc(desc, HDR_SHARE_SERIAL);
974
975 /* Skip key loading if it is loaded due to sharing */
976 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
977 JUMP_COND_SHRD);
978 if (cdata->key_inline)
979 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
980 cdata->keylen, CLASS_1 |
981 KEY_DEST_CLASS_REG);
982 else
983 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
984 KEY_DEST_CLASS_REG);
985 set_jump_tgt_here(desc, key_jump_cmd);
986
987 /* Class 1 operation */
988 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
989 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
990
991 if (is_qi) {
992 u32 *wait_load_cmd;
993
994 /* REG3 = assoclen */
995 append_seq_load(desc, 4, LDST_CLASS_DECO |
996 LDST_SRCDST_WORD_DECO_MATH3 |
997 (4 << LDST_OFFSET_SHIFT));
998
999 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1000 JUMP_COND_CALM | JUMP_COND_NCP |
1001 JUMP_COND_NOP | JUMP_COND_NIP |
1002 JUMP_COND_NIFP);
1003 set_jump_tgt_here(desc, wait_load_cmd);
1004
1005 /* Read salt and IV */
1006 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1007 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1008 FIFOLD_TYPE_IV);
1009 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1010 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1011 }
1012
1013 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
1014 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1015
1016 /* Read assoc data */
1017 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1018 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1019
1020 /* Skip IV */
1021 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
1022
1023 /* Will read cryptlen bytes */
1024 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1025
1026 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1027 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1028
1029 /* Skip assoc data */
1030 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1031
1032 /* Will write cryptlen bytes */
1033 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1034
1035 /* Store payload data */
1036 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1037
1038 /* Read encrypted data */
1039 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1040 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1041
1042 /* Read ICV */
1043 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1044 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1045
1046 print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ",
1047 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1048 1);
1049 }
1050 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
1051
1052 /**
1053 * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor
1054 * (non-protocol).
1055 * @desc: pointer to buffer used for descriptor construction
1056 * @cdata: pointer to block cipher transform definitions
1057 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1058 * @ivsize: initialization vector size
1059 * @icvsize: integrity check value (ICV) size (truncated or full)
1060 * @is_qi: true when called from caam/qi
1061 */
cnstr_shdsc_rfc4543_encap(u32 * const desc,struct alginfo * cdata,unsigned int ivsize,unsigned int icvsize,const bool is_qi)1062 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
1063 unsigned int ivsize, unsigned int icvsize,
1064 const bool is_qi)
1065 {
1066 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1067
1068 init_sh_desc(desc, HDR_SHARE_SERIAL);
1069
1070 /* Skip key loading if it is loaded due to sharing */
1071 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1072 JUMP_COND_SHRD);
1073 if (cdata->key_inline)
1074 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1075 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1076 else
1077 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1078 KEY_DEST_CLASS_REG);
1079 set_jump_tgt_here(desc, key_jump_cmd);
1080
1081 /* Class 1 operation */
1082 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1083 OP_ALG_ENCRYPT);
1084
1085 if (is_qi) {
1086 /* assoclen is not needed, skip it */
1087 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1088
1089 /* Read salt and IV */
1090 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1091 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1092 FIFOLD_TYPE_IV);
1093 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1094 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1095 }
1096
1097 /* assoclen + cryptlen = seqinlen */
1098 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1099
1100 /*
1101 * MOVE_LEN opcode is not available in all SEC HW revisions,
1102 * thus need to do some magic, i.e. self-patch the descriptor
1103 * buffer.
1104 */
1105 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1106 (0x6 << MOVE_LEN_SHIFT));
1107 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1108 (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1109
1110 /* Will read assoclen + cryptlen bytes */
1111 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1112
1113 /* Will write assoclen + cryptlen bytes */
1114 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1115
1116 /* Read and write assoclen + cryptlen bytes */
1117 aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1118
1119 set_move_tgt_here(desc, read_move_cmd);
1120 set_move_tgt_here(desc, write_move_cmd);
1121 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1122 /* Move payload data to OFIFO */
1123 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1124
1125 /* Write ICV */
1126 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
1127 LDST_SRCDST_BYTE_CONTEXT);
1128
1129 print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ",
1130 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1131 1);
1132 }
1133 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
1134
1135 /**
1136 * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor
1137 * (non-protocol).
1138 * @desc: pointer to buffer used for descriptor construction
1139 * @cdata: pointer to block cipher transform definitions
1140 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1141 * @ivsize: initialization vector size
1142 * @icvsize: integrity check value (ICV) size (truncated or full)
1143 * @is_qi: true when called from caam/qi
1144 */
cnstr_shdsc_rfc4543_decap(u32 * const desc,struct alginfo * cdata,unsigned int ivsize,unsigned int icvsize,const bool is_qi)1145 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
1146 unsigned int ivsize, unsigned int icvsize,
1147 const bool is_qi)
1148 {
1149 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1150
1151 init_sh_desc(desc, HDR_SHARE_SERIAL);
1152
1153 /* Skip key loading if it is loaded due to sharing */
1154 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1155 JUMP_COND_SHRD);
1156 if (cdata->key_inline)
1157 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1158 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1159 else
1160 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1161 KEY_DEST_CLASS_REG);
1162 set_jump_tgt_here(desc, key_jump_cmd);
1163
1164 /* Class 1 operation */
1165 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1166 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1167
1168 if (is_qi) {
1169 /* assoclen is not needed, skip it */
1170 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1171
1172 /* Read salt and IV */
1173 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1174 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1175 FIFOLD_TYPE_IV);
1176 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1177 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1178 }
1179
1180 /* assoclen + cryptlen = seqoutlen */
1181 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1182
1183 /*
1184 * MOVE_LEN opcode is not available in all SEC HW revisions,
1185 * thus need to do some magic, i.e. self-patch the descriptor
1186 * buffer.
1187 */
1188 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1189 (0x6 << MOVE_LEN_SHIFT));
1190 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1191 (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1192
1193 /* Will read assoclen + cryptlen bytes */
1194 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1195
1196 /* Will write assoclen + cryptlen bytes */
1197 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1198
1199 /* Store payload data */
1200 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1201
1202 /* In-snoop assoclen + cryptlen data */
1203 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1204 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1205
1206 set_move_tgt_here(desc, read_move_cmd);
1207 set_move_tgt_here(desc, write_move_cmd);
1208 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1209 /* Move payload data to OFIFO */
1210 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1211 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1212
1213 /* Read ICV */
1214 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1215 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1216
1217 print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ",
1218 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1219 1);
1220 }
1221 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
1222
1223 /**
1224 * cnstr_shdsc_chachapoly - Chacha20 + Poly1305 generic AEAD (rfc7539) and
1225 * IPsec ESP (rfc7634, a.k.a. rfc7539esp) shared
1226 * descriptor (non-protocol).
1227 * @desc: pointer to buffer used for descriptor construction
1228 * @cdata: pointer to block cipher transform definitions
1229 * Valid algorithm values - OP_ALG_ALGSEL_CHACHA20 ANDed with
1230 * OP_ALG_AAI_AEAD.
1231 * @adata: pointer to authentication transform definitions
1232 * Valid algorithm values - OP_ALG_ALGSEL_POLY1305 ANDed with
1233 * OP_ALG_AAI_AEAD.
1234 * @ivsize: initialization vector size
1235 * @icvsize: integrity check value (ICV) size (truncated or full)
1236 * @encap: true if encapsulation, false if decapsulation
1237 * @is_qi: true when called from caam/qi
1238 */
cnstr_shdsc_chachapoly(u32 * const desc,struct alginfo * cdata,struct alginfo * adata,unsigned int ivsize,unsigned int icvsize,const bool encap,const bool is_qi)1239 void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata,
1240 struct alginfo *adata, unsigned int ivsize,
1241 unsigned int icvsize, const bool encap,
1242 const bool is_qi)
1243 {
1244 u32 *key_jump_cmd, *wait_cmd;
1245 u32 nfifo;
1246 const bool is_ipsec = (ivsize != CHACHAPOLY_IV_SIZE);
1247
1248 /* Note: Context registers are saved. */
1249 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1250
1251 /* skip key loading if they are loaded due to sharing */
1252 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1253 JUMP_COND_SHRD);
1254
1255 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen,
1256 CLASS_1 | KEY_DEST_CLASS_REG);
1257
1258 /* For IPsec load the salt from keymat in the context register */
1259 if (is_ipsec)
1260 append_load_as_imm(desc, cdata->key_virt + cdata->keylen, 4,
1261 LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT |
1262 4 << LDST_OFFSET_SHIFT);
1263
1264 set_jump_tgt_here(desc, key_jump_cmd);
1265
1266 /* Class 2 and 1 operations: Poly & ChaCha */
1267 if (encap) {
1268 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1269 OP_ALG_ENCRYPT);
1270 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1271 OP_ALG_ENCRYPT);
1272 } else {
1273 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1274 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1275 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1276 OP_ALG_DECRYPT);
1277 }
1278
1279 if (is_qi) {
1280 u32 *wait_load_cmd;
1281 u32 ctx1_iv_off = is_ipsec ? 8 : 4;
1282
1283 /* REG3 = assoclen */
1284 append_seq_load(desc, 4, LDST_CLASS_DECO |
1285 LDST_SRCDST_WORD_DECO_MATH3 |
1286 4 << LDST_OFFSET_SHIFT);
1287
1288 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1289 JUMP_COND_CALM | JUMP_COND_NCP |
1290 JUMP_COND_NOP | JUMP_COND_NIP |
1291 JUMP_COND_NIFP);
1292 set_jump_tgt_here(desc, wait_load_cmd);
1293
1294 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
1295 LDST_SRCDST_BYTE_CONTEXT |
1296 ctx1_iv_off << LDST_OFFSET_SHIFT);
1297 }
1298
1299 /*
1300 * MAGIC with NFIFO
1301 * Read associated data from the input and send them to class1 and
1302 * class2 alignment blocks. From class1 send data to output fifo and
1303 * then write it to memory since we don't need to encrypt AD.
1304 */
1305 nfifo = NFIFOENTRY_DEST_BOTH | NFIFOENTRY_FC1 | NFIFOENTRY_FC2 |
1306 NFIFOENTRY_DTYPE_POLY | NFIFOENTRY_BND;
1307 append_load_imm_u32(desc, nfifo, LDST_CLASS_IND_CCB |
1308 LDST_SRCDST_WORD_INFO_FIFO_SM | LDLEN_MATH3);
1309
1310 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1311 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1312 append_seq_fifo_load(desc, 0, FIFOLD_TYPE_NOINFOFIFO |
1313 FIFOLD_CLASS_CLASS1 | LDST_VLF);
1314 append_move_len(desc, MOVE_AUX_LS | MOVE_SRC_AUX_ABLK |
1315 MOVE_DEST_OUTFIFO | MOVELEN_MRSEL_MATH3);
1316 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF);
1317
1318 /* IPsec - copy IV at the output */
1319 if (is_ipsec)
1320 append_seq_fifo_store(desc, ivsize, FIFOST_TYPE_METADATA |
1321 0x2 << 25);
1322
1323 wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL |
1324 JUMP_COND_NOP | JUMP_TEST_ALL);
1325 set_jump_tgt_here(desc, wait_cmd);
1326
1327 if (encap) {
1328 /* Read and write cryptlen bytes */
1329 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1330 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0,
1331 CAAM_CMD_SZ);
1332 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
1333
1334 /* Write ICV */
1335 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
1336 LDST_SRCDST_BYTE_CONTEXT);
1337 } else {
1338 /* Read and write cryptlen bytes */
1339 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0,
1340 CAAM_CMD_SZ);
1341 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0,
1342 CAAM_CMD_SZ);
1343 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
1344
1345 /* Load ICV for verification */
1346 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
1347 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
1348 }
1349
1350 print_hex_dump_debug("chachapoly shdesc@" __stringify(__LINE__)": ",
1351 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1352 1);
1353 }
1354 EXPORT_SYMBOL(cnstr_shdsc_chachapoly);
1355
1356 /* For skcipher encrypt and decrypt, read from req->src and write to req->dst */
skcipher_append_src_dst(u32 * desc)1357 static inline void skcipher_append_src_dst(u32 *desc)
1358 {
1359 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1360 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1361 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
1362 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1363 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
1364 }
1365
1366 /**
1367 * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor
1368 * @desc: pointer to buffer used for descriptor construction
1369 * @cdata: pointer to block cipher transform definitions
1370 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1371 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1372 * - OP_ALG_ALGSEL_CHACHA20
1373 * @ivsize: initialization vector size
1374 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1375 * @ctx1_iv_off: IV offset in CONTEXT1 register
1376 */
cnstr_shdsc_skcipher_encap(u32 * const desc,struct alginfo * cdata,unsigned int ivsize,const bool is_rfc3686,const u32 ctx1_iv_off)1377 void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
1378 unsigned int ivsize, const bool is_rfc3686,
1379 const u32 ctx1_iv_off)
1380 {
1381 u32 *key_jump_cmd;
1382
1383 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1384 /* Skip if already shared */
1385 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1386 JUMP_COND_SHRD);
1387
1388 /* Load class1 key only */
1389 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1390 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1391
1392 /* Load nonce into CONTEXT1 reg */
1393 if (is_rfc3686) {
1394 const u8 *nonce = cdata->key_virt + cdata->keylen;
1395
1396 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1397 LDST_CLASS_IND_CCB |
1398 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1399 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1400 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1401 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1402 }
1403
1404 set_jump_tgt_here(desc, key_jump_cmd);
1405
1406 /* Load IV, if there is one */
1407 if (ivsize)
1408 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1409 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1410 LDST_OFFSET_SHIFT));
1411
1412 /* Load counter into CONTEXT1 reg */
1413 if (is_rfc3686)
1414 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1415 LDST_SRCDST_BYTE_CONTEXT |
1416 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1417 LDST_OFFSET_SHIFT));
1418
1419 /* Load operation */
1420 append_operation(desc, cdata->algtype | OP_ALG_AS_INIT |
1421 OP_ALG_ENCRYPT);
1422
1423 /* Perform operation */
1424 skcipher_append_src_dst(desc);
1425
1426 /* Store IV */
1427 if (ivsize)
1428 append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1429 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1430 LDST_OFFSET_SHIFT));
1431
1432 print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ",
1433 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1434 1);
1435 }
1436 EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap);
1437
1438 /**
1439 * cnstr_shdsc_skcipher_decap - skcipher decapsulation shared descriptor
1440 * @desc: pointer to buffer used for descriptor construction
1441 * @cdata: pointer to block cipher transform definitions
1442 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1443 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1444 * - OP_ALG_ALGSEL_CHACHA20
1445 * @ivsize: initialization vector size
1446 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1447 * @ctx1_iv_off: IV offset in CONTEXT1 register
1448 */
cnstr_shdsc_skcipher_decap(u32 * const desc,struct alginfo * cdata,unsigned int ivsize,const bool is_rfc3686,const u32 ctx1_iv_off)1449 void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
1450 unsigned int ivsize, const bool is_rfc3686,
1451 const u32 ctx1_iv_off)
1452 {
1453 u32 *key_jump_cmd;
1454
1455 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1456 /* Skip if already shared */
1457 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1458 JUMP_COND_SHRD);
1459
1460 /* Load class1 key only */
1461 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1462 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1463
1464 /* Load nonce into CONTEXT1 reg */
1465 if (is_rfc3686) {
1466 const u8 *nonce = cdata->key_virt + cdata->keylen;
1467
1468 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1469 LDST_CLASS_IND_CCB |
1470 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1471 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1472 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1473 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1474 }
1475
1476 set_jump_tgt_here(desc, key_jump_cmd);
1477
1478 /* Load IV, if there is one */
1479 if (ivsize)
1480 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1481 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1482 LDST_OFFSET_SHIFT));
1483
1484 /* Load counter into CONTEXT1 reg */
1485 if (is_rfc3686)
1486 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1487 LDST_SRCDST_BYTE_CONTEXT |
1488 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1489 LDST_OFFSET_SHIFT));
1490
1491 /* Choose operation */
1492 if (ctx1_iv_off)
1493 append_operation(desc, cdata->algtype | OP_ALG_AS_INIT |
1494 OP_ALG_DECRYPT);
1495 else
1496 append_dec_op1(desc, cdata->algtype);
1497
1498 /* Perform operation */
1499 skcipher_append_src_dst(desc);
1500
1501 /* Store IV */
1502 if (ivsize)
1503 append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1504 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1505 LDST_OFFSET_SHIFT));
1506
1507 print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ",
1508 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1509 1);
1510 }
1511 EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap);
1512
1513 /**
1514 * cnstr_shdsc_xts_skcipher_encap - xts skcipher encapsulation shared descriptor
1515 * @desc: pointer to buffer used for descriptor construction
1516 * @cdata: pointer to block cipher transform definitions
1517 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1518 */
cnstr_shdsc_xts_skcipher_encap(u32 * const desc,struct alginfo * cdata)1519 void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata)
1520 {
1521 __be64 sector_size = cpu_to_be64(512);
1522 u32 *key_jump_cmd;
1523
1524 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1525 /* Skip if already shared */
1526 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1527 JUMP_COND_SHRD);
1528
1529 /* Load class1 keys only */
1530 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1531 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1532
1533 /* Load sector size with index 40 bytes (0x28) */
1534 append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB |
1535 LDST_SRCDST_BYTE_CONTEXT |
1536 (0x28 << LDST_OFFSET_SHIFT));
1537
1538 set_jump_tgt_here(desc, key_jump_cmd);
1539
1540 /*
1541 * create sequence for loading the sector index
1542 * Upper 8B of IV - will be used as sector index
1543 * Lower 8B of IV - will be discarded
1544 */
1545 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1546 (0x20 << LDST_OFFSET_SHIFT));
1547 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1548
1549 /* Load operation */
1550 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1551 OP_ALG_ENCRYPT);
1552
1553 /* Perform operation */
1554 skcipher_append_src_dst(desc);
1555
1556 /* Store upper 8B of IV */
1557 append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1558 (0x20 << LDST_OFFSET_SHIFT));
1559
1560 print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__)
1561 ": ", DUMP_PREFIX_ADDRESS, 16, 4,
1562 desc, desc_bytes(desc), 1);
1563 }
1564 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap);
1565
1566 /**
1567 * cnstr_shdsc_xts_skcipher_decap - xts skcipher decapsulation shared descriptor
1568 * @desc: pointer to buffer used for descriptor construction
1569 * @cdata: pointer to block cipher transform definitions
1570 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1571 */
cnstr_shdsc_xts_skcipher_decap(u32 * const desc,struct alginfo * cdata)1572 void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata)
1573 {
1574 __be64 sector_size = cpu_to_be64(512);
1575 u32 *key_jump_cmd;
1576
1577 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1578 /* Skip if already shared */
1579 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1580 JUMP_COND_SHRD);
1581
1582 /* Load class1 key only */
1583 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1584 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1585
1586 /* Load sector size with index 40 bytes (0x28) */
1587 append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB |
1588 LDST_SRCDST_BYTE_CONTEXT |
1589 (0x28 << LDST_OFFSET_SHIFT));
1590
1591 set_jump_tgt_here(desc, key_jump_cmd);
1592
1593 /*
1594 * create sequence for loading the sector index
1595 * Upper 8B of IV - will be used as sector index
1596 * Lower 8B of IV - will be discarded
1597 */
1598 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1599 (0x20 << LDST_OFFSET_SHIFT));
1600 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1601
1602 /* Load operation */
1603 append_dec_op1(desc, cdata->algtype);
1604
1605 /* Perform operation */
1606 skcipher_append_src_dst(desc);
1607
1608 /* Store upper 8B of IV */
1609 append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1610 (0x20 << LDST_OFFSET_SHIFT));
1611
1612 print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__)
1613 ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
1614 desc_bytes(desc), 1);
1615 }
1616 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap);
1617
1618 MODULE_LICENSE("GPL");
1619 MODULE_DESCRIPTION("FSL CAAM descriptor support");
1620 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
1621