1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __NITROX_REQ_H
3 #define __NITROX_REQ_H
4
5 #include <linux/dma-mapping.h>
6 #include <crypto/aes.h>
7
8 #include "nitrox_dev.h"
9
10 #define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL
11 #define PRIO 4001
12
13 typedef void (*sereq_completion_t)(void *req, int err);
14
15 /**
16 * struct gphdr - General purpose Header
17 * @param0: first parameter.
18 * @param1: second parameter.
19 * @param2: third parameter.
20 * @param3: fourth parameter.
21 *
22 * Params tell the iv and enc/dec data offsets.
23 */
24 struct gphdr {
25 __be16 param0;
26 __be16 param1;
27 __be16 param2;
28 __be16 param3;
29 };
30
31 /**
32 * struct se_req_ctrl - SE request information.
33 * @arg: Minor number of the opcode
34 * @ctxc: Context control.
35 * @unca: Uncertainity enabled.
36 * @info: Additional information for SE cores.
37 * @ctxl: Context length in bytes.
38 * @uddl: User defined data length
39 */
40 union se_req_ctrl {
41 u64 value;
42 struct {
43 u64 raz : 22;
44 u64 arg : 8;
45 u64 ctxc : 2;
46 u64 unca : 1;
47 u64 info : 3;
48 u64 unc : 8;
49 u64 ctxl : 12;
50 u64 uddl : 8;
51 } s;
52 };
53
54 #define MAX_IV_LEN 16
55
56 /**
57 * struct se_crypto_request - SE crypto request structure.
58 * @opcode: Request opcode (enc/dec)
59 * @flags: flags from crypto subsystem
60 * @ctx_handle: Crypto context handle.
61 * @gph: GP Header
62 * @ctrl: Request Information.
63 * @orh: ORH address
64 * @comp: completion address
65 * @src: Input sglist
66 * @dst: Output sglist
67 */
68 struct se_crypto_request {
69 u8 opcode;
70 gfp_t gfp;
71 u32 flags;
72 u64 ctx_handle;
73
74 struct gphdr gph;
75 union se_req_ctrl ctrl;
76 u64 *orh;
77 u64 *comp;
78
79 struct scatterlist *src;
80 struct scatterlist *dst;
81 };
82
83 /* Crypto opcodes */
84 #define FLEXI_CRYPTO_ENCRYPT_HMAC 0x33
85 #define ENCRYPT 0
86 #define DECRYPT 1
87
88 /* IV from context */
89 #define IV_FROM_CTX 0
90 /* IV from Input data */
91 #define IV_FROM_DPTR 1
92
93 /**
94 * cipher opcodes for firmware
95 */
96 enum flexi_cipher {
97 CIPHER_NULL = 0,
98 CIPHER_3DES_CBC,
99 CIPHER_3DES_ECB,
100 CIPHER_AES_CBC,
101 CIPHER_AES_ECB,
102 CIPHER_AES_CFB,
103 CIPHER_AES_CTR,
104 CIPHER_AES_GCM,
105 CIPHER_AES_XTS,
106 CIPHER_AES_CCM,
107 CIPHER_AES_CBC_CTS,
108 CIPHER_AES_ECB_CTS,
109 CIPHER_INVALID
110 };
111
112 enum flexi_auth {
113 AUTH_NULL = 0,
114 AUTH_MD5,
115 AUTH_SHA1,
116 AUTH_SHA2_SHA224,
117 AUTH_SHA2_SHA256,
118 AUTH_SHA2_SHA384,
119 AUTH_SHA2_SHA512,
120 AUTH_GMAC,
121 AUTH_INVALID
122 };
123
124 /**
125 * struct crypto_keys - Crypto keys
126 * @key: Encryption key or KEY1 for AES-XTS
127 * @iv: Encryption IV or Tweak for AES-XTS
128 */
129 struct crypto_keys {
130 union {
131 u8 key[AES_MAX_KEY_SIZE];
132 u8 key1[AES_MAX_KEY_SIZE];
133 } u;
134 u8 iv[AES_BLOCK_SIZE];
135 };
136
137 /**
138 * struct auth_keys - Authentication keys
139 * @ipad: IPAD or KEY2 for AES-XTS
140 * @opad: OPAD or AUTH KEY if auth_input_type = 1
141 */
142 struct auth_keys {
143 union {
144 u8 ipad[64];
145 u8 key2[64];
146 } u;
147 u8 opad[64];
148 };
149
150 union fc_ctx_flags {
151 __be64 f;
152 struct {
153 #if defined(__BIG_ENDIAN_BITFIELD)
154 u64 cipher_type : 4;
155 u64 reserved_59 : 1;
156 u64 aes_keylen : 2;
157 u64 iv_source : 1;
158 u64 hash_type : 4;
159 u64 reserved_49_51 : 3;
160 u64 auth_input_type: 1;
161 u64 mac_len : 8;
162 u64 reserved_0_39 : 40;
163 #else
164 u64 reserved_0_39 : 40;
165 u64 mac_len : 8;
166 u64 auth_input_type: 1;
167 u64 reserved_49_51 : 3;
168 u64 hash_type : 4;
169 u64 iv_source : 1;
170 u64 aes_keylen : 2;
171 u64 reserved_59 : 1;
172 u64 cipher_type : 4;
173 #endif
174 } w0;
175 };
176 /**
177 * struct flexi_crypto_context - Crypto context
178 * @cipher_type: Encryption cipher type
179 * @aes_keylen: AES key length
180 * @iv_source: Encryption IV source
181 * @hash_type: Authentication type
182 * @auth_input_type: Authentication input type
183 * 1 - Authentication IV and KEY, microcode calculates OPAD/IPAD
184 * 0 - Authentication OPAD/IPAD
185 * @mac_len: mac length
186 * @crypto: Crypto keys
187 * @auth: Authentication keys
188 */
189 struct flexi_crypto_context {
190 union fc_ctx_flags flags;
191 struct crypto_keys crypto;
192 struct auth_keys auth;
193 };
194
195 struct crypto_ctx_hdr {
196 struct dma_pool *pool;
197 dma_addr_t dma;
198 void *vaddr;
199 };
200
201 struct nitrox_crypto_ctx {
202 struct nitrox_device *ndev;
203 union {
204 u64 ctx_handle;
205 struct flexi_crypto_context *fctx;
206 } u;
207 struct crypto_ctx_hdr *chdr;
208 sereq_completion_t callback;
209 };
210
211 struct nitrox_kcrypt_request {
212 struct se_crypto_request creq;
213 u8 *src;
214 u8 *dst;
215 u8 *iv_out;
216 };
217
218 /**
219 * struct nitrox_aead_rctx - AEAD request context
220 * @nkreq: Base request context
221 * @cryptlen: Encryption/Decryption data length
222 * @assoclen: AAD length
223 * @srclen: Input buffer length
224 * @dstlen: Output buffer length
225 * @iv: IV data
226 * @ivsize: IV data length
227 * @flags: AEAD req flags
228 * @ctx_handle: Device context handle
229 * @src: Source sglist
230 * @dst: Destination sglist
231 * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT)
232 */
233 struct nitrox_aead_rctx {
234 struct nitrox_kcrypt_request nkreq;
235 unsigned int cryptlen;
236 unsigned int assoclen;
237 unsigned int srclen;
238 unsigned int dstlen;
239 u8 *iv;
240 int ivsize;
241 u32 flags;
242 u64 ctx_handle;
243 struct scatterlist *src;
244 struct scatterlist *dst;
245 u8 ctrl_arg;
246 };
247
248 /**
249 * struct nitrox_rfc4106_rctx - rfc4106 cipher request context
250 * @base: AEAD request context
251 * @src: Source sglist
252 * @dst: Destination sglist
253 * @assoc: AAD
254 */
255 struct nitrox_rfc4106_rctx {
256 struct nitrox_aead_rctx base;
257 struct scatterlist src[3];
258 struct scatterlist dst[3];
259 u8 assoc[20];
260 };
261
262 /**
263 * struct pkt_instr_hdr - Packet Instruction Header
264 * @g: Gather used
265 * When [G] is set and [GSZ] != 0, the instruction is
266 * indirect gather instruction.
267 * When [G] is set and [GSZ] = 0, the instruction is
268 * direct gather instruction.
269 * @gsz: Number of pointers in the indirect gather list
270 * @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr
271 * and adds them to the packet after the pkt_instr_hdr but before any UDD
272 * @ssz: Not used by the input hardware. But can become slc_store_int[SSZ]
273 * when [IHI] is set.
274 * @fsz: The number of front data bytes directly included in the
275 * PCIe instruction.
276 * @tlen: The length of the input packet in bytes, include:
277 * - 16B pkt_hdr
278 * - Inline context bytes if any,
279 * - UDD if any,
280 * - packet payload bytes
281 */
282 union pkt_instr_hdr {
283 u64 value;
284 struct {
285 #if defined(__BIG_ENDIAN_BITFIELD)
286 u64 raz_48_63 : 16;
287 u64 g : 1;
288 u64 gsz : 7;
289 u64 ihi : 1;
290 u64 ssz : 7;
291 u64 raz_30_31 : 2;
292 u64 fsz : 6;
293 u64 raz_16_23 : 8;
294 u64 tlen : 16;
295 #else
296 u64 tlen : 16;
297 u64 raz_16_23 : 8;
298 u64 fsz : 6;
299 u64 raz_30_31 : 2;
300 u64 ssz : 7;
301 u64 ihi : 1;
302 u64 gsz : 7;
303 u64 g : 1;
304 u64 raz_48_63 : 16;
305 #endif
306 } s;
307 };
308
309 /**
310 * struct pkt_hdr - Packet Input Header
311 * @opcode: Request opcode (Major)
312 * @arg: Request opcode (Minor)
313 * @ctxc: Context control.
314 * @unca: When set [UNC] is the uncertainty count for an input packet.
315 * The hardware uses uncertainty counts to predict
316 * output buffer use and avoid deadlock.
317 * @info: Not used by input hardware. Available for use
318 * during SE processing.
319 * @destport: The expected destination port/ring/channel for the packet.
320 * @unc: Uncertainty count for an input packet.
321 * @grp: SE group that will process the input packet.
322 * @ctxl: Context Length in 64-bit words.
323 * @uddl: User-defined data (UDD) length in bytes.
324 * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases.
325 */
326 union pkt_hdr {
327 u64 value[2];
328 struct {
329 #if defined(__BIG_ENDIAN_BITFIELD)
330 u64 opcode : 8;
331 u64 arg : 8;
332 u64 ctxc : 2;
333 u64 unca : 1;
334 u64 raz_44 : 1;
335 u64 info : 3;
336 u64 destport : 9;
337 u64 unc : 8;
338 u64 raz_19_23 : 5;
339 u64 grp : 3;
340 u64 raz_15 : 1;
341 u64 ctxl : 7;
342 u64 uddl : 8;
343 #else
344 u64 uddl : 8;
345 u64 ctxl : 7;
346 u64 raz_15 : 1;
347 u64 grp : 3;
348 u64 raz_19_23 : 5;
349 u64 unc : 8;
350 u64 destport : 9;
351 u64 info : 3;
352 u64 raz_44 : 1;
353 u64 unca : 1;
354 u64 ctxc : 2;
355 u64 arg : 8;
356 u64 opcode : 8;
357 #endif
358 __be64 ctxp;
359 } s;
360 };
361
362 /**
363 * struct slc_store_info - Solicited Paceket Output Store Information.
364 * @ssz: The number of scatterlist pointers for the solicited output port
365 * packet.
366 * @rptr: The result pointer for the solicited output port packet.
367 * If [SSZ]=0, [RPTR] must point directly to a buffer on the remote
368 * host that is large enough to hold the entire output packet.
369 * If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4
370 * sglist components at [RPTR] on the remote host.
371 */
372 union slc_store_info {
373 u64 value[2];
374 struct {
375 #if defined(__BIG_ENDIAN_BITFIELD)
376 u64 raz_39_63 : 25;
377 u64 ssz : 7;
378 u64 raz_0_31 : 32;
379 #else
380 u64 raz_0_31 : 32;
381 u64 ssz : 7;
382 u64 raz_39_63 : 25;
383 #endif
384 __be64 rptr;
385 } s;
386 };
387
388 /**
389 * struct nps_pkt_instr - NPS Packet Instruction of SE cores.
390 * @dptr0 : Input pointer points to buffer in remote host.
391 * @ih: Packet Instruction Header (8 bytes)
392 * @irh: Packet Input Header (16 bytes)
393 * @slc: Solicited Packet Output Store Information (16 bytes)
394 * @fdata: Front data
395 *
396 * 64-Byte Instruction Format
397 */
398 struct nps_pkt_instr {
399 __be64 dptr0;
400 union pkt_instr_hdr ih;
401 union pkt_hdr irh;
402 union slc_store_info slc;
403 u64 fdata[2];
404 };
405
406 /**
407 * struct aqmq_command_s - The 32 byte command for AE processing.
408 * @opcode: Request opcode
409 * @param1: Request control parameter 1
410 * @param2: Request control parameter 2
411 * @dlen: Input length
412 * @dptr: Input pointer points to buffer in remote host
413 * @rptr: Result pointer points to buffer in remote host
414 * @grp: AQM Group (0..7)
415 * @cptr: Context pointer
416 */
417 struct aqmq_command_s {
418 __be16 opcode;
419 __be16 param1;
420 __be16 param2;
421 __be16 dlen;
422 __be64 dptr;
423 __be64 rptr;
424 union {
425 __be64 word3;
426 #if defined(__BIG_ENDIAN_BITFIELD)
427 u64 grp : 3;
428 u64 cptr : 61;
429 #else
430 u64 cptr : 61;
431 u64 grp : 3;
432 #endif
433 };
434 };
435
436 /**
437 * struct ctx_hdr - Book keeping data about the crypto context
438 * @pool: Pool used to allocate crypto context
439 * @dma: Base DMA address of the cypto context
440 * @ctx_dma: Actual usable crypto context for NITROX
441 */
442 struct ctx_hdr {
443 struct dma_pool *pool;
444 dma_addr_t dma;
445 dma_addr_t ctx_dma;
446 };
447
448 /*
449 * struct sglist_component - SG list component format
450 * @len0: The number of bytes at [PTR0] on the remote host.
451 * @len1: The number of bytes at [PTR1] on the remote host.
452 * @len2: The number of bytes at [PTR2] on the remote host.
453 * @len3: The number of bytes at [PTR3] on the remote host.
454 * @dma0: First pointer point to buffer in remote host.
455 * @dma1: Second pointer point to buffer in remote host.
456 * @dma2: Third pointer point to buffer in remote host.
457 * @dma3: Fourth pointer point to buffer in remote host.
458 */
459 struct nitrox_sgcomp {
460 __be16 len[4];
461 __be64 dma[4];
462 };
463
464 /*
465 * strutct nitrox_sgtable - SG list information
466 * @sgmap_cnt: Number of buffers mapped
467 * @total_bytes: Total bytes in sglist.
468 * @sgcomp_len: Total sglist components length.
469 * @sgcomp_dma: DMA address of sglist component.
470 * @sg: crypto request buffer.
471 * @sgcomp: sglist component for NITROX.
472 */
473 struct nitrox_sgtable {
474 u8 sgmap_cnt;
475 u16 total_bytes;
476 u32 sgcomp_len;
477 dma_addr_t sgcomp_dma;
478 struct scatterlist *sg;
479 struct nitrox_sgcomp *sgcomp;
480 };
481
482 /* Response Header Length */
483 #define ORH_HLEN 8
484 /* Completion bytes Length */
485 #define COMP_HLEN 8
486
487 struct resp_hdr {
488 u64 *orh;
489 u64 *completion;
490 };
491
492 typedef void (*completion_t)(void *arg, int err);
493
494 /**
495 * struct nitrox_softreq - Represents the NIROX Request.
496 * @response: response list entry
497 * @backlog: Backlog list entry
498 * @ndev: Device used to submit the request
499 * @cmdq: Command queue for submission
500 * @resp: Response headers
501 * @instr: 64B instruction
502 * @in: SG table for input
503 * @out SG table for output
504 * @tstamp: Request submitted time in jiffies
505 * @callback: callback after request completion/timeout
506 * @cb_arg: callback argument
507 */
508 struct nitrox_softreq {
509 struct list_head response;
510 struct list_head backlog;
511
512 u32 flags;
513 gfp_t gfp;
514 atomic_t status;
515
516 struct nitrox_device *ndev;
517 struct nitrox_cmdq *cmdq;
518
519 struct nps_pkt_instr instr;
520 struct resp_hdr resp;
521 struct nitrox_sgtable in;
522 struct nitrox_sgtable out;
523
524 unsigned long tstamp;
525
526 completion_t callback;
527 void *cb_arg;
528 };
529
flexi_aes_keylen(int keylen)530 static inline int flexi_aes_keylen(int keylen)
531 {
532 int aes_keylen;
533
534 switch (keylen) {
535 case AES_KEYSIZE_128:
536 aes_keylen = 1;
537 break;
538 case AES_KEYSIZE_192:
539 aes_keylen = 2;
540 break;
541 case AES_KEYSIZE_256:
542 aes_keylen = 3;
543 break;
544 default:
545 aes_keylen = -EINVAL;
546 break;
547 }
548 return aes_keylen;
549 }
550
alloc_req_buf(int nents,int extralen,gfp_t gfp)551 static inline void *alloc_req_buf(int nents, int extralen, gfp_t gfp)
552 {
553 size_t size;
554
555 size = sizeof(struct scatterlist) * nents;
556 size += extralen;
557
558 return kzalloc(size, gfp);
559 }
560
561 /**
562 * create_single_sg - Point SG entry to the data
563 * @sg: Destination SG list
564 * @buf: Data
565 * @buflen: Data length
566 *
567 * Returns next free entry in the destination SG list
568 **/
create_single_sg(struct scatterlist * sg,void * buf,int buflen)569 static inline struct scatterlist *create_single_sg(struct scatterlist *sg,
570 void *buf, int buflen)
571 {
572 sg_set_buf(sg, buf, buflen);
573 sg++;
574 return sg;
575 }
576
577 /**
578 * create_multi_sg - Create multiple sg entries with buflen data length from
579 * source sglist
580 * @to_sg: Destination SG list
581 * @from_sg: Source SG list
582 * @buflen: Data length
583 *
584 * Returns next free entry in the destination SG list
585 **/
create_multi_sg(struct scatterlist * to_sg,struct scatterlist * from_sg,int buflen)586 static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg,
587 struct scatterlist *from_sg,
588 int buflen)
589 {
590 struct scatterlist *sg = to_sg;
591 unsigned int sglen;
592
593 for (; buflen && from_sg; buflen -= sglen) {
594 sglen = from_sg->length;
595 if (sglen > buflen)
596 sglen = buflen;
597
598 sg_set_buf(sg, sg_virt(from_sg), sglen);
599 from_sg = sg_next(from_sg);
600 sg++;
601 }
602
603 return sg;
604 }
605
set_orh_value(u64 * orh)606 static inline void set_orh_value(u64 *orh)
607 {
608 WRITE_ONCE(*orh, PENDING_SIG);
609 }
610
set_comp_value(u64 * comp)611 static inline void set_comp_value(u64 *comp)
612 {
613 WRITE_ONCE(*comp, PENDING_SIG);
614 }
615
alloc_src_req_buf(struct nitrox_kcrypt_request * nkreq,int nents,int ivsize)616 static inline int alloc_src_req_buf(struct nitrox_kcrypt_request *nkreq,
617 int nents, int ivsize)
618 {
619 struct se_crypto_request *creq = &nkreq->creq;
620
621 nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp);
622 if (!nkreq->src)
623 return -ENOMEM;
624
625 return 0;
626 }
627
nitrox_creq_copy_iv(char * dst,char * src,int size)628 static inline void nitrox_creq_copy_iv(char *dst, char *src, int size)
629 {
630 memcpy(dst, src, size);
631 }
632
nitrox_creq_src_sg(char * iv,int ivsize)633 static inline struct scatterlist *nitrox_creq_src_sg(char *iv, int ivsize)
634 {
635 return (struct scatterlist *)(iv + ivsize);
636 }
637
nitrox_creq_set_src_sg(struct nitrox_kcrypt_request * nkreq,int nents,int ivsize,struct scatterlist * src,int buflen)638 static inline void nitrox_creq_set_src_sg(struct nitrox_kcrypt_request *nkreq,
639 int nents, int ivsize,
640 struct scatterlist *src, int buflen)
641 {
642 char *iv = nkreq->src;
643 struct scatterlist *sg;
644 struct se_crypto_request *creq = &nkreq->creq;
645
646 creq->src = nitrox_creq_src_sg(iv, ivsize);
647 sg = creq->src;
648 sg_init_table(sg, nents);
649
650 /* Input format:
651 * +----+----------------+
652 * | IV | SRC sg entries |
653 * +----+----------------+
654 */
655
656 /* IV */
657 sg = create_single_sg(sg, iv, ivsize);
658 /* SRC entries */
659 create_multi_sg(sg, src, buflen);
660 }
661
alloc_dst_req_buf(struct nitrox_kcrypt_request * nkreq,int nents)662 static inline int alloc_dst_req_buf(struct nitrox_kcrypt_request *nkreq,
663 int nents)
664 {
665 int extralen = ORH_HLEN + COMP_HLEN;
666 struct se_crypto_request *creq = &nkreq->creq;
667
668 nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp);
669 if (!nkreq->dst)
670 return -ENOMEM;
671
672 return 0;
673 }
674
nitrox_creq_set_orh(struct nitrox_kcrypt_request * nkreq)675 static inline void nitrox_creq_set_orh(struct nitrox_kcrypt_request *nkreq)
676 {
677 struct se_crypto_request *creq = &nkreq->creq;
678
679 creq->orh = (u64 *)(nkreq->dst);
680 set_orh_value(creq->orh);
681 }
682
nitrox_creq_set_comp(struct nitrox_kcrypt_request * nkreq)683 static inline void nitrox_creq_set_comp(struct nitrox_kcrypt_request *nkreq)
684 {
685 struct se_crypto_request *creq = &nkreq->creq;
686
687 creq->comp = (u64 *)(nkreq->dst + ORH_HLEN);
688 set_comp_value(creq->comp);
689 }
690
nitrox_creq_dst_sg(char * dst)691 static inline struct scatterlist *nitrox_creq_dst_sg(char *dst)
692 {
693 return (struct scatterlist *)(dst + ORH_HLEN + COMP_HLEN);
694 }
695
nitrox_creq_set_dst_sg(struct nitrox_kcrypt_request * nkreq,int nents,int ivsize,struct scatterlist * dst,int buflen)696 static inline void nitrox_creq_set_dst_sg(struct nitrox_kcrypt_request *nkreq,
697 int nents, int ivsize,
698 struct scatterlist *dst, int buflen)
699 {
700 struct se_crypto_request *creq = &nkreq->creq;
701 struct scatterlist *sg;
702 char *iv = nkreq->src;
703
704 creq->dst = nitrox_creq_dst_sg(nkreq->dst);
705 sg = creq->dst;
706 sg_init_table(sg, nents);
707
708 /* Output format:
709 * +-----+----+----------------+-----------------+
710 * | ORH | IV | DST sg entries | COMPLETION Bytes|
711 * +-----+----+----------------+-----------------+
712 */
713
714 /* ORH */
715 sg = create_single_sg(sg, creq->orh, ORH_HLEN);
716 /* IV */
717 sg = create_single_sg(sg, iv, ivsize);
718 /* DST entries */
719 sg = create_multi_sg(sg, dst, buflen);
720 /* COMPLETION Bytes */
721 create_single_sg(sg, creq->comp, COMP_HLEN);
722 }
723
724 #endif /* __NITROX_REQ_H */
725