1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Cryptographic API.
4 *
5 * Support for OMAP AES GCM HW acceleration.
6 *
7 * Copyright (c) 2016 Texas Instruments Incorporated
8 */
9
10 #include <linux/errno.h>
11 #include <linux/scatterlist.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/dmaengine.h>
14 #include <linux/omap-dma.h>
15 #include <linux/interrupt.h>
16 #include <crypto/aes.h>
17 #include <crypto/gcm.h>
18 #include <crypto/scatterwalk.h>
19 #include <crypto/skcipher.h>
20 #include <crypto/internal/aead.h>
21
22 #include "omap-crypto.h"
23 #include "omap-aes.h"
24
25 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
26 struct aead_request *req);
27
omap_aes_gcm_finish_req(struct omap_aes_dev * dd,int ret)28 static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)
29 {
30 struct aead_request *req = dd->aead_req;
31
32 dd->flags &= ~FLAGS_BUSY;
33 dd->in_sg = NULL;
34 dd->out_sg = NULL;
35
36 req->base.complete(&req->base, ret);
37 }
38
omap_aes_gcm_done_task(struct omap_aes_dev * dd)39 static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)
40 {
41 u8 *tag;
42 int alen, clen, i, ret = 0, nsg;
43 struct omap_aes_reqctx *rctx;
44
45 alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);
46 clen = ALIGN(dd->total, AES_BLOCK_SIZE);
47 rctx = aead_request_ctx(dd->aead_req);
48
49 nsg = !!(dd->assoc_len && dd->total);
50
51 dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
52 DMA_FROM_DEVICE);
53 dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
54 dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);
55 omap_aes_crypt_dma_stop(dd);
56
57 omap_crypto_cleanup(dd->out_sg, dd->orig_out,
58 dd->aead_req->assoclen, dd->total,
59 FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
60
61 if (dd->flags & FLAGS_ENCRYPT)
62 scatterwalk_map_and_copy(rctx->auth_tag,
63 dd->aead_req->dst,
64 dd->total + dd->aead_req->assoclen,
65 dd->authsize, 1);
66
67 omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,
68 FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);
69
70 omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,
71 FLAGS_IN_DATA_ST_SHIFT, dd->flags);
72
73 if (!(dd->flags & FLAGS_ENCRYPT)) {
74 tag = (u8 *)rctx->auth_tag;
75 for (i = 0; i < dd->authsize; i++) {
76 if (tag[i]) {
77 dev_err(dd->dev, "GCM decryption: Tag Message is wrong\n");
78 ret = -EBADMSG;
79 }
80 }
81 }
82
83 omap_aes_gcm_finish_req(dd, ret);
84 omap_aes_gcm_handle_queue(dd, NULL);
85 }
86
omap_aes_gcm_copy_buffers(struct omap_aes_dev * dd,struct aead_request * req)87 static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,
88 struct aead_request *req)
89 {
90 int alen, clen, cryptlen, assoclen, ret;
91 struct crypto_aead *aead = crypto_aead_reqtfm(req);
92 unsigned int authlen = crypto_aead_authsize(aead);
93 struct scatterlist *tmp, sg_arr[2];
94 int nsg;
95 u16 flags;
96
97 assoclen = req->assoclen;
98 cryptlen = req->cryptlen;
99
100 if (dd->flags & FLAGS_RFC4106_GCM)
101 assoclen -= 8;
102
103 if (!(dd->flags & FLAGS_ENCRYPT))
104 cryptlen -= authlen;
105
106 alen = ALIGN(assoclen, AES_BLOCK_SIZE);
107 clen = ALIGN(cryptlen, AES_BLOCK_SIZE);
108
109 nsg = !!(assoclen && cryptlen);
110
111 omap_aes_clear_copy_flags(dd);
112
113 sg_init_table(dd->in_sgl, nsg + 1);
114 if (assoclen) {
115 tmp = req->src;
116 ret = omap_crypto_align_sg(&tmp, assoclen,
117 AES_BLOCK_SIZE, dd->in_sgl,
118 OMAP_CRYPTO_COPY_DATA |
119 OMAP_CRYPTO_ZERO_BUF |
120 OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
121 FLAGS_ASSOC_DATA_ST_SHIFT,
122 &dd->flags);
123 }
124
125 if (cryptlen) {
126 tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen);
127
128 ret = omap_crypto_align_sg(&tmp, cryptlen,
129 AES_BLOCK_SIZE, &dd->in_sgl[nsg],
130 OMAP_CRYPTO_COPY_DATA |
131 OMAP_CRYPTO_ZERO_BUF |
132 OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
133 FLAGS_IN_DATA_ST_SHIFT,
134 &dd->flags);
135 }
136
137 dd->in_sg = dd->in_sgl;
138 dd->total = cryptlen;
139 dd->assoc_len = assoclen;
140 dd->authsize = authlen;
141
142 dd->out_sg = req->dst;
143 dd->orig_out = req->dst;
144
145 dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, assoclen);
146
147 flags = 0;
148 if (req->src == req->dst || dd->out_sg == sg_arr)
149 flags |= OMAP_CRYPTO_FORCE_COPY;
150
151 ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,
152 AES_BLOCK_SIZE, &dd->out_sgl,
153 flags,
154 FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
155 if (ret)
156 return ret;
157
158 dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);
159 dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);
160
161 return 0;
162 }
163
omap_aes_gcm_complete(struct crypto_async_request * req,int err)164 static void omap_aes_gcm_complete(struct crypto_async_request *req, int err)
165 {
166 struct omap_aes_gcm_result *res = req->data;
167
168 if (err == -EINPROGRESS)
169 return;
170
171 res->err = err;
172 complete(&res->completion);
173 }
174
do_encrypt_iv(struct aead_request * req,u32 * tag,u32 * iv)175 static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv)
176 {
177 struct scatterlist iv_sg, tag_sg;
178 struct skcipher_request *sk_req;
179 struct omap_aes_gcm_result result;
180 struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
181 int ret = 0;
182
183 sk_req = skcipher_request_alloc(ctx->ctr, GFP_KERNEL);
184 if (!sk_req) {
185 pr_err("skcipher: Failed to allocate request\n");
186 return -ENOMEM;
187 }
188
189 init_completion(&result.completion);
190
191 sg_init_one(&iv_sg, iv, AES_BLOCK_SIZE);
192 sg_init_one(&tag_sg, tag, AES_BLOCK_SIZE);
193 skcipher_request_set_callback(sk_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
194 omap_aes_gcm_complete, &result);
195 ret = crypto_skcipher_setkey(ctx->ctr, (u8 *)ctx->key, ctx->keylen);
196 skcipher_request_set_crypt(sk_req, &iv_sg, &tag_sg, AES_BLOCK_SIZE,
197 NULL);
198 ret = crypto_skcipher_encrypt(sk_req);
199 switch (ret) {
200 case 0:
201 break;
202 case -EINPROGRESS:
203 case -EBUSY:
204 ret = wait_for_completion_interruptible(&result.completion);
205 if (!ret) {
206 ret = result.err;
207 if (!ret) {
208 reinit_completion(&result.completion);
209 break;
210 }
211 }
212 /* fall through */
213 default:
214 pr_err("Encryption of IV failed for GCM mode\n");
215 break;
216 }
217
218 skcipher_request_free(sk_req);
219 return ret;
220 }
221
omap_aes_gcm_dma_out_callback(void * data)222 void omap_aes_gcm_dma_out_callback(void *data)
223 {
224 struct omap_aes_dev *dd = data;
225 struct omap_aes_reqctx *rctx;
226 int i, val;
227 u32 *auth_tag, tag[4];
228
229 if (!(dd->flags & FLAGS_ENCRYPT))
230 scatterwalk_map_and_copy(tag, dd->aead_req->src,
231 dd->total + dd->aead_req->assoclen,
232 dd->authsize, 0);
233
234 rctx = aead_request_ctx(dd->aead_req);
235 auth_tag = (u32 *)rctx->auth_tag;
236 for (i = 0; i < 4; i++) {
237 val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));
238 auth_tag[i] = val ^ auth_tag[i];
239 if (!(dd->flags & FLAGS_ENCRYPT))
240 auth_tag[i] = auth_tag[i] ^ tag[i];
241 }
242
243 omap_aes_gcm_done_task(dd);
244 }
245
omap_aes_gcm_handle_queue(struct omap_aes_dev * dd,struct aead_request * req)246 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
247 struct aead_request *req)
248 {
249 struct omap_aes_ctx *ctx;
250 struct aead_request *backlog;
251 struct omap_aes_reqctx *rctx;
252 unsigned long flags;
253 int err, ret = 0;
254
255 spin_lock_irqsave(&dd->lock, flags);
256 if (req)
257 ret = aead_enqueue_request(&dd->aead_queue, req);
258 if (dd->flags & FLAGS_BUSY) {
259 spin_unlock_irqrestore(&dd->lock, flags);
260 return ret;
261 }
262
263 backlog = aead_get_backlog(&dd->aead_queue);
264 req = aead_dequeue_request(&dd->aead_queue);
265 if (req)
266 dd->flags |= FLAGS_BUSY;
267 spin_unlock_irqrestore(&dd->lock, flags);
268
269 if (!req)
270 return ret;
271
272 if (backlog)
273 backlog->base.complete(&backlog->base, -EINPROGRESS);
274
275 ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
276 rctx = aead_request_ctx(req);
277
278 dd->ctx = ctx;
279 rctx->dd = dd;
280 dd->aead_req = req;
281
282 rctx->mode &= FLAGS_MODE_MASK;
283 dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
284
285 err = omap_aes_gcm_copy_buffers(dd, req);
286 if (err)
287 return err;
288
289 err = omap_aes_write_ctrl(dd);
290 if (!err)
291 err = omap_aes_crypt_dma_start(dd);
292
293 if (err) {
294 omap_aes_gcm_finish_req(dd, err);
295 omap_aes_gcm_handle_queue(dd, NULL);
296 }
297
298 return ret;
299 }
300
omap_aes_gcm_crypt(struct aead_request * req,unsigned long mode)301 static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode)
302 {
303 struct omap_aes_reqctx *rctx = aead_request_ctx(req);
304 struct crypto_aead *aead = crypto_aead_reqtfm(req);
305 unsigned int authlen = crypto_aead_authsize(aead);
306 struct omap_aes_dev *dd;
307 __be32 counter = cpu_to_be32(1);
308 int err, assoclen;
309
310 memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag));
311 memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4);
312
313 err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv);
314 if (err)
315 return err;
316
317 if (mode & FLAGS_RFC4106_GCM)
318 assoclen = req->assoclen - 8;
319 else
320 assoclen = req->assoclen;
321 if (assoclen + req->cryptlen == 0) {
322 scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen,
323 1);
324 return 0;
325 }
326
327 dd = omap_aes_find_dev(rctx);
328 if (!dd)
329 return -ENODEV;
330 rctx->mode = mode;
331
332 return omap_aes_gcm_handle_queue(dd, req);
333 }
334
omap_aes_gcm_encrypt(struct aead_request * req)335 int omap_aes_gcm_encrypt(struct aead_request *req)
336 {
337 struct omap_aes_reqctx *rctx = aead_request_ctx(req);
338
339 memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
340 return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM);
341 }
342
omap_aes_gcm_decrypt(struct aead_request * req)343 int omap_aes_gcm_decrypt(struct aead_request *req)
344 {
345 struct omap_aes_reqctx *rctx = aead_request_ctx(req);
346
347 memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
348 return omap_aes_gcm_crypt(req, FLAGS_GCM);
349 }
350
omap_aes_4106gcm_encrypt(struct aead_request * req)351 int omap_aes_4106gcm_encrypt(struct aead_request *req)
352 {
353 struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
354 struct omap_aes_reqctx *rctx = aead_request_ctx(req);
355
356 memcpy(rctx->iv, ctx->nonce, 4);
357 memcpy(rctx->iv + 4, req->iv, 8);
358 return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM |
359 FLAGS_RFC4106_GCM);
360 }
361
omap_aes_4106gcm_decrypt(struct aead_request * req)362 int omap_aes_4106gcm_decrypt(struct aead_request *req)
363 {
364 struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
365 struct omap_aes_reqctx *rctx = aead_request_ctx(req);
366
367 memcpy(rctx->iv, ctx->nonce, 4);
368 memcpy(rctx->iv + 4, req->iv, 8);
369 return omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM);
370 }
371
omap_aes_gcm_setkey(struct crypto_aead * tfm,const u8 * key,unsigned int keylen)372 int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
373 unsigned int keylen)
374 {
375 struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
376
377 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
378 keylen != AES_KEYSIZE_256)
379 return -EINVAL;
380
381 memcpy(ctx->key, key, keylen);
382 ctx->keylen = keylen;
383
384 return 0;
385 }
386
omap_aes_4106gcm_setkey(struct crypto_aead * tfm,const u8 * key,unsigned int keylen)387 int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key,
388 unsigned int keylen)
389 {
390 struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
391
392 if (keylen < 4)
393 return -EINVAL;
394
395 keylen -= 4;
396 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
397 keylen != AES_KEYSIZE_256)
398 return -EINVAL;
399
400 memcpy(ctx->key, key, keylen);
401 memcpy(ctx->nonce, key + keylen, 4);
402 ctx->keylen = keylen;
403
404 return 0;
405 }
406