Lines Matching +full:tightly +full:- +full:coupled
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Driver for ARTPEC-6 crypto block using the kernel asynchronous crypto api.
5 * Copyright (C) 2014-2017 Axis Communications AB
13 #include <linux/dma-mapping.h>
14 #include <linux/fault-inject.h>
185 #define MODULE_NAME "Artpec-6 CA"
197 /* The PDMA is a DMA-engine tightly coupled with a ciphering engine.
202 * a 4-byte metadata that is inserted at the beginning of each dma packet.
211 * +------+------+------+~~+-------+------+----
213 * +--+---+--+---+----+-+~~+-------+----+-+----
216 * __|__ +-------++-------++-------+ +----+
218 * +-----+ +-------++-------++-------+ +----+
268 void *pad_buffer; /* cache-aligned block padding buffer */
388 awalk->sg = sg; in artpec6_crypto_walk_init()
389 awalk->offset = 0; in artpec6_crypto_walk_init()
395 while (nbytes && awalk->sg) { in artpec6_crypto_walk_advance()
398 WARN_ON(awalk->offset > awalk->sg->length); in artpec6_crypto_walk_advance()
400 piece = min(nbytes, (size_t)awalk->sg->length - awalk->offset); in artpec6_crypto_walk_advance()
401 nbytes -= piece; in artpec6_crypto_walk_advance()
402 awalk->offset += piece; in artpec6_crypto_walk_advance()
403 if (awalk->offset == awalk->sg->length) { in artpec6_crypto_walk_advance()
404 awalk->sg = sg_next(awalk->sg); in artpec6_crypto_walk_advance()
405 awalk->offset = 0; in artpec6_crypto_walk_advance()
416 WARN_ON(awalk->sg->length == awalk->offset); in artpec6_crypto_walk_chunklen()
418 return awalk->sg->length - awalk->offset; in artpec6_crypto_walk_chunklen()
424 return sg_phys(awalk->sg) + awalk->offset; in artpec6_crypto_walk_chunk_phys()
430 struct artpec6_crypto_dma_descriptors *dma = common->dma; in artpec6_crypto_copy_bounce_buffers()
434 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { in artpec6_crypto_copy_bounce_buffers()
436 b, b->length, b->offset, b->buf); in artpec6_crypto_copy_bounce_buffers()
437 sg_pcopy_from_buffer(b->sg, in artpec6_crypto_copy_bounce_buffers()
439 b->buf, in artpec6_crypto_copy_bounce_buffers()
440 b->length, in artpec6_crypto_copy_bounce_buffers()
441 b->offset); in artpec6_crypto_copy_bounce_buffers()
443 list_del(&b->list); in artpec6_crypto_copy_bounce_buffers()
451 int fifo_count = ac->pending_count; in artpec6_crypto_busy()
459 int ret = -EBUSY; in artpec6_crypto_submit()
461 spin_lock_bh(&ac->queue_lock); in artpec6_crypto_submit()
464 list_add_tail(&req->list, &ac->pending); in artpec6_crypto_submit()
466 ret = -EINPROGRESS; in artpec6_crypto_submit()
467 } else if (req->req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) { in artpec6_crypto_submit()
468 list_add_tail(&req->list, &ac->queue); in artpec6_crypto_submit()
473 spin_unlock_bh(&ac->queue_lock); in artpec6_crypto_submit()
481 enum artpec6_crypto_variant variant = ac->variant; in artpec6_crypto_start_dma()
482 void __iomem *base = ac->base; in artpec6_crypto_start_dma()
483 struct artpec6_crypto_dma_descriptors *dma = common->dma; in artpec6_crypto_start_dma()
489 ind = FIELD_PREP(PDMA_IN_DESCRQ_PUSH_LEN, dma->in_cnt - 1) | in artpec6_crypto_start_dma()
490 FIELD_PREP(PDMA_IN_DESCRQ_PUSH_ADDR, dma->in_dma_addr >> 6); in artpec6_crypto_start_dma()
492 statd = FIELD_PREP(PDMA_IN_STATQ_PUSH_LEN, dma->in_cnt - 1) | in artpec6_crypto_start_dma()
493 FIELD_PREP(PDMA_IN_STATQ_PUSH_ADDR, dma->stat_dma_addr >> 6); in artpec6_crypto_start_dma()
495 outd = FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_LEN, dma->out_cnt - 1) | in artpec6_crypto_start_dma()
496 FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_ADDR, dma->out_dma_addr >> 6); in artpec6_crypto_start_dma()
511 ac->pending_count++; in artpec6_crypto_start_dma()
517 struct artpec6_crypto_dma_descriptors *dma = common->dma; in artpec6_crypto_init_dma_operation()
519 dma->out_cnt = 0; in artpec6_crypto_init_dma_operation()
520 dma->in_cnt = 0; in artpec6_crypto_init_dma_operation()
521 dma->map_count = 0; in artpec6_crypto_init_dma_operation()
522 INIT_LIST_HEAD(&dma->bounce_buffers); in artpec6_crypto_init_dma_operation()
534 /** artpec6_crypto_setup_out_descr_phys - Setup an out channel with a
541 * @return 0 on success or -ENOSPC if there are no more descriptors available
547 struct artpec6_crypto_dma_descriptors *dma = common->dma; in artpec6_crypto_setup_out_descr_phys()
550 if (dma->out_cnt >= PDMA_DESCR_COUNT || in artpec6_crypto_setup_out_descr_phys()
553 return -ENOSPC; in artpec6_crypto_setup_out_descr_phys()
556 d = &dma->out[dma->out_cnt++]; in artpec6_crypto_setup_out_descr_phys()
559 d->ctrl.short_descr = 0; in artpec6_crypto_setup_out_descr_phys()
560 d->ctrl.eop = eop; in artpec6_crypto_setup_out_descr_phys()
561 d->data.len = len; in artpec6_crypto_setup_out_descr_phys()
562 d->data.buf = addr; in artpec6_crypto_setup_out_descr_phys()
566 /** artpec6_crypto_setup_out_descr_short - Setup a short out descriptor
573 * -ENOSPC if no more descriptors are available
574 * -EINVAL if the data length exceeds 7 bytes
580 struct artpec6_crypto_dma_descriptors *dma = common->dma; in artpec6_crypto_setup_out_descr_short()
583 if (dma->out_cnt >= PDMA_DESCR_COUNT || in artpec6_crypto_setup_out_descr_short()
586 return -ENOSPC; in artpec6_crypto_setup_out_descr_short()
588 return -EINVAL; in artpec6_crypto_setup_out_descr_short()
590 d = &dma->out[dma->out_cnt++]; in artpec6_crypto_setup_out_descr_short()
593 d->ctrl.short_descr = 1; in artpec6_crypto_setup_out_descr_short()
594 d->ctrl.short_len = len; in artpec6_crypto_setup_out_descr_short()
595 d->ctrl.eop = eop; in artpec6_crypto_setup_out_descr_short()
596 memcpy(d->shrt.data, dst, len); in artpec6_crypto_setup_out_descr_short()
606 struct artpec6_crypto_dma_descriptors *dma = common->dma; in artpec6_crypto_dma_map_page()
613 if (dma->map_count >= ARRAY_SIZE(dma->maps)) in artpec6_crypto_dma_map_page()
614 return -ENOMEM; in artpec6_crypto_dma_map_page()
618 return -ENOMEM; in artpec6_crypto_dma_map_page()
620 map = &dma->maps[dma->map_count++]; in artpec6_crypto_dma_map_page()
621 map->size = size; in artpec6_crypto_dma_map_page()
622 map->dma_addr = dma_addr; in artpec6_crypto_dma_map_page()
623 map->dir = dir; in artpec6_crypto_dma_map_page()
646 struct artpec6_crypto_dma_descriptors *dma = common->dma; in artpec6_crypto_dma_map_descs()
649 ret = artpec6_crypto_dma_map_single(common, dma->in, in artpec6_crypto_dma_map_descs()
650 sizeof(dma->in[0]) * dma->in_cnt, in artpec6_crypto_dma_map_descs()
651 DMA_TO_DEVICE, &dma->in_dma_addr); in artpec6_crypto_dma_map_descs()
655 ret = artpec6_crypto_dma_map_single(common, dma->out, in artpec6_crypto_dma_map_descs()
656 sizeof(dma->out[0]) * dma->out_cnt, in artpec6_crypto_dma_map_descs()
657 DMA_TO_DEVICE, &dma->out_dma_addr); in artpec6_crypto_dma_map_descs()
662 dma->stat[dma->in_cnt - 1] = 0; in artpec6_crypto_dma_map_descs()
669 dma->stat, in artpec6_crypto_dma_map_descs()
670 sizeof(dma->stat[0]) * dma->in_cnt, in artpec6_crypto_dma_map_descs()
672 &dma->stat_dma_addr); in artpec6_crypto_dma_map_descs()
678 struct artpec6_crypto_dma_descriptors *dma = common->dma; in artpec6_crypto_dma_unmap_all()
682 for (i = 0; i < dma->map_count; i++) { in artpec6_crypto_dma_unmap_all()
683 struct artpec6_crypto_dma_map *map = &dma->maps[i]; in artpec6_crypto_dma_unmap_all()
685 dma_unmap_page(dev, map->dma_addr, map->size, map->dir); in artpec6_crypto_dma_unmap_all()
688 dma->map_count = 0; in artpec6_crypto_dma_unmap_all()
691 /** artpec6_crypto_setup_out_descr - Setup an out descriptor
726 /** artpec6_crypto_setup_in_descr_phys - Setup an in channel with a
739 struct artpec6_crypto_dma_descriptors *dma = common->dma; in artpec6_crypto_setup_in_descr_phys()
742 if (dma->in_cnt >= PDMA_DESCR_COUNT || in artpec6_crypto_setup_in_descr_phys()
745 return -ENOSPC; in artpec6_crypto_setup_in_descr_phys()
747 d = &dma->in[dma->in_cnt++]; in artpec6_crypto_setup_in_descr_phys()
750 d->ctrl.intr = intr; in artpec6_crypto_setup_in_descr_phys()
751 d->data.len = len; in artpec6_crypto_setup_in_descr_phys()
752 d->data.buf = addr; in artpec6_crypto_setup_in_descr_phys()
756 /** artpec6_crypto_setup_in_descr - Setup an in channel descriptor
792 bbuf->buf = PTR_ALIGN(base, ARTPEC_CACHE_LINE_MAX); in artpec6_crypto_alloc_bounce()
802 bbuf = artpec6_crypto_alloc_bounce(common->gfp_flags); in setup_bounce_buffer_in()
804 return -ENOMEM; in setup_bounce_buffer_in()
806 bbuf->length = size; in setup_bounce_buffer_in()
807 bbuf->sg = walk->sg; in setup_bounce_buffer_in()
808 bbuf->offset = walk->offset; in setup_bounce_buffer_in()
810 ret = artpec6_crypto_setup_in_descr(common, bbuf->buf, size, false); in setup_bounce_buffer_in()
816 pr_debug("BOUNCE %zu offset %zu\n", size, walk->offset); in setup_bounce_buffer_in()
817 list_add_tail(&bbuf->list, &common->dma->bounce_buffers); in setup_bounce_buffer_in()
830 while (walk->sg && count) { in artpec6_crypto_setup_sg_descrs_in()
835 * size we need bounce buffers. The DMA-API requires that the in artpec6_crypto_setup_sg_descrs_in()
841 ALIGN(addr, ARTPEC_CACHE_LINE_MAX) - in artpec6_crypto_setup_sg_descrs_in()
844 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); in artpec6_crypto_setup_sg_descrs_in()
847 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); in artpec6_crypto_setup_sg_descrs_in()
852 chunk = chunk & ~(ARTPEC_CACHE_LINE_MAX-1); in artpec6_crypto_setup_sg_descrs_in()
857 sg_page(walk->sg), in artpec6_crypto_setup_sg_descrs_in()
858 walk->sg->offset + in artpec6_crypto_setup_sg_descrs_in()
859 walk->offset, in artpec6_crypto_setup_sg_descrs_in()
874 count = count - chunk; in artpec6_crypto_setup_sg_descrs_in()
881 return count ? -EINVAL : 0; in artpec6_crypto_setup_sg_descrs_in()
893 while (walk->sg && count) { in artpec6_crypto_setup_sg_descrs_out()
897 pr_debug("OUT-CHUNK %pad:%zu\n", &addr, chunk); in artpec6_crypto_setup_sg_descrs_out()
902 chunk = min_t(size_t, chunk, (4-(addr&3))); in artpec6_crypto_setup_sg_descrs_out()
904 sg_pcopy_to_buffer(walk->sg, 1, buf, chunk, in artpec6_crypto_setup_sg_descrs_out()
905 walk->offset); in artpec6_crypto_setup_sg_descrs_out()
914 sg_page(walk->sg), in artpec6_crypto_setup_sg_descrs_out()
915 walk->sg->offset + in artpec6_crypto_setup_sg_descrs_out()
916 walk->offset, in artpec6_crypto_setup_sg_descrs_out()
931 count = count - chunk; in artpec6_crypto_setup_sg_descrs_out()
938 return count ? -EINVAL : 0; in artpec6_crypto_setup_sg_descrs_out()
942 /** artpec6_crypto_terminate_out_descrs - Set the EOP on the last out descriptor
944 * If the out descriptor list is non-empty, then the eop flag on the
948 * -EINVAL if the out descriptor is empty or has overflown
953 struct artpec6_crypto_dma_descriptors *dma = common->dma; in artpec6_crypto_terminate_out_descrs()
956 if (!dma->out_cnt || dma->out_cnt > PDMA_DESCR_COUNT) { in artpec6_crypto_terminate_out_descrs()
958 MODULE_NAME, dma->out_cnt ? "empty" : "full"); in artpec6_crypto_terminate_out_descrs()
959 return -EINVAL; in artpec6_crypto_terminate_out_descrs()
963 d = &dma->out[dma->out_cnt-1]; in artpec6_crypto_terminate_out_descrs()
964 d->ctrl.eop = 1; in artpec6_crypto_terminate_out_descrs()
969 /** artpec6_crypto_terminate_in_descrs - Set the interrupt flag on the last
977 struct artpec6_crypto_dma_descriptors *dma = common->dma; in artpec6_crypto_terminate_in_descrs()
980 if (!dma->in_cnt || dma->in_cnt > PDMA_DESCR_COUNT) { in artpec6_crypto_terminate_in_descrs()
982 MODULE_NAME, dma->in_cnt ? "empty" : "full"); in artpec6_crypto_terminate_in_descrs()
983 return -EINVAL; in artpec6_crypto_terminate_in_descrs()
986 d = &dma->in[dma->in_cnt-1]; in artpec6_crypto_terminate_in_descrs()
987 d->ctrl.intr = 1; in artpec6_crypto_terminate_in_descrs()
991 /** create_hash_pad - Create a Secure Hash conformant pad
1021 target -= 1; in create_hash_pad()
1022 diff = dgstlen & (mod - 1); in create_hash_pad()
1023 pad_bytes = diff > target ? target + mod - diff : target - diff; in create_hash_pad()
1046 flags = (parent->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? in artpec6_crypto_common_init()
1049 common->gfp_flags = flags; in artpec6_crypto_common_init()
1050 common->dma = kmem_cache_alloc(ac->dma_cache, flags); in artpec6_crypto_common_init()
1051 if (!common->dma) in artpec6_crypto_common_init()
1052 return -ENOMEM; in artpec6_crypto_common_init()
1054 common->req = parent; in artpec6_crypto_common_init()
1055 common->complete = complete; in artpec6_crypto_common_init()
1065 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { in artpec6_crypto_bounce_destroy()
1076 artpec6_crypto_bounce_destroy(common->dma); in artpec6_crypto_common_destroy()
1077 kmem_cache_free(ac->dma_cache, common->dma); in artpec6_crypto_common_destroy()
1078 common->dma = NULL; in artpec6_crypto_common_destroy()
1095 switch (ctx->crypto_type) { in artpec6_crypto_encrypt()
1099 req_ctx->decrypt = 0; in artpec6_crypto_encrypt()
1105 switch (ctx->crypto_type) { in artpec6_crypto_encrypt()
1114 ret = artpec6_crypto_common_init(&req_ctx->common, in artpec6_crypto_encrypt()
1115 &req->base, in artpec6_crypto_encrypt()
1117 req->dst, req->cryptlen); in artpec6_crypto_encrypt()
1123 artpec6_crypto_common_destroy(&req_ctx->common); in artpec6_crypto_encrypt()
1127 return artpec6_crypto_submit(&req_ctx->common); in artpec6_crypto_encrypt()
1140 switch (ctx->crypto_type) { in artpec6_crypto_decrypt()
1144 req_ctx->decrypt = 1; in artpec6_crypto_decrypt()
1151 switch (ctx->crypto_type) { in artpec6_crypto_decrypt()
1160 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, in artpec6_crypto_decrypt()
1162 req->dst, req->cryptlen); in artpec6_crypto_decrypt()
1168 artpec6_crypto_common_destroy(&req_ctx->common); in artpec6_crypto_decrypt()
1172 return artpec6_crypto_submit(&req_ctx->common); in artpec6_crypto_decrypt()
1182 (req->iv + iv_len - 4)); in artpec6_crypto_ctr_crypt()
1183 unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) / in artpec6_crypto_ctr_crypt()
1187 * The hardware uses only the last 32-bits as the counter while the in artpec6_crypto_ctr_crypt()
1198 ret = crypto_sync_skcipher_setkey(ctx->fallback, ctx->aes_key, in artpec6_crypto_ctr_crypt()
1199 ctx->key_length); in artpec6_crypto_ctr_crypt()
1204 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); in artpec6_crypto_ctr_crypt()
1206 skcipher_request_set_sync_tfm(subreq, ctx->fallback); in artpec6_crypto_ctr_crypt()
1207 skcipher_request_set_callback(subreq, req->base.flags, in artpec6_crypto_ctr_crypt()
1209 skcipher_request_set_crypt(subreq, req->src, req->dst, in artpec6_crypto_ctr_crypt()
1210 req->cryptlen, req->iv); in artpec6_crypto_ctr_crypt()
1250 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(&tfm->base); in artpec6_crypto_aead_set_key()
1253 return -EINVAL; in artpec6_crypto_aead_set_key()
1255 ctx->key_length = len; in artpec6_crypto_aead_set_key()
1257 memcpy(ctx->aes_key, key, len); in artpec6_crypto_aead_set_key()
1266 req_ctx->decrypt = false; in artpec6_crypto_aead_encrypt()
1267 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, in artpec6_crypto_aead_encrypt()
1275 artpec6_crypto_common_destroy(&req_ctx->common); in artpec6_crypto_aead_encrypt()
1279 return artpec6_crypto_submit(&req_ctx->common); in artpec6_crypto_aead_encrypt()
1287 req_ctx->decrypt = true; in artpec6_crypto_aead_decrypt()
1288 if (req->cryptlen < AES_BLOCK_SIZE) in artpec6_crypto_aead_decrypt()
1289 return -EINVAL; in artpec6_crypto_aead_decrypt()
1291 ret = artpec6_crypto_common_init(&req_ctx->common, in artpec6_crypto_aead_decrypt()
1292 &req->base, in artpec6_crypto_aead_decrypt()
1300 artpec6_crypto_common_destroy(&req_ctx->common); in artpec6_crypto_aead_decrypt()
1304 return artpec6_crypto_submit(&req_ctx->common); in artpec6_crypto_aead_decrypt()
1309 struct artpec6_hashalg_context *ctx = crypto_tfm_ctx(areq->base.tfm); in artpec6_crypto_prepare_hash()
1315 struct artpec6_crypto_req_common *common = &req_ctx->common; in artpec6_crypto_prepare_hash()
1317 enum artpec6_crypto_variant variant = ac->variant; in artpec6_crypto_prepare_hash()
1326 if (req_ctx->hash_flags & HASH_FLAG_HMAC) { in artpec6_crypto_prepare_hash()
1328 req_ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, in artpec6_crypto_prepare_hash()
1331 req_ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, in artpec6_crypto_prepare_hash()
1336 memcpy(req_ctx->key_buffer, ctx->hmac_key, in artpec6_crypto_prepare_hash()
1337 ctx->hmac_key_length); in artpec6_crypto_prepare_hash()
1338 memset(req_ctx->key_buffer + ctx->hmac_key_length, 0, in artpec6_crypto_prepare_hash()
1339 blocksize - ctx->hmac_key_length); in artpec6_crypto_prepare_hash()
1342 (void *)&req_ctx->key_md, in artpec6_crypto_prepare_hash()
1343 sizeof(req_ctx->key_md), false, false); in artpec6_crypto_prepare_hash()
1348 req_ctx->key_buffer, blocksize, in artpec6_crypto_prepare_hash()
1354 if (!(req_ctx->hash_flags & HASH_FLAG_INIT_CTX)) { in artpec6_crypto_prepare_hash()
1363 req_ctx->hash_md &= ~A6_CRY_MD_HASH_SEL_CTX; in artpec6_crypto_prepare_hash()
1364 req_ctx->hash_md |= FIELD_PREP(A6_CRY_MD_HASH_SEL_CTX, sel_ctx); in artpec6_crypto_prepare_hash()
1367 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) in artpec6_crypto_prepare_hash()
1368 req_ctx->hash_md |= A6_CRY_MD_HASH_HMAC_FIN; in artpec6_crypto_prepare_hash()
1370 req_ctx->hash_md &= ~A7_CRY_MD_HASH_SEL_CTX; in artpec6_crypto_prepare_hash()
1371 req_ctx->hash_md |= FIELD_PREP(A7_CRY_MD_HASH_SEL_CTX, sel_ctx); in artpec6_crypto_prepare_hash()
1374 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) in artpec6_crypto_prepare_hash()
1375 req_ctx->hash_md |= A7_CRY_MD_HASH_HMAC_FIN; in artpec6_crypto_prepare_hash()
1380 (void *)&req_ctx->hash_md, in artpec6_crypto_prepare_hash()
1381 sizeof(req_ctx->hash_md), false, false); in artpec6_crypto_prepare_hash()
1385 error = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); in artpec6_crypto_prepare_hash()
1391 req_ctx->digeststate, in artpec6_crypto_prepare_hash()
1398 if (req_ctx->hash_flags & HASH_FLAG_UPDATE) { in artpec6_crypto_prepare_hash()
1400 size_t total_bytes = areq->nbytes + req_ctx->partial_bytes; in artpec6_crypto_prepare_hash()
1405 if (req_ctx->partial_bytes && ready_bytes) { in artpec6_crypto_prepare_hash()
1410 memcpy(req_ctx->partial_buffer_out, in artpec6_crypto_prepare_hash()
1411 req_ctx->partial_buffer, in artpec6_crypto_prepare_hash()
1412 req_ctx->partial_bytes); in artpec6_crypto_prepare_hash()
1415 req_ctx->partial_buffer_out, in artpec6_crypto_prepare_hash()
1416 req_ctx->partial_bytes, in artpec6_crypto_prepare_hash()
1422 done_bytes += req_ctx->partial_bytes; in artpec6_crypto_prepare_hash()
1423 req_ctx->partial_bytes = 0; in artpec6_crypto_prepare_hash()
1426 artpec6_crypto_walk_init(&walk, areq->src); in artpec6_crypto_prepare_hash()
1429 ready_bytes - in artpec6_crypto_prepare_hash()
1435 size_t sg_skip = ready_bytes - done_bytes; in artpec6_crypto_prepare_hash()
1436 size_t sg_rem = areq->nbytes - sg_skip; in artpec6_crypto_prepare_hash()
1438 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), in artpec6_crypto_prepare_hash()
1439 req_ctx->partial_buffer + in artpec6_crypto_prepare_hash()
1440 req_ctx->partial_bytes, in artpec6_crypto_prepare_hash()
1443 req_ctx->partial_bytes += sg_rem; in artpec6_crypto_prepare_hash()
1446 req_ctx->digcnt += ready_bytes; in artpec6_crypto_prepare_hash()
1447 req_ctx->hash_flags &= ~(HASH_FLAG_UPDATE); in artpec6_crypto_prepare_hash()
1451 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) { in artpec6_crypto_prepare_hash()
1457 oper = FIELD_GET(A6_CRY_MD_OPER, req_ctx->hash_md); in artpec6_crypto_prepare_hash()
1459 oper = FIELD_GET(A7_CRY_MD_OPER, req_ctx->hash_md); in artpec6_crypto_prepare_hash()
1462 if (req_ctx->partial_bytes) { in artpec6_crypto_prepare_hash()
1463 memcpy(req_ctx->partial_buffer_out, in artpec6_crypto_prepare_hash()
1464 req_ctx->partial_buffer, in artpec6_crypto_prepare_hash()
1465 req_ctx->partial_bytes); in artpec6_crypto_prepare_hash()
1467 req_ctx->partial_buffer_out, in artpec6_crypto_prepare_hash()
1468 req_ctx->partial_bytes, in artpec6_crypto_prepare_hash()
1473 req_ctx->digcnt += req_ctx->partial_bytes; in artpec6_crypto_prepare_hash()
1474 req_ctx->partial_bytes = 0; in artpec6_crypto_prepare_hash()
1477 if (req_ctx->hash_flags & HASH_FLAG_HMAC) in artpec6_crypto_prepare_hash()
1478 digest_bits = 8 * (req_ctx->digcnt + blocksize); in artpec6_crypto_prepare_hash()
1480 digest_bits = 8 * req_ctx->digcnt; in artpec6_crypto_prepare_hash()
1483 hash_pad_len = create_hash_pad(oper, req_ctx->pad_buffer, in artpec6_crypto_prepare_hash()
1484 req_ctx->digcnt, digest_bits); in artpec6_crypto_prepare_hash()
1486 req_ctx->pad_buffer, in artpec6_crypto_prepare_hash()
1489 req_ctx->digcnt = 0; in artpec6_crypto_prepare_hash()
1495 error = artpec6_crypto_setup_in_descr(common, areq->result, in artpec6_crypto_prepare_hash()
1507 req_ctx->digeststate, in artpec6_crypto_prepare_hash()
1514 req_ctx->hash_flags &= ~(HASH_FLAG_INIT_CTX | HASH_FLAG_UPDATE | in artpec6_crypto_prepare_hash()
1537 tfm->reqsize = sizeof(struct artpec6_crypto_request_context); in artpec6_crypto_aes_ecb_init()
1538 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_ECB; in artpec6_crypto_aes_ecb_init()
1547 ctx->fallback = in artpec6_crypto_aes_ctr_init()
1548 crypto_alloc_sync_skcipher(crypto_tfm_alg_name(&tfm->base), in artpec6_crypto_aes_ctr_init()
1550 if (IS_ERR(ctx->fallback)) in artpec6_crypto_aes_ctr_init()
1551 return PTR_ERR(ctx->fallback); in artpec6_crypto_aes_ctr_init()
1553 tfm->reqsize = sizeof(struct artpec6_crypto_request_context); in artpec6_crypto_aes_ctr_init()
1554 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CTR; in artpec6_crypto_aes_ctr_init()
1563 tfm->reqsize = sizeof(struct artpec6_crypto_request_context); in artpec6_crypto_aes_cbc_init()
1564 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CBC; in artpec6_crypto_aes_cbc_init()
1573 tfm->reqsize = sizeof(struct artpec6_crypto_request_context); in artpec6_crypto_aes_xts_init()
1574 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_XTS; in artpec6_crypto_aes_xts_init()
1590 crypto_free_sync_skcipher(ctx->fallback); in artpec6_crypto_aes_ctr_exit()
1607 return -EINVAL; in artpec6_crypto_cipher_set_key()
1610 memcpy(ctx->aes_key, key, keylen); in artpec6_crypto_cipher_set_key()
1611 ctx->key_length = keylen; in artpec6_crypto_cipher_set_key()
1623 ret = xts_check_key(&cipher->base, key, keylen); in artpec6_crypto_xts_set_key()
1633 return -EINVAL; in artpec6_crypto_xts_set_key()
1636 memcpy(ctx->aes_key, key, keylen); in artpec6_crypto_xts_set_key()
1637 ctx->key_length = keylen; in artpec6_crypto_xts_set_key()
1641 /** artpec6_crypto_process_crypto - Prepare an async block cipher crypto request
1650 * The required padding is added for AES-CTR using a statically defined
1655 * OUT: [KEY_MD][KEY][EOP]<CIPHER_MD>[IV]<data_0>...[data_n][AES-CTR_pad]<eop>
1668 enum artpec6_crypto_variant variant = ac->variant; in artpec6_crypto_prepare_crypto()
1676 common = &req_ctx->common; in artpec6_crypto_prepare_crypto()
1681 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, a6_regk_crypto_dlkey); in artpec6_crypto_prepare_crypto()
1683 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, a7_regk_crypto_dlkey); in artpec6_crypto_prepare_crypto()
1685 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, in artpec6_crypto_prepare_crypto()
1686 sizeof(ctx->key_md), false, false); in artpec6_crypto_prepare_crypto()
1690 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, in artpec6_crypto_prepare_crypto()
1691 ctx->key_length, true, false); in artpec6_crypto_prepare_crypto()
1695 req_ctx->cipher_md = 0; in artpec6_crypto_prepare_crypto()
1697 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) in artpec6_crypto_prepare_crypto()
1698 cipher_klen = ctx->key_length/2; in artpec6_crypto_prepare_crypto()
1700 cipher_klen = ctx->key_length; in artpec6_crypto_prepare_crypto()
1715 MODULE_NAME, ctx->key_length); in artpec6_crypto_prepare_crypto()
1716 return -EINVAL; in artpec6_crypto_prepare_crypto()
1719 switch (ctx->crypto_type) { in artpec6_crypto_prepare_crypto()
1722 cipher_decr = req_ctx->decrypt; in artpec6_crypto_prepare_crypto()
1727 cipher_decr = req_ctx->decrypt; in artpec6_crypto_prepare_crypto()
1737 cipher_decr = req_ctx->decrypt; in artpec6_crypto_prepare_crypto()
1740 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DSEQ; in artpec6_crypto_prepare_crypto()
1742 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DSEQ; in artpec6_crypto_prepare_crypto()
1747 MODULE_NAME, ctx->crypto_type); in artpec6_crypto_prepare_crypto()
1748 return -EINVAL; in artpec6_crypto_prepare_crypto()
1752 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, oper); in artpec6_crypto_prepare_crypto()
1753 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, in artpec6_crypto_prepare_crypto()
1756 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; in artpec6_crypto_prepare_crypto()
1758 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, oper); in artpec6_crypto_prepare_crypto()
1759 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, in artpec6_crypto_prepare_crypto()
1762 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; in artpec6_crypto_prepare_crypto()
1766 &req_ctx->cipher_md, in artpec6_crypto_prepare_crypto()
1767 sizeof(req_ctx->cipher_md), in artpec6_crypto_prepare_crypto()
1772 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); in artpec6_crypto_prepare_crypto()
1777 ret = artpec6_crypto_setup_out_descr(common, areq->iv, iv_len, in artpec6_crypto_prepare_crypto()
1783 artpec6_crypto_walk_init(&walk, areq->src); in artpec6_crypto_prepare_crypto()
1784 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, areq->cryptlen); in artpec6_crypto_prepare_crypto()
1789 artpec6_crypto_walk_init(&walk, areq->dst); in artpec6_crypto_prepare_crypto()
1790 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, areq->cryptlen); in artpec6_crypto_prepare_crypto()
1794 /* CTR-mode padding required by the HW. */ in artpec6_crypto_prepare_crypto()
1795 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_CTR || in artpec6_crypto_prepare_crypto()
1796 ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) { in artpec6_crypto_prepare_crypto()
1797 size_t pad = ALIGN(areq->cryptlen, AES_BLOCK_SIZE) - in artpec6_crypto_prepare_crypto()
1798 areq->cryptlen; in artpec6_crypto_prepare_crypto()
1802 ac->pad_buffer, in artpec6_crypto_prepare_crypto()
1808 ac->pad_buffer, pad, in artpec6_crypto_prepare_crypto()
1831 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(areq->base.tfm); in artpec6_crypto_prepare_aead()
1834 struct artpec6_crypto_req_common *common = &req_ctx->common; in artpec6_crypto_prepare_aead()
1836 enum artpec6_crypto_variant variant = ac->variant; in artpec6_crypto_prepare_aead()
1843 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, in artpec6_crypto_prepare_aead()
1846 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, in artpec6_crypto_prepare_aead()
1849 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, in artpec6_crypto_prepare_aead()
1850 sizeof(ctx->key_md), false, false); in artpec6_crypto_prepare_aead()
1854 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, in artpec6_crypto_prepare_aead()
1855 ctx->key_length, true, false); in artpec6_crypto_prepare_aead()
1859 req_ctx->cipher_md = 0; in artpec6_crypto_prepare_aead()
1861 switch (ctx->key_length) { in artpec6_crypto_prepare_aead()
1872 return -EINVAL; in artpec6_crypto_prepare_aead()
1876 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, in artpec6_crypto_prepare_aead()
1878 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, in artpec6_crypto_prepare_aead()
1880 if (req_ctx->decrypt) in artpec6_crypto_prepare_aead()
1881 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; in artpec6_crypto_prepare_aead()
1883 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, in artpec6_crypto_prepare_aead()
1885 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, in artpec6_crypto_prepare_aead()
1887 if (req_ctx->decrypt) in artpec6_crypto_prepare_aead()
1888 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; in artpec6_crypto_prepare_aead()
1892 (void *) &req_ctx->cipher_md, in artpec6_crypto_prepare_aead()
1893 sizeof(req_ctx->cipher_md), false, in artpec6_crypto_prepare_aead()
1898 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); in artpec6_crypto_prepare_aead()
1903 input_length = areq->cryptlen; in artpec6_crypto_prepare_aead()
1904 if (req_ctx->decrypt) in artpec6_crypto_prepare_aead()
1905 input_length -= crypto_aead_authsize(cipher); in artpec6_crypto_prepare_aead()
1908 req_ctx->hw_ctx.aad_length_bits = in artpec6_crypto_prepare_aead()
1909 __cpu_to_be64(8*areq->assoclen); in artpec6_crypto_prepare_aead()
1911 req_ctx->hw_ctx.text_length_bits = in artpec6_crypto_prepare_aead()
1914 memcpy(req_ctx->hw_ctx.J0, areq->iv, crypto_aead_ivsize(cipher)); in artpec6_crypto_prepare_aead()
1916 memcpy(req_ctx->hw_ctx.J0 + GCM_AES_IV_SIZE, "\x00\x00\x00\x01", 4); in artpec6_crypto_prepare_aead()
1918 ret = artpec6_crypto_setup_out_descr(common, &req_ctx->hw_ctx, in artpec6_crypto_prepare_aead()
1926 artpec6_crypto_walk_init(&walk, areq->src); in artpec6_crypto_prepare_aead()
1929 count = areq->assoclen; in artpec6_crypto_prepare_aead()
1934 if (!IS_ALIGNED(areq->assoclen, 16)) { in artpec6_crypto_prepare_aead()
1935 size_t assoc_pad = 16 - (areq->assoclen % 16); in artpec6_crypto_prepare_aead()
1938 ac->zero_buffer, in artpec6_crypto_prepare_aead()
1952 size_t crypto_pad = 16 - (input_length % 16); in artpec6_crypto_prepare_aead()
1955 ac->zero_buffer, in artpec6_crypto_prepare_aead()
1967 size_t output_len = areq->cryptlen; in artpec6_crypto_prepare_aead()
1969 if (req_ctx->decrypt) in artpec6_crypto_prepare_aead()
1970 output_len -= crypto_aead_authsize(cipher); in artpec6_crypto_prepare_aead()
1972 artpec6_crypto_walk_init(&walk, areq->dst); in artpec6_crypto_prepare_aead()
1975 count = artpec6_crypto_walk_advance(&walk, areq->assoclen); in artpec6_crypto_prepare_aead()
1977 return -EINVAL; in artpec6_crypto_prepare_aead()
1986 size_t crypto_pad = 16 - (output_len % 16); in artpec6_crypto_prepare_aead()
1989 ac->pad_buffer, in artpec6_crypto_prepare_aead()
2000 if (req_ctx->decrypt) { in artpec6_crypto_prepare_aead()
2002 req_ctx->decryption_tag, AES_BLOCK_SIZE, false); in artpec6_crypto_prepare_aead()
2018 count = AES_BLOCK_SIZE - authsize; in artpec6_crypto_prepare_aead()
2020 ac->pad_buffer, in artpec6_crypto_prepare_aead()
2045 while (!list_empty(&ac->queue) && !artpec6_crypto_busy()) { in artpec6_crypto_process_queue()
2046 req = list_first_entry(&ac->queue, in artpec6_crypto_process_queue()
2049 list_move_tail(&req->list, &ac->pending); in artpec6_crypto_process_queue()
2052 list_add_tail(&req->complete_in_progress, completions); in artpec6_crypto_process_queue()
2062 if (ac->pending_count) in artpec6_crypto_process_queue()
2063 mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100)); in artpec6_crypto_process_queue()
2065 del_timer(&ac->timer); in artpec6_crypto_process_queue()
2074 tasklet_schedule(&ac->task); in artpec6_crypto_timeout()
2088 if (list_empty(&ac->pending)) { in artpec6_crypto_task()
2093 spin_lock_bh(&ac->queue_lock); in artpec6_crypto_task()
2095 list_for_each_entry_safe(req, n, &ac->pending, list) { in artpec6_crypto_task()
2096 struct artpec6_crypto_dma_descriptors *dma = req->dma; in artpec6_crypto_task()
2100 stataddr = dma->stat_dma_addr + 4 * (req->dma->in_cnt - 1); in artpec6_crypto_task()
2106 stat = req->dma->stat[req->dma->in_cnt-1]; in artpec6_crypto_task()
2108 /* A non-zero final status descriptor indicates in artpec6_crypto_task()
2123 list_move_tail(&req->list, &complete_done); in artpec6_crypto_task()
2125 ac->pending_count--; in artpec6_crypto_task()
2130 spin_unlock_bh(&ac->queue_lock); in artpec6_crypto_task()
2140 req->complete(req->req); in artpec6_crypto_task()
2145 req->req->complete(req->req, -EINPROGRESS); in artpec6_crypto_task()
2151 req->complete(req, 0); in artpec6_crypto_complete_crypto()
2160 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->src, in artpec6_crypto_complete_cbc_decrypt()
2161 cipher_req->cryptlen - AES_BLOCK_SIZE, in artpec6_crypto_complete_cbc_decrypt()
2163 req->complete(req, 0); in artpec6_crypto_complete_cbc_decrypt()
2172 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->dst, in artpec6_crypto_complete_cbc_encrypt()
2173 cipher_req->cryptlen - AES_BLOCK_SIZE, in artpec6_crypto_complete_cbc_encrypt()
2175 req->complete(req, 0); in artpec6_crypto_complete_cbc_encrypt()
2188 if (req_ctx->decrypt) { in artpec6_crypto_complete_aead()
2192 sg_pcopy_to_buffer(areq->src, in artpec6_crypto_complete_aead()
2193 sg_nents(areq->src), in artpec6_crypto_complete_aead()
2196 areq->assoclen + areq->cryptlen - in artpec6_crypto_complete_aead()
2199 if (crypto_memneq(req_ctx->decryption_tag, in artpec6_crypto_complete_aead()
2206 req_ctx->decryption_tag, in artpec6_crypto_complete_aead()
2209 result = -EBADMSG; in artpec6_crypto_complete_aead()
2213 req->complete(req, result); in artpec6_crypto_complete_aead()
2218 req->complete(req, 0); in artpec6_crypto_complete_hash()
2222 /*------------------- Hash functions -----------------------------------------*/
2227 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(&tfm->base); in artpec6_crypto_hash_set_key()
2234 return -EINVAL; in artpec6_crypto_hash_set_key()
2237 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); in artpec6_crypto_hash_set_key()
2242 tfm_ctx->hmac_key_length = blocksize; in artpec6_crypto_hash_set_key()
2244 ret = crypto_shash_tfm_digest(tfm_ctx->child_hash, key, keylen, in artpec6_crypto_hash_set_key()
2245 tfm_ctx->hmac_key); in artpec6_crypto_hash_set_key()
2249 memcpy(tfm_ctx->hmac_key, key, keylen); in artpec6_crypto_hash_set_key()
2250 tfm_ctx->hmac_key_length = keylen; in artpec6_crypto_hash_set_key()
2260 enum artpec6_crypto_variant variant = ac->variant; in artpec6_crypto_init_hash()
2266 req_ctx->hash_flags = HASH_FLAG_INIT_CTX; in artpec6_crypto_init_hash()
2268 req_ctx->hash_flags |= (HASH_FLAG_HMAC | HASH_FLAG_UPDATE_KEY); in artpec6_crypto_init_hash()
2279 return -EINVAL; in artpec6_crypto_init_hash()
2283 req_ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, oper); in artpec6_crypto_init_hash()
2285 req_ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, oper); in artpec6_crypto_init_hash()
2295 if (!req_ctx->common.dma) { in artpec6_crypto_prepare_submit_hash()
2296 ret = artpec6_crypto_common_init(&req_ctx->common, in artpec6_crypto_prepare_submit_hash()
2297 &req->base, in artpec6_crypto_prepare_submit_hash()
2308 ret = artpec6_crypto_submit(&req_ctx->common); in artpec6_crypto_prepare_submit_hash()
2316 artpec6_crypto_common_destroy(&req_ctx->common); in artpec6_crypto_prepare_submit_hash()
2327 req_ctx->hash_flags |= HASH_FLAG_FINALIZE; in artpec6_crypto_hash_final()
2336 req_ctx->hash_flags |= HASH_FLAG_UPDATE; in artpec6_crypto_hash_update()
2352 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; in artpec6_crypto_sha1_digest()
2367 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; in artpec6_crypto_sha256_digest()
2382 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; in artpec6_crypto_hmac_sha256_digest()
2405 tfm_ctx->child_hash = child; in artpec6_crypto_ahash_init_common()
2425 if (tfm_ctx->child_hash) in artpec6_crypto_ahash_exit()
2426 crypto_free_shash(tfm_ctx->child_hash); in artpec6_crypto_ahash_exit()
2428 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); in artpec6_crypto_ahash_exit()
2429 tfm_ctx->hmac_key_length = 0; in artpec6_crypto_ahash_exit()
2437 enum artpec6_crypto_variant variant = ac->variant; in artpec6_crypto_hash_export()
2439 BUILD_BUG_ON(sizeof(state->partial_buffer) != in artpec6_crypto_hash_export()
2440 sizeof(ctx->partial_buffer)); in artpec6_crypto_hash_export()
2441 BUILD_BUG_ON(sizeof(state->digeststate) != sizeof(ctx->digeststate)); in artpec6_crypto_hash_export()
2443 state->digcnt = ctx->digcnt; in artpec6_crypto_hash_export()
2444 state->partial_bytes = ctx->partial_bytes; in artpec6_crypto_hash_export()
2445 state->hash_flags = ctx->hash_flags; in artpec6_crypto_hash_export()
2448 state->oper = FIELD_GET(A6_CRY_MD_OPER, ctx->hash_md); in artpec6_crypto_hash_export()
2450 state->oper = FIELD_GET(A7_CRY_MD_OPER, ctx->hash_md); in artpec6_crypto_hash_export()
2452 memcpy(state->partial_buffer, ctx->partial_buffer, in artpec6_crypto_hash_export()
2453 sizeof(state->partial_buffer)); in artpec6_crypto_hash_export()
2454 memcpy(state->digeststate, ctx->digeststate, in artpec6_crypto_hash_export()
2455 sizeof(state->digeststate)); in artpec6_crypto_hash_export()
2465 enum artpec6_crypto_variant variant = ac->variant; in artpec6_crypto_hash_import()
2469 ctx->digcnt = state->digcnt; in artpec6_crypto_hash_import()
2470 ctx->partial_bytes = state->partial_bytes; in artpec6_crypto_hash_import()
2471 ctx->hash_flags = state->hash_flags; in artpec6_crypto_hash_import()
2474 ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, state->oper); in artpec6_crypto_hash_import()
2476 ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, state->oper); in artpec6_crypto_hash_import()
2478 memcpy(ctx->partial_buffer, state->partial_buffer, in artpec6_crypto_hash_import()
2479 sizeof(state->partial_buffer)); in artpec6_crypto_hash_import()
2480 memcpy(ctx->digeststate, state->digeststate, in artpec6_crypto_hash_import()
2481 sizeof(state->digeststate)); in artpec6_crypto_hash_import()
2488 enum artpec6_crypto_variant variant = ac->variant; in init_crypto_hw()
2489 void __iomem *base = ac->base; in init_crypto_hw()
2548 enum artpec6_crypto_variant variant = ac->variant; in artpec6_crypto_disable_hw()
2549 void __iomem *base = ac->base; in artpec6_crypto_disable_hw()
2568 enum artpec6_crypto_variant variant = ac->variant; in artpec6_crypto_irq()
2569 void __iomem *base = ac->base; in artpec6_crypto_irq()
2594 * we request a status flush command to write the per-job in artpec6_crypto_irq()
2610 tasklet_schedule(&ac->task); in artpec6_crypto_irq()
2615 /*------------------- Algorithm definitions ----------------------------------*/
2619 /* SHA-1 */
2631 .cra_driver_name = "artpec-sha1",
2643 /* SHA-256 */
2655 .cra_driver_name = "artpec-sha256",
2667 /* HMAC SHA-256 */
2680 .cra_driver_name = "artpec-hmac-sha256",
2696 /* AES - ECB */
2700 .cra_driver_name = "artpec6-ecb-aes",
2717 /* AES - CTR */
2721 .cra_driver_name = "artpec6-ctr-aes",
2740 /* AES - CBC */
2744 .cra_driver_name = "artpec6-cbc-aes",
2762 /* AES - XTS */
2766 .cra_driver_name = "artpec6-xts-aes",
2797 .cra_driver_name = "artpec-gcm-aes",
2842 { .compatible = "axis,artpec6-crypto", .data = (void *)ARTPEC6_CRYPTO },
2843 { .compatible = "axis,artpec7-crypto", .data = (void *)ARTPEC7_CRYPTO },
2853 struct device *dev = &pdev->dev; in artpec6_crypto_probe()
2859 return -ENODEV; in artpec6_crypto_probe()
2861 match = of_match_node(artpec6_crypto_of_match, dev->of_node); in artpec6_crypto_probe()
2863 return -EINVAL; in artpec6_crypto_probe()
2865 variant = (enum artpec6_crypto_variant)match->data; in artpec6_crypto_probe()
2873 return -ENODEV; in artpec6_crypto_probe()
2875 ac = devm_kzalloc(&pdev->dev, sizeof(struct artpec6_crypto), in artpec6_crypto_probe()
2878 return -ENOMEM; in artpec6_crypto_probe()
2881 ac->variant = variant; in artpec6_crypto_probe()
2883 spin_lock_init(&ac->queue_lock); in artpec6_crypto_probe()
2884 INIT_LIST_HEAD(&ac->queue); in artpec6_crypto_probe()
2885 INIT_LIST_HEAD(&ac->pending); in artpec6_crypto_probe()
2886 timer_setup(&ac->timer, artpec6_crypto_timeout, 0); in artpec6_crypto_probe()
2888 ac->base = base; in artpec6_crypto_probe()
2890 ac->dma_cache = kmem_cache_create("artpec6_crypto_dma", in artpec6_crypto_probe()
2895 if (!ac->dma_cache) in artpec6_crypto_probe()
2896 return -ENOMEM; in artpec6_crypto_probe()
2902 tasklet_init(&ac->task, artpec6_crypto_task, in artpec6_crypto_probe()
2905 ac->pad_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, in artpec6_crypto_probe()
2907 if (!ac->pad_buffer) in artpec6_crypto_probe()
2908 return -ENOMEM; in artpec6_crypto_probe()
2909 ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX); in artpec6_crypto_probe()
2911 ac->zero_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, in artpec6_crypto_probe()
2913 if (!ac->zero_buffer) in artpec6_crypto_probe()
2914 return -ENOMEM; in artpec6_crypto_probe()
2915 ac->zero_buffer = PTR_ALIGN(ac->zero_buffer, ARTPEC_CACHE_LINE_MAX); in artpec6_crypto_probe()
2921 err = devm_request_irq(&pdev->dev, irq, artpec6_crypto_irq, 0, in artpec6_crypto_probe()
2922 "artpec6-crypto", ac); in artpec6_crypto_probe()
2926 artpec6_crypto_dev = &pdev->dev; in artpec6_crypto_probe()
2955 kmem_cache_destroy(ac->dma_cache); in artpec6_crypto_probe()
2968 tasklet_disable(&ac->task); in artpec6_crypto_remove()
2969 devm_free_irq(&pdev->dev, irq, ac); in artpec6_crypto_remove()
2970 tasklet_kill(&ac->task); in artpec6_crypto_remove()
2971 del_timer_sync(&ac->timer); in artpec6_crypto_remove()
2975 kmem_cache_destroy(ac->dma_cache); in artpec6_crypto_remove()
2986 .name = "artpec6-crypto",
2994 MODULE_DESCRIPTION("ARTPEC-6 Crypto driver");