Lines Matching +full:aes +full:- +full:gcm
2 * Support for Intel AES-NI instructions. This file contains glue
3 * code, the real AES implementation is in intel-aes_asm.S.
8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
9 * interface for 64-bit kernels.
27 #include <crypto/aes.h>
31 #include <crypto/gcm.h>
35 #include <asm/crypto/aes.h>
49 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1))
51 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
122 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
124 * u8 *out, Ciphertext output. Encrypt in-place is allowed.
127 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
128 * 16-byte aligned pointer.
129 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
143 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
145 * u8 *out, Plaintext output. Decrypt in-place is allowed.
148 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
149 * 16-byte aligned pointer.
150 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
191 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
212 if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)){ in aesni_gcm_enc_avx()
230 if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { in aesni_gcm_dec_avx()
246 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
267 if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { in aesni_gcm_enc_avx2()
289 if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { in aesni_gcm_dec_avx2()
354 u32 *flags = &tfm->crt_flags; in aes_set_key_common()
360 return -EINVAL; in aes_set_key_common()
441 nbytes &= AES_BLOCK_SIZE - 1; in ecb_encrypt()
463 nbytes &= AES_BLOCK_SIZE - 1; in ecb_decrypt()
485 nbytes &= AES_BLOCK_SIZE - 1; in cbc_encrypt()
507 nbytes &= AES_BLOCK_SIZE - 1; in cbc_decrypt()
519 u8 *ctrblk = walk->iv; in ctr_crypt_final()
521 u8 *src = walk->src.virt.addr; in ctr_crypt_final()
522 u8 *dst = walk->dst.virt.addr; in ctr_crypt_final()
523 unsigned int nbytes = walk->nbytes; in ctr_crypt_final()
541 if (ctx->key_length == AES_KEYSIZE_128) in aesni_ctr_enc_avx_tfm()
543 else if (ctx->key_length == AES_KEYSIZE_192) in aesni_ctr_enc_avx_tfm()
564 nbytes &= AES_BLOCK_SIZE - 1; in ctr_crypt()
588 /* first half of xts-key is for crypt */ in xts_aesni_setkey()
589 err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx, in xts_aesni_setkey()
594 /* second half of xts-key is for tweak */ in xts_aesni_setkey()
595 return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx, in xts_aesni_setkey()
658 aes_ctx(ctx->raw_tweak_ctx), in xts_encrypt()
659 aes_ctx(ctx->raw_crypt_ctx)); in xts_encrypt()
669 aes_ctx(ctx->raw_tweak_ctx), in xts_decrypt()
670 aes_ctx(ctx->raw_crypt_ctx)); in xts_decrypt()
678 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", in rfc4106_init()
685 crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base)); in rfc4106_init()
702 tfm = crypto_alloc_cipher("aes", 0, 0); in rfc4106_set_hash_subkey()
728 return -EINVAL; in common_rfc4106_set_key()
731 key_len -= 4; in common_rfc4106_set_key()
733 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); in common_rfc4106_set_key()
736 &ctx->aes_key_expanded, key, key_len) ?: in common_rfc4106_set_key()
737 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); in common_rfc4106_set_key()
746 return crypto_aead_setkey(&cryptd_tfm->base, key, key_len); in gcmaes_wrapper_set_key()
758 return -EINVAL; in common_rfc4106_set_authsize()
772 return crypto_aead_setauthsize(&cryptd_tfm->base, authsize); in gcmaes_wrapper_set_authsize()
788 return -EINVAL; in generic_gcmaes_set_authsize()
802 unsigned long left = req->cryptlen; in gcmaes_crypt_by_sg()
815 left -= auth_tag_len; in gcmaes_crypt_by_sg()
818 if (req->src->length >= assoclen && req->src->length && in gcmaes_crypt_by_sg()
819 (!PageHighMem(sg_page(req->src)) || in gcmaes_crypt_by_sg()
820 req->src->offset + req->src->length <= PAGE_SIZE)) { in gcmaes_crypt_by_sg()
821 scatterwalk_start(&assoc_sg_walk, req->src); in gcmaes_crypt_by_sg()
827 return -ENOMEM; in gcmaes_crypt_by_sg()
830 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0); in gcmaes_crypt_by_sg()
834 src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen); in gcmaes_crypt_by_sg()
836 if (req->src != req->dst) { in gcmaes_crypt_by_sg()
837 dst_sg = scatterwalk_ffwd(dst_start, req->dst, in gcmaes_crypt_by_sg()
838 req->assoclen); in gcmaes_crypt_by_sg()
846 if (req->src != req->dst) { in gcmaes_crypt_by_sg()
861 left -= len; in gcmaes_crypt_by_sg()
882 left -= len; in gcmaes_crypt_by_sg()
900 scatterwalk_map_and_copy(authTagMsg, req->src, in gcmaes_crypt_by_sg()
901 req->assoclen + req->cryptlen - in gcmaes_crypt_by_sg()
907 -EBADMSG : 0; in gcmaes_crypt_by_sg()
911 scatterwalk_map_and_copy(authTag, req->dst, in gcmaes_crypt_by_sg()
912 req->assoclen + req->cryptlen, in gcmaes_crypt_by_sg()
929 if (((struct crypto_aes_ctx *)aes_ctx)->key_length != AES_KEYSIZE_128 || in gcmaes_encrypt()
931 req->cryptlen < AVX_GEN2_OPTSIZE) { in gcmaes_encrypt()
935 if (sg_is_last(req->src) && in gcmaes_encrypt()
936 (!PageHighMem(sg_page(req->src)) || in gcmaes_encrypt()
937 req->src->offset + req->src->length <= PAGE_SIZE) && in gcmaes_encrypt()
938 sg_is_last(req->dst) && in gcmaes_encrypt()
939 (!PageHighMem(sg_page(req->dst)) || in gcmaes_encrypt()
940 req->dst->offset + req->dst->length <= PAGE_SIZE)) { in gcmaes_encrypt()
942 scatterwalk_start(&src_sg_walk, req->src); in gcmaes_encrypt()
944 src = assoc + req->assoclen; in gcmaes_encrypt()
946 if (unlikely(req->src != req->dst)) { in gcmaes_encrypt()
947 scatterwalk_start(&dst_sg_walk, req->dst); in gcmaes_encrypt()
948 dst = scatterwalk_map(&dst_sg_walk) + req->assoclen; in gcmaes_encrypt()
952 assoc = kmalloc(req->cryptlen + auth_tag_len + req->assoclen, in gcmaes_encrypt()
955 return -ENOMEM; in gcmaes_encrypt()
956 scatterwalk_map_and_copy(assoc, req->src, 0, in gcmaes_encrypt()
957 req->assoclen + req->cryptlen, 0); in gcmaes_encrypt()
958 src = assoc + req->assoclen; in gcmaes_encrypt()
963 aesni_gcm_enc_tfm(aes_ctx, &data, dst, src, req->cryptlen, iv, in gcmaes_encrypt()
965 dst + req->cryptlen, auth_tag_len); in gcmaes_encrypt()
971 if (unlikely(req->src != req->dst)) { in gcmaes_encrypt()
972 scatterwalk_unmap(dst - req->assoclen); in gcmaes_encrypt()
973 scatterwalk_advance(&dst_sg_walk, req->dst->length); in gcmaes_encrypt()
977 scatterwalk_advance(&src_sg_walk, req->src->length); in gcmaes_encrypt()
978 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0); in gcmaes_encrypt()
980 scatterwalk_map_and_copy(dst, req->dst, req->assoclen, in gcmaes_encrypt()
981 req->cryptlen + auth_tag_len, 1); in gcmaes_encrypt()
1001 if (((struct crypto_aes_ctx *)aes_ctx)->key_length != AES_KEYSIZE_128 || in gcmaes_decrypt()
1003 req->cryptlen < AVX_GEN2_OPTSIZE) { in gcmaes_decrypt()
1007 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len); in gcmaes_decrypt()
1009 if (sg_is_last(req->src) && in gcmaes_decrypt()
1010 (!PageHighMem(sg_page(req->src)) || in gcmaes_decrypt()
1011 req->src->offset + req->src->length <= PAGE_SIZE) && in gcmaes_decrypt()
1012 sg_is_last(req->dst) && req->dst->length && in gcmaes_decrypt()
1013 (!PageHighMem(sg_page(req->dst)) || in gcmaes_decrypt()
1014 req->dst->offset + req->dst->length <= PAGE_SIZE)) { in gcmaes_decrypt()
1016 scatterwalk_start(&src_sg_walk, req->src); in gcmaes_decrypt()
1018 src = assoc + req->assoclen; in gcmaes_decrypt()
1020 if (unlikely(req->src != req->dst)) { in gcmaes_decrypt()
1021 scatterwalk_start(&dst_sg_walk, req->dst); in gcmaes_decrypt()
1022 dst = scatterwalk_map(&dst_sg_walk) + req->assoclen; in gcmaes_decrypt()
1026 assoc = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC); in gcmaes_decrypt()
1028 return -ENOMEM; in gcmaes_decrypt()
1029 scatterwalk_map_and_copy(assoc, req->src, 0, in gcmaes_decrypt()
1030 req->assoclen + req->cryptlen, 0); in gcmaes_decrypt()
1031 src = assoc + req->assoclen; in gcmaes_decrypt()
1044 -EBADMSG : 0; in gcmaes_decrypt()
1047 if (unlikely(req->src != req->dst)) { in gcmaes_decrypt()
1048 scatterwalk_unmap(dst - req->assoclen); in gcmaes_decrypt()
1049 scatterwalk_advance(&dst_sg_walk, req->dst->length); in gcmaes_decrypt()
1053 scatterwalk_advance(&src_sg_walk, req->src->length); in gcmaes_decrypt()
1054 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0); in gcmaes_decrypt()
1056 scatterwalk_map_and_copy(dst, req->dst, req->assoclen, in gcmaes_decrypt()
1068 void *aes_ctx = &(ctx->aes_key_expanded); in helper_rfc4106_encrypt()
1073 /* Assuming we are supporting rfc4106 64-bit extended */ in helper_rfc4106_encrypt()
1076 if (unlikely(req->assoclen != 16 && req->assoclen != 20)) in helper_rfc4106_encrypt()
1077 return -EINVAL; in helper_rfc4106_encrypt()
1081 *(iv+i) = ctx->nonce[i]; in helper_rfc4106_encrypt()
1083 *(iv+4+i) = req->iv[i]; in helper_rfc4106_encrypt()
1086 return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, in helper_rfc4106_encrypt()
1095 void *aes_ctx = &(ctx->aes_key_expanded); in helper_rfc4106_decrypt()
1099 if (unlikely(req->assoclen != 16 && req->assoclen != 20)) in helper_rfc4106_decrypt()
1100 return -EINVAL; in helper_rfc4106_decrypt()
1102 /* Assuming we are supporting rfc4106 64-bit extended */ in helper_rfc4106_decrypt()
1108 *(iv+i) = ctx->nonce[i]; in helper_rfc4106_decrypt()
1110 *(iv+4+i) = req->iv[i]; in helper_rfc4106_decrypt()
1113 return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, in helper_rfc4106_decrypt()
1123 tfm = &cryptd_tfm->base; in gcmaes_wrapper_encrypt()
1139 tfm = &cryptd_tfm->base; in gcmaes_wrapper_decrypt()
1151 .cra_name = "aes",
1152 .cra_driver_name = "aes-aesni",
1169 .cra_driver_name = "__aes-aesni",
1189 .cra_name = "__ecb(aes)",
1190 .cra_driver_name = "__ecb-aes-aesni",
1204 .cra_name = "__cbc(aes)",
1205 .cra_driver_name = "__cbc-aes-aesni",
1221 .cra_name = "__ctr(aes)",
1222 .cra_driver_name = "__ctr-aes-aesni",
1238 .cra_name = "__xts(aes)",
1239 .cra_driver_name = "__xts-aes-aesni",
1268 .algname = "pcbc(aes)",
1269 .drvname = "pcbc-aes-aesni",
1270 .basename = "fpu(pcbc(__aes-aesni))",
1282 &ctx->aes_key_expanded, key, key_len) ?: in generic_gcmaes_set_key()
1283 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); in generic_gcmaes_set_key()
1290 void *aes_ctx = &(ctx->aes_key_expanded); in generic_gcmaes_encrypt()
1294 memcpy(iv, req->iv, 12); in generic_gcmaes_encrypt()
1297 return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv, in generic_gcmaes_encrypt()
1306 void *aes_ctx = &(ctx->aes_key_expanded); in generic_gcmaes_decrypt()
1309 memcpy(iv, req->iv, 12); in generic_gcmaes_decrypt()
1312 return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv, in generic_gcmaes_decrypt()
1321 cryptd_tfm = cryptd_alloc_aead("__driver-generic-gcm-aes-aesni", in generic_gcmaes_init()
1328 crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base)); in generic_gcmaes_init()
1348 .cra_name = "__gcm-aes-aesni",
1349 .cra_driver_name = "__driver-gcm-aes-aesni",
1353 .cra_alignmask = AESNI_ALIGN - 1,
1366 .cra_name = "rfc4106(gcm(aes))",
1367 .cra_driver_name = "rfc4106-gcm-aesni",
1382 .cra_name = "__generic-gcm-aes-aesni",
1383 .cra_driver_name = "__driver-generic-gcm-aes-aesni",
1388 .cra_alignmask = AESNI_ALIGN - 1,
1401 .cra_name = "gcm(aes)",
1402 .cra_driver_name = "generic-gcm-aesni",
1444 return -ENODEV; in aesni_init()
1470 pr_info("AES CTR mode by8 optimization enabled\n"); in aesni_init()
1546 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1548 MODULE_ALIAS_CRYPTO("aes");