Lines Matching +full:aes +full:- +full:gcm
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Support for Intel AES-NI instructions. This file contains glue
4 * code, the real AES implementation is in intel-aes_asm.S.
9 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
10 * interface for 64-bit kernels.
23 #include <crypto/aes.h>
26 #include <crypto/gcm.h>
43 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1))
45 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
114 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
116 * u8 *out, Ciphertext output. Encrypt in-place is allowed.
119 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
120 * 16-byte aligned pointer.
121 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
135 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
137 * u8 *out, Plaintext output. Decrypt in-place is allowed.
140 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
141 * 16-byte aligned pointer.
142 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
200 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
242 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
320 return -EINVAL; in aes_set_key_common()
386 nbytes &= AES_BLOCK_SIZE - 1; in ecb_encrypt()
408 nbytes &= AES_BLOCK_SIZE - 1; in ecb_decrypt()
430 nbytes &= AES_BLOCK_SIZE - 1; in cbc_encrypt()
452 nbytes &= AES_BLOCK_SIZE - 1; in cbc_decrypt()
464 u8 *ctrblk = walk->iv; in ctr_crypt_final()
466 u8 *src = walk->src.virt.addr; in ctr_crypt_final()
467 u8 *dst = walk->dst.virt.addr; in ctr_crypt_final()
468 unsigned int nbytes = walk->nbytes; in ctr_crypt_final()
485 if (ctx->key_length == AES_KEYSIZE_128) in aesni_ctr_enc_avx_tfm()
487 else if (ctx->key_length == AES_KEYSIZE_192) in aesni_ctr_enc_avx_tfm()
507 nbytes &= AES_BLOCK_SIZE - 1; in ctr_crypt()
531 /* first half of xts-key is for crypt */ in xts_aesni_setkey()
532 err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx, in xts_aesni_setkey()
537 /* second half of xts-key is for tweak */ in xts_aesni_setkey()
538 return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx, in xts_aesni_setkey()
595 aes_ctx(ctx->raw_tweak_ctx), in xts_encrypt()
596 aes_ctx(ctx->raw_crypt_ctx), in xts_encrypt()
606 aes_ctx(ctx->raw_tweak_ctx), in xts_decrypt()
607 aes_ctx(ctx->raw_crypt_ctx), in xts_decrypt()
637 return -EINVAL; in common_rfc4106_set_key()
640 key_len -= 4; in common_rfc4106_set_key()
642 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); in common_rfc4106_set_key()
645 &ctx->aes_key_expanded, key, key_len) ?: in common_rfc4106_set_key()
646 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); in common_rfc4106_set_key()
660 return -EINVAL; in common_rfc4106_set_authsize()
679 return -EINVAL; in generic_gcmaes_set_authsize()
692 u8 databuf[sizeof(struct gcm_context_data) + (AESNI_ALIGN - 8)] __aligned(8); in gcmaes_crypt_by_sg()
695 unsigned long left = req->cryptlen; in gcmaes_crypt_by_sg()
708 left -= auth_tag_len; in gcmaes_crypt_by_sg()
716 if (req->src->length >= assoclen && req->src->length && in gcmaes_crypt_by_sg()
717 (!PageHighMem(sg_page(req->src)) || in gcmaes_crypt_by_sg()
718 req->src->offset + req->src->length <= PAGE_SIZE)) { in gcmaes_crypt_by_sg()
719 scatterwalk_start(&assoc_sg_walk, req->src); in gcmaes_crypt_by_sg()
725 return -ENOMEM; in gcmaes_crypt_by_sg()
728 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0); in gcmaes_crypt_by_sg()
732 src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen); in gcmaes_crypt_by_sg()
734 if (req->src != req->dst) { in gcmaes_crypt_by_sg()
735 dst_sg = scatterwalk_ffwd(dst_start, req->dst, in gcmaes_crypt_by_sg()
736 req->assoclen); in gcmaes_crypt_by_sg()
742 gcm_tfm->init(aes_ctx, data, iv, hash_subkey, assoc, assoclen); in gcmaes_crypt_by_sg()
743 if (req->src != req->dst) { in gcmaes_crypt_by_sg()
752 gcm_tfm->enc_update(aes_ctx, data, in gcmaes_crypt_by_sg()
755 gcm_tfm->dec_update(aes_ctx, data, in gcmaes_crypt_by_sg()
758 left -= len; in gcmaes_crypt_by_sg()
773 gcm_tfm->enc_update(aes_ctx, data, in gcmaes_crypt_by_sg()
776 gcm_tfm->dec_update(aes_ctx, data, in gcmaes_crypt_by_sg()
779 left -= len; in gcmaes_crypt_by_sg()
785 gcm_tfm->finalize(aes_ctx, data, authTag, auth_tag_len); in gcmaes_crypt_by_sg()
797 scatterwalk_map_and_copy(authTagMsg, req->src, in gcmaes_crypt_by_sg()
798 req->assoclen + req->cryptlen - in gcmaes_crypt_by_sg()
804 -EBADMSG : 0; in gcmaes_crypt_by_sg()
808 scatterwalk_map_and_copy(authTag, req->dst, in gcmaes_crypt_by_sg()
809 req->assoclen + req->cryptlen, in gcmaes_crypt_by_sg()
833 void *aes_ctx = &(ctx->aes_key_expanded); in helper_rfc4106_encrypt()
834 u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); in helper_rfc4106_encrypt()
839 /* Assuming we are supporting rfc4106 64-bit extended */ in helper_rfc4106_encrypt()
842 if (unlikely(req->assoclen != 16 && req->assoclen != 20)) in helper_rfc4106_encrypt()
843 return -EINVAL; in helper_rfc4106_encrypt()
847 *(iv+i) = ctx->nonce[i]; in helper_rfc4106_encrypt()
849 *(iv+4+i) = req->iv[i]; in helper_rfc4106_encrypt()
852 return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, in helper_rfc4106_encrypt()
861 void *aes_ctx = &(ctx->aes_key_expanded); in helper_rfc4106_decrypt()
862 u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); in helper_rfc4106_decrypt()
866 if (unlikely(req->assoclen != 16 && req->assoclen != 20)) in helper_rfc4106_decrypt()
867 return -EINVAL; in helper_rfc4106_decrypt()
869 /* Assuming we are supporting rfc4106 64-bit extended */ in helper_rfc4106_decrypt()
875 *(iv+i) = ctx->nonce[i]; in helper_rfc4106_decrypt()
877 *(iv+4+i) = req->iv[i]; in helper_rfc4106_decrypt()
880 return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, in helper_rfc4106_decrypt()
886 .cra_name = "aes",
887 .cra_driver_name = "aes-aesni",
907 .cra_name = "__ecb(aes)",
908 .cra_driver_name = "__ecb-aes-aesni",
922 .cra_name = "__cbc(aes)",
923 .cra_driver_name = "__cbc-aes-aesni",
939 .cra_name = "__ctr(aes)",
940 .cra_driver_name = "__ctr-aes-aesni",
956 .cra_name = "__xts(aes)",
957 .cra_driver_name = "__xts-aes-aesni",
984 &ctx->aes_key_expanded, key, key_len) ?: in generic_gcmaes_set_key()
985 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); in generic_gcmaes_set_key()
992 void *aes_ctx = &(ctx->aes_key_expanded); in generic_gcmaes_encrypt()
993 u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); in generic_gcmaes_encrypt()
997 memcpy(iv, req->iv, 12); in generic_gcmaes_encrypt()
1000 return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv, in generic_gcmaes_encrypt()
1009 void *aes_ctx = &(ctx->aes_key_expanded); in generic_gcmaes_decrypt()
1010 u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); in generic_gcmaes_decrypt()
1013 memcpy(iv, req->iv, 12); in generic_gcmaes_decrypt()
1016 return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv, in generic_gcmaes_decrypt()
1028 .cra_name = "__rfc4106(gcm(aes))",
1029 .cra_driver_name = "__rfc4106-gcm-aesni",
1034 .cra_alignmask = AESNI_ALIGN - 1,
1045 .cra_name = "__gcm(aes)",
1046 .cra_driver_name = "__generic-gcm-aesni",
1051 .cra_alignmask = AESNI_ALIGN - 1,
1072 return -ENODEV; in aesni_init()
1089 pr_info("AES CTR mode by8 optimization enabled\n"); in aesni_init()
1130 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1132 MODULE_ALIAS_CRYPTO("aes");