31 static int crypto_check_alg(struct crypto_alg *alg)  in crypto_check_alg()  argument
33 	crypto_check_module_sig(alg->cra_module);  in crypto_check_alg()
35 	if (!alg->cra_name[0] || !alg->cra_driver_name[0])  in crypto_check_alg()
38 	if (alg->cra_alignmask & (alg->cra_alignmask + 1))  in crypto_check_alg()
42 	if (alg->cra_alignmask > MAX_ALGAPI_ALIGNMASK)  in crypto_check_alg()
45 	if (alg->cra_blocksize > MAX_ALGAPI_BLOCKSIZE)  in crypto_check_alg()
49 	if (!alg->cra_type && (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==  in crypto_check_alg()
51 		if (alg->cra_alignmask > MAX_CIPHER_ALIGNMASK)  in crypto_check_alg()
54 		if (alg->cra_blocksize > MAX_CIPHER_BLOCKSIZE)  in crypto_check_alg()
58 	if (alg->cra_priority < 0)  in crypto_check_alg()
61 	refcount_set(&alg->cra_refcnt, 1);  in crypto_check_alg()
68 	if (!inst->alg.cra_type->free) {  in crypto_free_instance()
73 	inst->alg.cra_type->free(inst);  in crypto_free_instance()
76 static void crypto_destroy_instance(struct crypto_alg *alg)  in crypto_destroy_instance()  argument
78 	struct crypto_instance *inst = (void *)alg;  in crypto_destroy_instance()
85 static struct list_head *crypto_more_spawns(struct crypto_alg *alg,  in crypto_more_spawns()  argument
98 	if (spawn->alg && &n->list != stack && !n->alg)  in crypto_more_spawns()
99 		n->alg = (n->list.next == stack) ? alg :  in crypto_more_spawns()
100 			 &list_next_entry(n, list)->inst->alg;  in crypto_more_spawns()
104 	return &n->list == stack ? top : &n->inst->alg.cra_users;  in crypto_more_spawns()
112 	if (crypto_is_dead(&inst->alg))  in crypto_remove_instance()
115 	inst->alg.cra_flags |= CRYPTO_ALG_DEAD;  in crypto_remove_instance()
122 	list_move(&inst->alg.cra_list, list);  in crypto_remove_instance()
124 	inst->alg.cra_destroy = crypto_destroy_instance;  in crypto_remove_instance()
126 	BUG_ON(!list_empty(&inst->alg.cra_users));  in crypto_remove_instance()
129 void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list,  in crypto_remove_spawns()  argument
132 	u32 new_type = (nalg ?: alg)->cra_flags;  in crypto_remove_spawns()
139 	spawns = &alg->cra_users;  in crypto_remove_spawns()
141 		if ((spawn->alg->cra_flags ^ new_type) & spawn->mask)  in crypto_remove_spawns()
156 			BUG_ON(&inst->alg == alg);  in crypto_remove_spawns()
160 			if (&inst->alg == nalg)  in crypto_remove_spawns()
163 			spawn->alg = NULL;  in crypto_remove_spawns()
164 			spawns = &inst->alg.cra_users;  in crypto_remove_spawns()
178 	} while ((spawns = crypto_more_spawns(alg, &stack, &top,  in crypto_remove_spawns()
182 		if (spawn->alg)  in crypto_remove_spawns()
183 			list_move(&spawn->list, &spawn->alg->cra_users);  in crypto_remove_spawns()
190 static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg)  in __crypto_register_alg()  argument
196 	if (crypto_is_dead(alg))  in __crypto_register_alg()
199 	INIT_LIST_HEAD(&alg->cra_users);  in __crypto_register_alg()
202 	alg->cra_flags &= ~CRYPTO_ALG_TESTED;  in __crypto_register_alg()
207 		if (q == alg)  in __crypto_register_alg()
214 			if (!strcmp(alg->cra_driver_name, q->cra_driver_name))  in __crypto_register_alg()
219 		if (!strcmp(q->cra_driver_name, alg->cra_name) ||  in __crypto_register_alg()
220 		    !strcmp(q->cra_name, alg->cra_driver_name))  in __crypto_register_alg()
224 	larval = crypto_larval_alloc(alg->cra_name,  in __crypto_register_alg()
225 				     alg->cra_flags | CRYPTO_ALG_TESTED, 0);  in __crypto_register_alg()
230 	larval->adult = crypto_mod_get(alg);  in __crypto_register_alg()
234 	refcount_set(&larval->alg.cra_refcnt, 1);  in __crypto_register_alg()
235 	memcpy(larval->alg.cra_driver_name, alg->cra_driver_name,  in __crypto_register_alg()
237 	larval->alg.cra_priority = alg->cra_priority;  in __crypto_register_alg()
239 	list_add(&alg->cra_list, &crypto_alg_list);  in __crypto_register_alg()
240 	list_add(&larval->alg.cra_list, &crypto_alg_list);  in __crypto_register_alg()
242 	crypto_stats_init(alg);  in __crypto_register_alg()
257 	struct crypto_alg *alg;  in crypto_alg_tested()  local
277 	alg = test->adult;  in crypto_alg_tested()
278 	if (err || list_empty(&alg->cra_list))  in crypto_alg_tested()
281 	alg->cra_flags |= CRYPTO_ALG_TESTED;  in crypto_alg_tested()
284 		if (q == alg)  in crypto_alg_tested()
298 			if (strcmp(alg->cra_name, q->cra_name) &&  in crypto_alg_tested()
299 			    strcmp(alg->cra_driver_name, q->cra_name))  in crypto_alg_tested()
304 			if ((q->cra_flags ^ alg->cra_flags) & larval->mask)  in crypto_alg_tested()
306 			if (!crypto_mod_get(alg))  in crypto_alg_tested()
309 			larval->adult = alg;  in crypto_alg_tested()
313 		if (strcmp(alg->cra_name, q->cra_name))  in crypto_alg_tested()
316 		if (strcmp(alg->cra_driver_name, q->cra_driver_name) &&  in crypto_alg_tested()
317 		    q->cra_priority > alg->cra_priority)  in crypto_alg_tested()
320 		crypto_remove_spawns(q, &list, alg);  in crypto_alg_tested()
335 	struct crypto_alg *alg;  in crypto_remove_final()  local
338 	list_for_each_entry_safe(alg, n, list, cra_list) {  in crypto_remove_final()
339 		list_del_init(&alg->cra_list);  in crypto_remove_final()
340 		crypto_alg_put(alg);  in crypto_remove_final()
353 		crypto_alg_tested(larval->alg.cra_driver_name, 0);  in crypto_wait_for_test()
362 	crypto_larval_kill(&larval->alg);  in crypto_wait_for_test()
365 int crypto_register_alg(struct crypto_alg *alg)  in crypto_register_alg()  argument
370 	alg->cra_flags &= ~CRYPTO_ALG_DEAD;  in crypto_register_alg()
371 	err = crypto_check_alg(alg);  in crypto_register_alg()
376 	larval = __crypto_register_alg(alg);  in crypto_register_alg()
387 static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list)  in crypto_remove_alg()  argument
389 	if (unlikely(list_empty(&alg->cra_list)))  in crypto_remove_alg()
392 	alg->cra_flags |= CRYPTO_ALG_DEAD;  in crypto_remove_alg()
394 	list_del_init(&alg->cra_list);  in crypto_remove_alg()
395 	crypto_remove_spawns(alg, list, NULL);  in crypto_remove_alg()
400 int crypto_unregister_alg(struct crypto_alg *alg)  in crypto_unregister_alg()  argument
406 	ret = crypto_remove_alg(alg, &list);  in crypto_unregister_alg()
412 	BUG_ON(refcount_read(&alg->cra_refcnt) != 1);  in crypto_unregister_alg()
413 	if (alg->cra_destroy)  in crypto_unregister_alg()
414 		alg->cra_destroy(alg);  in crypto_unregister_alg()
510 		int err = crypto_remove_alg(&inst->alg, &users);  in crypto_unregister_template()
518 		BUG_ON(refcount_read(&inst->alg.cra_refcnt) != 1);  in crypto_unregister_template()
566 	err = crypto_check_alg(&inst->alg);  in crypto_register_instance()
570 	inst->alg.cra_module = tmpl->module;  in crypto_register_instance()
571 	inst->alg.cra_flags |= CRYPTO_ALG_INSTANCE;  in crypto_register_instance()
575 	larval = __crypto_register_alg(&inst->alg);  in crypto_register_instance()
603 	crypto_remove_spawns(&inst->alg, &list, NULL);  in crypto_unregister_instance()
614 int crypto_init_spawn(struct crypto_spawn *spawn, struct crypto_alg *alg,  in crypto_init_spawn()  argument
626 	if (!crypto_is_moribund(alg)) {  in crypto_init_spawn()
627 		list_add(&spawn->list, &alg->cra_users);  in crypto_init_spawn()
628 		spawn->alg = alg;  in crypto_init_spawn()
637 int crypto_init_spawn2(struct crypto_spawn *spawn, struct crypto_alg *alg,  in crypto_init_spawn2()  argument
643 	if ((alg->cra_flags ^ frontend->type) & frontend->maskset)  in crypto_init_spawn2()
647 	err = crypto_init_spawn(spawn, alg, inst, frontend->maskset);  in crypto_init_spawn2()
657 	struct crypto_alg *alg;  in crypto_grab_spawn()  local
660 	alg = crypto_find_alg(name, spawn->frontend, type, mask);  in crypto_grab_spawn()
661 	if (IS_ERR(alg))  in crypto_grab_spawn()
662 		return PTR_ERR(alg);  in crypto_grab_spawn()
664 	err = crypto_init_spawn(spawn, alg, spawn->inst, mask);  in crypto_grab_spawn()
665 	crypto_mod_put(alg);  in crypto_grab_spawn()
672 	if (!spawn->alg)  in crypto_drop_spawn()
683 	struct crypto_alg *alg;  in crypto_spawn_alg()  local
687 	alg = spawn->alg;  in crypto_spawn_alg()
688 	alg2 = alg;  in crypto_spawn_alg()
694 		if (alg)  in crypto_spawn_alg()
695 			crypto_shoot_alg(alg);  in crypto_spawn_alg()
699 	return alg;  in crypto_spawn_alg()
705 	struct crypto_alg *alg;  in crypto_spawn_tfm()  local
708 	alg = crypto_spawn_alg(spawn);  in crypto_spawn_tfm()
709 	if (IS_ERR(alg))  in crypto_spawn_tfm()
710 		return ERR_CAST(alg);  in crypto_spawn_tfm()
713 	if (unlikely((alg->cra_flags ^ type) & mask))  in crypto_spawn_tfm()
716 	tfm = __crypto_alloc_tfm(alg, type, mask);  in crypto_spawn_tfm()
723 	crypto_mod_put(alg);  in crypto_spawn_tfm()
730 	struct crypto_alg *alg;  in crypto_spawn_tfm2()  local
733 	alg = crypto_spawn_alg(spawn);  in crypto_spawn_tfm2()
734 	if (IS_ERR(alg))  in crypto_spawn_tfm2()
735 		return ERR_CAST(alg);  in crypto_spawn_tfm2()
737 	tfm = crypto_create_tfm(alg, spawn->frontend);  in crypto_spawn_tfm2()
744 	crypto_mod_put(alg);  in crypto_spawn_tfm2()
845 			struct crypto_alg *alg)  in crypto_inst_setname()  argument
847 	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name,  in crypto_inst_setname()
848 		     alg->cra_name) >= CRYPTO_MAX_ALG_NAME)  in crypto_inst_setname()
851 	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",  in crypto_inst_setname()
852 		     name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)  in crypto_inst_setname()
859 void *crypto_alloc_instance(const char *name, struct crypto_alg *alg,  in crypto_alloc_instance()  argument
873 	err = crypto_inst_setname(inst, name, alg);  in crypto_alloc_instance()
1020 unsigned int crypto_alg_extsize(struct crypto_alg *alg)  in crypto_alg_extsize()  argument
1022 	return alg->cra_ctxsize +  in crypto_alg_extsize()
1023 	       (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1));  in crypto_alg_extsize()
1031 	struct crypto_alg *alg = crypto_find_alg(name, frontend, type, mask);  in crypto_type_has_alg()  local
1033 	if (!IS_ERR(alg)) {  in crypto_type_has_alg()
1034 		crypto_mod_put(alg);  in crypto_type_has_alg()
1043 void crypto_stats_init(struct crypto_alg *alg)  in crypto_stats_init()  argument
1045 	memset(&alg->stats, 0, sizeof(alg->stats));  in crypto_stats_init()
1049 void crypto_stats_get(struct crypto_alg *alg)  in crypto_stats_get()  argument
1051 	crypto_alg_get(alg);  in crypto_stats_get()
1056 				     struct crypto_alg *alg)  in crypto_stats_ablkcipher_encrypt()  argument
1059 		atomic64_inc(&alg->stats.cipher.err_cnt);  in crypto_stats_ablkcipher_encrypt()
1061 		atomic64_inc(&alg->stats.cipher.encrypt_cnt);  in crypto_stats_ablkcipher_encrypt()
1062 		atomic64_add(nbytes, &alg->stats.cipher.encrypt_tlen);  in crypto_stats_ablkcipher_encrypt()
1064 	crypto_alg_put(alg);  in crypto_stats_ablkcipher_encrypt()
1069 				     struct crypto_alg *alg)  in crypto_stats_ablkcipher_decrypt()  argument
1072 		atomic64_inc(&alg->stats.cipher.err_cnt);  in crypto_stats_ablkcipher_decrypt()
1074 		atomic64_inc(&alg->stats.cipher.decrypt_cnt);  in crypto_stats_ablkcipher_decrypt()
1075 		atomic64_add(nbytes, &alg->stats.cipher.decrypt_tlen);  in crypto_stats_ablkcipher_decrypt()
1077 	crypto_alg_put(alg);  in crypto_stats_ablkcipher_decrypt()
1081 void crypto_stats_aead_encrypt(unsigned int cryptlen, struct crypto_alg *alg,  in crypto_stats_aead_encrypt()  argument
1085 		atomic64_inc(&alg->stats.aead.err_cnt);  in crypto_stats_aead_encrypt()
1087 		atomic64_inc(&alg->stats.aead.encrypt_cnt);  in crypto_stats_aead_encrypt()
1088 		atomic64_add(cryptlen, &alg->stats.aead.encrypt_tlen);  in crypto_stats_aead_encrypt()
1090 	crypto_alg_put(alg);  in crypto_stats_aead_encrypt()
1094 void crypto_stats_aead_decrypt(unsigned int cryptlen, struct crypto_alg *alg,  in crypto_stats_aead_decrypt()  argument
1098 		atomic64_inc(&alg->stats.aead.err_cnt);  in crypto_stats_aead_decrypt()
1100 		atomic64_inc(&alg->stats.aead.decrypt_cnt);  in crypto_stats_aead_decrypt()
1101 		atomic64_add(cryptlen, &alg->stats.aead.decrypt_tlen);  in crypto_stats_aead_decrypt()
1103 	crypto_alg_put(alg);  in crypto_stats_aead_decrypt()
1108 				   struct crypto_alg *alg)  in crypto_stats_akcipher_encrypt()  argument
1111 		atomic64_inc(&alg->stats.akcipher.err_cnt);  in crypto_stats_akcipher_encrypt()
1113 		atomic64_inc(&alg->stats.akcipher.encrypt_cnt);  in crypto_stats_akcipher_encrypt()
1114 		atomic64_add(src_len, &alg->stats.akcipher.encrypt_tlen);  in crypto_stats_akcipher_encrypt()
1116 	crypto_alg_put(alg);  in crypto_stats_akcipher_encrypt()
1121 				   struct crypto_alg *alg)  in crypto_stats_akcipher_decrypt()  argument
1124 		atomic64_inc(&alg->stats.akcipher.err_cnt);  in crypto_stats_akcipher_decrypt()
1126 		atomic64_inc(&alg->stats.akcipher.decrypt_cnt);  in crypto_stats_akcipher_decrypt()
1127 		atomic64_add(src_len, &alg->stats.akcipher.decrypt_tlen);  in crypto_stats_akcipher_decrypt()
1129 	crypto_alg_put(alg);  in crypto_stats_akcipher_decrypt()
1133 void crypto_stats_akcipher_sign(int ret, struct crypto_alg *alg)  in crypto_stats_akcipher_sign()  argument
1136 		atomic64_inc(&alg->stats.akcipher.err_cnt);  in crypto_stats_akcipher_sign()
1138 		atomic64_inc(&alg->stats.akcipher.sign_cnt);  in crypto_stats_akcipher_sign()
1139 	crypto_alg_put(alg);  in crypto_stats_akcipher_sign()
1143 void crypto_stats_akcipher_verify(int ret, struct crypto_alg *alg)  in crypto_stats_akcipher_verify()  argument
1146 		atomic64_inc(&alg->stats.akcipher.err_cnt);  in crypto_stats_akcipher_verify()
1148 		atomic64_inc(&alg->stats.akcipher.verify_cnt);  in crypto_stats_akcipher_verify()
1149 	crypto_alg_put(alg);  in crypto_stats_akcipher_verify()
1153 void crypto_stats_compress(unsigned int slen, int ret, struct crypto_alg *alg)  in crypto_stats_compress()  argument
1156 		atomic64_inc(&alg->stats.compress.err_cnt);  in crypto_stats_compress()
1158 		atomic64_inc(&alg->stats.compress.compress_cnt);  in crypto_stats_compress()
1159 		atomic64_add(slen, &alg->stats.compress.compress_tlen);  in crypto_stats_compress()
1161 	crypto_alg_put(alg);  in crypto_stats_compress()
1165 void crypto_stats_decompress(unsigned int slen, int ret, struct crypto_alg *alg)  in crypto_stats_decompress()  argument
1168 		atomic64_inc(&alg->stats.compress.err_cnt);  in crypto_stats_decompress()
1170 		atomic64_inc(&alg->stats.compress.decompress_cnt);  in crypto_stats_decompress()
1171 		atomic64_add(slen, &alg->stats.compress.decompress_tlen);  in crypto_stats_decompress()
1173 	crypto_alg_put(alg);  in crypto_stats_decompress()
1178 			       struct crypto_alg *alg)  in crypto_stats_ahash_update()  argument
1181 		atomic64_inc(&alg->stats.hash.err_cnt);  in crypto_stats_ahash_update()
1183 		atomic64_add(nbytes, &alg->stats.hash.hash_tlen);  in crypto_stats_ahash_update()
1184 	crypto_alg_put(alg);  in crypto_stats_ahash_update()
1189 			      struct crypto_alg *alg)  in crypto_stats_ahash_final()  argument
1192 		atomic64_inc(&alg->stats.hash.err_cnt);  in crypto_stats_ahash_final()
1194 		atomic64_inc(&alg->stats.hash.hash_cnt);  in crypto_stats_ahash_final()
1195 		atomic64_add(nbytes, &alg->stats.hash.hash_tlen);  in crypto_stats_ahash_final()
1197 	crypto_alg_put(alg);  in crypto_stats_ahash_final()
1201 void crypto_stats_kpp_set_secret(struct crypto_alg *alg, int ret)  in crypto_stats_kpp_set_secret()  argument
1204 		atomic64_inc(&alg->stats.kpp.err_cnt);  in crypto_stats_kpp_set_secret()
1206 		atomic64_inc(&alg->stats.kpp.setsecret_cnt);  in crypto_stats_kpp_set_secret()
1207 	crypto_alg_put(alg);  in crypto_stats_kpp_set_secret()
1211 void crypto_stats_kpp_generate_public_key(struct crypto_alg *alg, int ret)  in crypto_stats_kpp_generate_public_key()  argument
1214 		atomic64_inc(&alg->stats.kpp.err_cnt);  in crypto_stats_kpp_generate_public_key()
1216 		atomic64_inc(&alg->stats.kpp.generate_public_key_cnt);  in crypto_stats_kpp_generate_public_key()
1217 	crypto_alg_put(alg);  in crypto_stats_kpp_generate_public_key()
1221 void crypto_stats_kpp_compute_shared_secret(struct crypto_alg *alg, int ret)  in crypto_stats_kpp_compute_shared_secret()  argument
1224 		atomic64_inc(&alg->stats.kpp.err_cnt);  in crypto_stats_kpp_compute_shared_secret()
1226 		atomic64_inc(&alg->stats.kpp.compute_shared_secret_cnt);  in crypto_stats_kpp_compute_shared_secret()
1227 	crypto_alg_put(alg);  in crypto_stats_kpp_compute_shared_secret()
1231 void crypto_stats_rng_seed(struct crypto_alg *alg, int ret)  in crypto_stats_rng_seed()  argument
1234 		atomic64_inc(&alg->stats.rng.err_cnt);  in crypto_stats_rng_seed()
1236 		atomic64_inc(&alg->stats.rng.seed_cnt);  in crypto_stats_rng_seed()
1237 	crypto_alg_put(alg);  in crypto_stats_rng_seed()
1241 void crypto_stats_rng_generate(struct crypto_alg *alg, unsigned int dlen,  in crypto_stats_rng_generate()  argument
1245 		atomic64_inc(&alg->stats.rng.err_cnt);  in crypto_stats_rng_generate()
1247 		atomic64_inc(&alg->stats.rng.generate_cnt);  in crypto_stats_rng_generate()
1248 		atomic64_add(dlen, &alg->stats.rng.generate_tlen);  in crypto_stats_rng_generate()
1250 	crypto_alg_put(alg);  in crypto_stats_rng_generate()
1255 				   struct crypto_alg *alg)  in crypto_stats_skcipher_encrypt()  argument
1258 		atomic64_inc(&alg->stats.cipher.err_cnt);  in crypto_stats_skcipher_encrypt()
1260 		atomic64_inc(&alg->stats.cipher.encrypt_cnt);  in crypto_stats_skcipher_encrypt()
1261 		atomic64_add(cryptlen, &alg->stats.cipher.encrypt_tlen);  in crypto_stats_skcipher_encrypt()
1263 	crypto_alg_put(alg);  in crypto_stats_skcipher_encrypt()
1268 				   struct crypto_alg *alg)  in crypto_stats_skcipher_decrypt()  argument
1271 		atomic64_inc(&alg->stats.cipher.err_cnt);  in crypto_stats_skcipher_decrypt()
1273 		atomic64_inc(&alg->stats.cipher.decrypt_cnt);  in crypto_stats_skcipher_decrypt()
1274 		atomic64_add(cryptlen, &alg->stats.cipher.decrypt_tlen);  in crypto_stats_skcipher_decrypt()
1276 	crypto_alg_put(alg);  in crypto_stats_skcipher_decrypt()