41 	struct blk_crypto_profile *profile;  member
44 static inline void blk_crypto_hw_enter(struct blk_crypto_profile *profile)  in blk_crypto_hw_enter()  argument
51 	if (profile->dev)  in blk_crypto_hw_enter()
52 		pm_runtime_get_sync(profile->dev);  in blk_crypto_hw_enter()
53 	down_write(&profile->lock);  in blk_crypto_hw_enter()
56 static inline void blk_crypto_hw_exit(struct blk_crypto_profile *profile)  in blk_crypto_hw_exit()  argument
58 	up_write(&profile->lock);  in blk_crypto_hw_exit()
59 	if (profile->dev)  in blk_crypto_hw_exit()
60 		pm_runtime_put_sync(profile->dev);  in blk_crypto_hw_exit()
73 int blk_crypto_profile_init(struct blk_crypto_profile *profile,  in blk_crypto_profile_init()  argument
80 	memset(profile, 0, sizeof(*profile));  in blk_crypto_profile_init()
88 	lockdep_register_key(&profile->lockdep_key);  in blk_crypto_profile_init()
89 	__init_rwsem(&profile->lock, "&profile->lock", &profile->lockdep_key);  in blk_crypto_profile_init()
91 	init_rwsem(&profile->lock);  in blk_crypto_profile_init()
99 	profile->slots = kvcalloc(num_slots, sizeof(profile->slots[0]),  in blk_crypto_profile_init()
101 	if (!profile->slots)  in blk_crypto_profile_init()
104 	profile->num_slots = num_slots;  in blk_crypto_profile_init()
106 	init_waitqueue_head(&profile->idle_slots_wait_queue);  in blk_crypto_profile_init()
107 	INIT_LIST_HEAD(&profile->idle_slots);  in blk_crypto_profile_init()
110 		profile->slots[slot].profile = profile;  in blk_crypto_profile_init()
111 		list_add_tail(&profile->slots[slot].idle_slot_node,  in blk_crypto_profile_init()
112 			      &profile->idle_slots);  in blk_crypto_profile_init()
115 	spin_lock_init(&profile->idle_slots_lock);  in blk_crypto_profile_init()
125 	profile->log_slot_ht_size = ilog2(slot_hashtable_size);  in blk_crypto_profile_init()
126 	profile->slot_hashtable =  in blk_crypto_profile_init()
128 			       sizeof(profile->slot_hashtable[0]), GFP_KERNEL);  in blk_crypto_profile_init()
129 	if (!profile->slot_hashtable)  in blk_crypto_profile_init()
132 		INIT_HLIST_HEAD(&profile->slot_hashtable[i]);  in blk_crypto_profile_init()
137 	blk_crypto_profile_destroy(profile);  in blk_crypto_profile_init()
142 static void blk_crypto_profile_destroy_callback(void *profile)  in blk_crypto_profile_destroy_callback()  argument
144 	blk_crypto_profile_destroy(profile);  in blk_crypto_profile_destroy_callback()
159 				 struct blk_crypto_profile *profile,  in devm_blk_crypto_profile_init()  argument
162 	int err = blk_crypto_profile_init(profile, num_slots);  in devm_blk_crypto_profile_init()
169 					profile);  in devm_blk_crypto_profile_init()
174 blk_crypto_hash_bucket_for_key(struct blk_crypto_profile *profile,  in blk_crypto_hash_bucket_for_key()  argument
177 	return &profile->slot_hashtable[  in blk_crypto_hash_bucket_for_key()
178 			hash_ptr(key, profile->log_slot_ht_size)];  in blk_crypto_hash_bucket_for_key()
184 	struct blk_crypto_profile *profile = slot->profile;  in blk_crypto_remove_slot_from_lru_list()  local
187 	spin_lock_irqsave(&profile->idle_slots_lock, flags);  in blk_crypto_remove_slot_from_lru_list()
189 	spin_unlock_irqrestore(&profile->idle_slots_lock, flags);  in blk_crypto_remove_slot_from_lru_list()
193 blk_crypto_find_keyslot(struct blk_crypto_profile *profile,  in blk_crypto_find_keyslot()  argument
197 		blk_crypto_hash_bucket_for_key(profile, key);  in blk_crypto_find_keyslot()
208 blk_crypto_find_and_grab_keyslot(struct blk_crypto_profile *profile,  in blk_crypto_find_and_grab_keyslot()  argument
213 	slot = blk_crypto_find_keyslot(profile, key);  in blk_crypto_find_and_grab_keyslot()
231 	return slot - slot->profile->slots;  in blk_crypto_keyslot_index()
252 blk_status_t blk_crypto_get_keyslot(struct blk_crypto_profile *profile,  in blk_crypto_get_keyslot()  argument
266 	if (profile->num_slots == 0)  in blk_crypto_get_keyslot()
269 	down_read(&profile->lock);  in blk_crypto_get_keyslot()
270 	slot = blk_crypto_find_and_grab_keyslot(profile, key);  in blk_crypto_get_keyslot()
271 	up_read(&profile->lock);  in blk_crypto_get_keyslot()
276 		blk_crypto_hw_enter(profile);  in blk_crypto_get_keyslot()
277 		slot = blk_crypto_find_and_grab_keyslot(profile, key);  in blk_crypto_get_keyslot()
279 			blk_crypto_hw_exit(profile);  in blk_crypto_get_keyslot()
287 		if (!list_empty(&profile->idle_slots))  in blk_crypto_get_keyslot()
290 		blk_crypto_hw_exit(profile);  in blk_crypto_get_keyslot()
291 		wait_event(profile->idle_slots_wait_queue,  in blk_crypto_get_keyslot()
292 			   !list_empty(&profile->idle_slots));  in blk_crypto_get_keyslot()
295 	slot = list_first_entry(&profile->idle_slots, struct blk_crypto_keyslot,  in blk_crypto_get_keyslot()
299 	err = profile->ll_ops.keyslot_program(profile, key, slot_idx);  in blk_crypto_get_keyslot()
301 		wake_up(&profile->idle_slots_wait_queue);  in blk_crypto_get_keyslot()
302 		blk_crypto_hw_exit(profile);  in blk_crypto_get_keyslot()
311 		       blk_crypto_hash_bucket_for_key(profile, key));  in blk_crypto_get_keyslot()
317 	blk_crypto_hw_exit(profile);  in blk_crypto_get_keyslot()
331 	struct blk_crypto_profile *profile;  in blk_crypto_put_keyslot()  local
337 	profile = slot->profile;  in blk_crypto_put_keyslot()
340 					&profile->idle_slots_lock, flags)) {  in blk_crypto_put_keyslot()
341 		list_add_tail(&slot->idle_slot_node, &profile->idle_slots);  in blk_crypto_put_keyslot()
342 		spin_unlock_irqrestore(&profile->idle_slots_lock, flags);  in blk_crypto_put_keyslot()
343 		wake_up(&profile->idle_slots_wait_queue);  in blk_crypto_put_keyslot()
355 bool __blk_crypto_cfg_supported(struct blk_crypto_profile *profile,  in __blk_crypto_cfg_supported()  argument
358 	if (!profile)  in __blk_crypto_cfg_supported()
360 	if (!(profile->modes_supported[cfg->crypto_mode] & cfg->data_unit_size))  in __blk_crypto_cfg_supported()
362 	if (profile->max_dun_bytes_supported < cfg->dun_bytes)  in __blk_crypto_cfg_supported()
364 	if (!(profile->key_types_supported & cfg->key_type))  in __blk_crypto_cfg_supported()
374 int __blk_crypto_evict_key(struct blk_crypto_profile *profile,  in __blk_crypto_evict_key()  argument
380 	if (profile->num_slots == 0) {  in __blk_crypto_evict_key()
381 		if (profile->ll_ops.keyslot_evict) {  in __blk_crypto_evict_key()
382 			blk_crypto_hw_enter(profile);  in __blk_crypto_evict_key()
383 			err = profile->ll_ops.keyslot_evict(profile, key, -1);  in __blk_crypto_evict_key()
384 			blk_crypto_hw_exit(profile);  in __blk_crypto_evict_key()
390 	blk_crypto_hw_enter(profile);  in __blk_crypto_evict_key()
391 	slot = blk_crypto_find_keyslot(profile, key);  in __blk_crypto_evict_key()
406 	err = profile->ll_ops.keyslot_evict(profile, key,  in __blk_crypto_evict_key()
416 	blk_crypto_hw_exit(profile);  in __blk_crypto_evict_key()
429 void blk_crypto_reprogram_all_keys(struct blk_crypto_profile *profile)  in blk_crypto_reprogram_all_keys()  argument
433 	if (profile->num_slots == 0)  in blk_crypto_reprogram_all_keys()
437 	down_write(&profile->lock);  in blk_crypto_reprogram_all_keys()
438 	for (slot = 0; slot < profile->num_slots; slot++) {  in blk_crypto_reprogram_all_keys()
439 		const struct blk_crypto_key *key = profile->slots[slot].key;  in blk_crypto_reprogram_all_keys()
445 		err = profile->ll_ops.keyslot_program(profile, key, slot);  in blk_crypto_reprogram_all_keys()
448 	up_write(&profile->lock);  in blk_crypto_reprogram_all_keys()
452 void blk_crypto_profile_destroy(struct blk_crypto_profile *profile)  in blk_crypto_profile_destroy()  argument
454 	if (!profile)  in blk_crypto_profile_destroy()
457 	lockdep_unregister_key(&profile->lockdep_key);  in blk_crypto_profile_destroy()
459 	kvfree(profile->slot_hashtable);  in blk_crypto_profile_destroy()
460 	kvfree_sensitive(profile->slots,  in blk_crypto_profile_destroy()
461 			 sizeof(profile->slots[0]) * profile->num_slots);  in blk_crypto_profile_destroy()
462 	memzero_explicit(profile, sizeof(*profile));  in blk_crypto_profile_destroy()
466 bool blk_crypto_register(struct blk_crypto_profile *profile,  in blk_crypto_register()  argument
473 	q->crypto_profile = profile;  in blk_crypto_register()
499 	struct blk_crypto_profile *profile =  in blk_crypto_derive_sw_secret()  local
503 	if (!profile)  in blk_crypto_derive_sw_secret()
505 	if (!(profile->key_types_supported & BLK_CRYPTO_KEY_TYPE_HW_WRAPPED))  in blk_crypto_derive_sw_secret()
507 	if (!profile->ll_ops.derive_sw_secret)  in blk_crypto_derive_sw_secret()
509 	blk_crypto_hw_enter(profile);  in blk_crypto_derive_sw_secret()
510 	err = profile->ll_ops.derive_sw_secret(profile, eph_key, eph_key_size,  in blk_crypto_derive_sw_secret()
512 	blk_crypto_hw_exit(profile);  in blk_crypto_derive_sw_secret()