Lines Matching refs:aead
206 struct tipc_aead __rcu *aead[KEY_MAX + 1]; member
240 struct tipc_aead *aead; member
247 struct tipc_aead *aead; member
251 static struct tipc_aead *tipc_aead_get(struct tipc_aead __rcu *aead);
252 static inline void tipc_aead_put(struct tipc_aead *aead);
254 static int tipc_aead_users(struct tipc_aead __rcu *aead);
255 static void tipc_aead_users_inc(struct tipc_aead __rcu *aead, int lim);
256 static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim);
257 static void tipc_aead_users_set(struct tipc_aead __rcu *aead, int val);
258 static struct crypto_aead *tipc_aead_tfm_next(struct tipc_aead *aead);
259 static int tipc_aead_init(struct tipc_aead **aead, struct tipc_aead_key *ukey,
266 static int tipc_aead_encrypt(struct tipc_aead *aead, struct sk_buff *skb,
271 static int tipc_aead_decrypt(struct net *net, struct tipc_aead *aead,
275 static int tipc_ehdr_build(struct net *net, struct tipc_aead *aead,
283 struct tipc_aead *aead, u8 pos,
296 static void tipc_crypto_rcv_complete(struct net *net, struct tipc_aead *aead,
383 static struct tipc_aead *tipc_aead_get(struct tipc_aead __rcu *aead) in tipc_aead_get() argument
388 tmp = rcu_dereference(aead); in tipc_aead_get()
396 static inline void tipc_aead_put(struct tipc_aead *aead) in tipc_aead_put() argument
398 if (aead && refcount_dec_and_test(&aead->refcnt)) in tipc_aead_put()
399 call_rcu(&aead->rcu, tipc_aead_free); in tipc_aead_put()
408 struct tipc_aead *aead = container_of(rp, struct tipc_aead, rcu); in tipc_aead_free() local
411 if (aead->cloned) { in tipc_aead_free()
412 tipc_aead_put(aead->cloned); in tipc_aead_free()
414 head = *get_cpu_ptr(aead->tfm_entry); in tipc_aead_free()
415 put_cpu_ptr(aead->tfm_entry); in tipc_aead_free()
426 free_percpu(aead->tfm_entry); in tipc_aead_free()
427 kfree_sensitive(aead->key); in tipc_aead_free()
428 kfree(aead); in tipc_aead_free()
431 static int tipc_aead_users(struct tipc_aead __rcu *aead) in tipc_aead_users() argument
437 tmp = rcu_dereference(aead); in tipc_aead_users()
445 static void tipc_aead_users_inc(struct tipc_aead __rcu *aead, int lim) in tipc_aead_users_inc() argument
450 tmp = rcu_dereference(aead); in tipc_aead_users_inc()
456 static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim) in tipc_aead_users_dec() argument
461 tmp = rcu_dereference(aead); in tipc_aead_users_dec()
463 atomic_add_unless(&rcu_dereference(aead)->users, -1, lim); in tipc_aead_users_dec()
467 static void tipc_aead_users_set(struct tipc_aead __rcu *aead, int val) in tipc_aead_users_set() argument
473 tmp = rcu_dereference(aead); in tipc_aead_users_set()
488 static struct crypto_aead *tipc_aead_tfm_next(struct tipc_aead *aead) in tipc_aead_tfm_next() argument
493 tfm_entry = get_cpu_ptr(aead->tfm_entry); in tipc_aead_tfm_next()
514 static int tipc_aead_init(struct tipc_aead **aead, struct tipc_aead_key *ukey, in tipc_aead_init() argument
523 if (unlikely(*aead)) in tipc_aead_init()
609 *aead = tmp; in tipc_aead_init()
628 struct tipc_aead *aead; in tipc_aead_clone() local
640 aead = kzalloc(sizeof(*aead), GFP_ATOMIC); in tipc_aead_clone()
641 if (unlikely(!aead)) in tipc_aead_clone()
644 aead->tfm_entry = alloc_percpu_gfp(struct tipc_tfm *, GFP_ATOMIC); in tipc_aead_clone()
645 if (unlikely(!aead->tfm_entry)) { in tipc_aead_clone()
646 kfree_sensitive(aead); in tipc_aead_clone()
651 *per_cpu_ptr(aead->tfm_entry, cpu) = in tipc_aead_clone()
655 memcpy(aead->hint, src->hint, sizeof(src->hint)); in tipc_aead_clone()
656 aead->mode = src->mode; in tipc_aead_clone()
657 aead->salt = src->salt; in tipc_aead_clone()
658 aead->authsize = src->authsize; in tipc_aead_clone()
659 atomic_set(&aead->users, 0); in tipc_aead_clone()
660 atomic64_set(&aead->seqno, 0); in tipc_aead_clone()
661 refcount_set(&aead->refcnt, 1); in tipc_aead_clone()
664 aead->cloned = src; in tipc_aead_clone()
666 *dst = aead; in tipc_aead_clone()
732 static int tipc_aead_encrypt(struct tipc_aead *aead, struct sk_buff *skb, in tipc_aead_encrypt() argument
737 struct crypto_aead *tfm = tipc_aead_tfm_next(aead); in tipc_aead_encrypt()
750 tailen = len - skb->len + aead->authsize; in tipc_aead_encrypt()
803 salt = aead->salt; in tipc_aead_encrypt()
804 if (aead->mode == CLUSTER_KEY) in tipc_aead_encrypt()
821 tx_ctx->aead = aead; in tipc_aead_encrypt()
849 struct tipc_aead *aead = tx_ctx->aead; in tipc_aead_encrypt_done() local
850 struct tipc_crypto *tx = aead->crypto; in tipc_aead_encrypt_done()
873 tipc_aead_put(aead); in tipc_aead_encrypt_done()
888 static int tipc_aead_decrypt(struct net *net, struct tipc_aead *aead, in tipc_aead_decrypt() argument
902 if (unlikely(!aead)) in tipc_aead_decrypt()
912 tfm = tipc_aead_tfm_next(aead); in tipc_aead_decrypt()
928 salt = aead->salt; in tipc_aead_decrypt()
929 if (aead->mode == CLUSTER_KEY) in tipc_aead_decrypt()
946 rx_ctx->aead = aead; in tipc_aead_decrypt()
973 struct tipc_aead *aead = rx_ctx->aead; in tipc_aead_decrypt_done() local
974 struct tipc_crypto_stats __percpu *stats = aead->crypto->stats; in tipc_aead_decrypt_done()
975 struct net *net = aead->crypto->net; in tipc_aead_decrypt_done()
989 tipc_crypto_rcv_complete(net, aead, b, &skb, err); in tipc_aead_decrypt_done()
1041 static int tipc_ehdr_build(struct net *net, struct tipc_aead *aead, in tipc_ehdr_build() argument
1060 if (!__rx || aead->mode == CLUSTER_KEY) in tipc_ehdr_build()
1061 seqno = atomic64_inc_return(&aead->seqno); in tipc_ehdr_build()
1080 ehdr->master_key = aead->crypto->key_master; in tipc_ehdr_build()
1133 struct tipc_aead *aead = NULL; in tipc_crypto_key_init() local
1137 rc = tipc_aead_init(&aead, ukey, mode); in tipc_crypto_key_init()
1141 rc = tipc_crypto_key_attach(c, aead, 0, master_key); in tipc_crypto_key_init()
1143 tipc_aead_free(&aead->rcu); in tipc_crypto_key_init()
1159 struct tipc_aead *aead, u8 pos, in tipc_crypto_key_attach() argument
1175 if (tipc_aead_users(c->aead[key.pending]) > 0) in tipc_crypto_key_attach()
1197 aead->crypto = c; in tipc_crypto_key_attach()
1198 aead->gen = (is_tx(c)) ? ++c->key_gen : c->key_gen; in tipc_crypto_key_attach()
1199 tipc_aead_rcu_replace(c->aead[new_key], aead, &c->lock); in tipc_crypto_key_attach()
1232 tipc_aead_users_dec(tx->aead[k], 0); in tipc_crypto_key_flush()
1241 tipc_crypto_key_detach(c->aead[k], &c->lock); in tipc_crypto_key_flush()
1277 if (tipc_aead_users(rx->aead[key.pending]) > 0) in tipc_crypto_key_try_align()
1281 tmp1 = tipc_aead_rcu_ptr(rx->aead[key.pending], &rx->lock); in tipc_crypto_key_try_align()
1284 rcu_assign_pointer(rx->aead[key.pending], NULL); in tipc_crypto_key_try_align()
1288 tmp2 = rcu_replace_pointer(rx->aead[key.passive], tmp2, lockdep_is_held(&rx->lock)); in tipc_crypto_key_try_align()
1295 rcu_assign_pointer(rx->aead[new_pending], tmp1); in tipc_crypto_key_try_align()
1297 rcu_assign_pointer(rx->aead[new_passive], tmp2); in tipc_crypto_key_try_align()
1327 struct tipc_aead *aead = NULL; in tipc_crypto_key_pick_tx() local
1344 aead = tipc_aead_rcu_ptr(tx->aead[KEY_MASTER], &tx->lock); in tipc_crypto_key_pick_tx()
1352 aead = tipc_aead_rcu_ptr(tx->aead[k], &tx->lock); in tipc_crypto_key_pick_tx()
1353 if (!aead) in tipc_crypto_key_pick_tx()
1355 if (aead->mode != CLUSTER_KEY || in tipc_crypto_key_pick_tx()
1356 aead == skb_cb->tx_clone_ctx.last) { in tipc_crypto_key_pick_tx()
1357 aead = NULL; in tipc_crypto_key_pick_tx()
1361 skb_cb->tx_clone_ctx.last = aead; in tipc_crypto_key_pick_tx()
1370 if (likely(aead)) in tipc_crypto_key_pick_tx()
1371 WARN_ON(!refcount_inc_not_zero(&aead->refcnt)); in tipc_crypto_key_pick_tx()
1374 return aead; in tipc_crypto_key_pick_tx()
1437 tipc_aead_users_inc(tx->aead[new], INT_MAX); in tipc_crypto_key_synch()
1439 tipc_aead_users_dec(tx->aead[cur], 0); in tipc_crypto_key_synch()
1461 tipc_crypto_key_detach(tx->aead[key.active], &tx->lock); in tipc_crypto_key_revoke()
1542 tipc_aead_put(rcu_dereference(c->aead[k])); in tipc_crypto_stop()
1563 if (key.active && tipc_aead_users(tx->aead[key.active]) > 0) in tipc_crypto_timeout()
1565 if (!key.pending || tipc_aead_users(tx->aead[key.pending]) <= 0) in tipc_crypto_timeout()
1572 tipc_crypto_key_detach(tx->aead[key.active], &tx->lock); in tipc_crypto_timeout()
1582 if (!key.pending || tipc_aead_users(rx->aead[key.pending]) <= 0) in tipc_crypto_timeout()
1596 if (!key.pending || tipc_aead_users(rx->aead[key.pending]) > -10) in tipc_crypto_timeout()
1600 tipc_crypto_key_detach(rx->aead[key.pending], &rx->lock); in tipc_crypto_timeout()
1609 tipc_aead_users(rx->aead[key.active]) > 0) in tipc_crypto_timeout()
1618 tipc_aead_users_set(rx->aead[key.pending], 0); in tipc_crypto_timeout()
1627 tipc_aead_users(rx->aead[key.passive]) > -10) in tipc_crypto_timeout()
1631 tipc_crypto_key_detach(rx->aead[key.passive], &rx->lock); in tipc_crypto_timeout()
1700 struct tipc_aead *aead = NULL; in tipc_crypto_xmit() local
1762 aead = tipc_aead_get(tx->aead[tx_key]); in tipc_crypto_xmit()
1763 if (unlikely(!aead)) in tipc_crypto_xmit()
1765 rc = tipc_ehdr_build(net, aead, tx_key, *skb, __rx); in tipc_crypto_xmit()
1767 rc = tipc_aead_encrypt(aead, *skb, b, dst, __dnode); in tipc_crypto_xmit()
1790 tipc_aead_put(aead); in tipc_crypto_xmit()
1821 struct tipc_aead *aead = NULL; in tipc_crypto_rcv() local
1846 aead = tipc_crypto_key_pick_tx(tx, rx, *skb, tx_key); in tipc_crypto_rcv()
1847 if (aead) in tipc_crypto_rcv()
1853 if (!aead) in tipc_crypto_rcv()
1854 aead = tipc_aead_get(rx->aead[tx_key]); in tipc_crypto_rcv()
1855 rc = tipc_aead_decrypt(net, aead, *skb, b); in tipc_crypto_rcv()
1881 rcu_access_pointer(rx->aead[n])); in tipc_crypto_rcv()
1895 tipc_crypto_rcv_complete(net, aead, b, skb, rc); in tipc_crypto_rcv()
1899 static void tipc_crypto_rcv_complete(struct net *net, struct tipc_aead *aead, in tipc_crypto_rcv_complete() argument
1904 struct tipc_crypto *rx = aead->crypto; in tipc_crypto_rcv_complete()
1910 if (unlikely(is_tx(aead->crypto))) { in tipc_crypto_rcv_complete()
1913 (rx) ? tipc_node_get_id_str(rx->node) : "-", err, aead, in tipc_crypto_rcv_complete()
1917 aead->crypto->aead[1], aead->crypto->aead[2], in tipc_crypto_rcv_complete()
1918 aead->crypto->aead[3]); in tipc_crypto_rcv_complete()
1942 if (tipc_aead_clone(&tmp, aead) < 0) in tipc_crypto_rcv_complete()
1949 tipc_aead_put(aead); in tipc_crypto_rcv_complete()
1950 aead = tmp; in tipc_crypto_rcv_complete()
1954 tipc_aead_users_dec((struct tipc_aead __force __rcu *)aead, INT_MIN); in tipc_crypto_rcv_complete()
1959 tipc_aead_users_set((struct tipc_aead __force __rcu *)aead, 1); in tipc_crypto_rcv_complete()
1974 if (pskb_trim(*skb, (*skb)->len - aead->authsize)) in tipc_crypto_rcv_complete()
2004 tipc_aead_put(aead); in tipc_crypto_rcv_complete()
2085 struct tipc_aead *aead; in tipc_crypto_key_dump() local
2111 aead = rcu_dereference(c->aead[k]); in tipc_crypto_key_dump()
2112 if (aead) in tipc_crypto_key_dump()
2115 aead->hint, in tipc_crypto_key_dump()
2116 (aead->mode == CLUSTER_KEY) ? "c" : "p", in tipc_crypto_key_dump()
2117 atomic_read(&aead->users), in tipc_crypto_key_dump()
2118 refcount_read(&aead->refcnt)); in tipc_crypto_key_dump()
2205 struct tipc_aead *aead; in tipc_crypto_key_distr() local
2214 aead = tipc_aead_get(tx->aead[key]); in tipc_crypto_key_distr()
2215 if (likely(aead)) { in tipc_crypto_key_distr()
2216 rc = tipc_crypto_key_xmit(tx->net, aead->key, in tipc_crypto_key_distr()
2217 aead->gen, aead->mode, in tipc_crypto_key_distr()
2219 tipc_aead_put(aead); in tipc_crypto_key_distr()
2452 struct tipc_aead *aead; in tipc_crypto_work_tx() local
2460 aead = rcu_dereference(tx->aead[key.active ?: KEY_MASTER]); in tipc_crypto_work_tx()
2461 if (unlikely(!aead)) { in tipc_crypto_work_tx()
2468 skey = kmemdup(aead->key, tipc_aead_key_size(aead->key), GFP_ATOMIC); in tipc_crypto_work_tx()