Lines Matching +full:p1 +full:- +full:retry +full:- +full:params
1 // SPDX-License-Identifier: GPL-2.0-only
12 * Split up af-specific portion
80 * +---- root_d: sorted by daddr:prefix
84 * | +- root: sorted by saddr/prefix
92 * | +- coarse policies and all any:daddr policies
94 * +---- root_s: sorted by saddr:prefix
102 * +---- coarse policies and all any:any policies
105 * 1. any:any list from top-level xfrm_pol_inexact_bin
194 return refcount_inc_not_zero(&policy->refcnt); in xfrm_pol_hold_rcu()
200 const struct flowi4 *fl4 = &fl->u.ip4; in __xfrm4_selector_match()
202 return addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) && in __xfrm4_selector_match()
203 addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) && in __xfrm4_selector_match()
204 !((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) && in __xfrm4_selector_match()
205 !((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) && in __xfrm4_selector_match()
206 (fl4->flowi4_proto == sel->proto || !sel->proto) && in __xfrm4_selector_match()
207 (fl4->flowi4_oif == sel->ifindex || !sel->ifindex); in __xfrm4_selector_match()
213 const struct flowi6 *fl6 = &fl->u.ip6; in __xfrm6_selector_match()
215 return addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) && in __xfrm6_selector_match()
216 addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) && in __xfrm6_selector_match()
217 !((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) && in __xfrm6_selector_match()
218 !((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) && in __xfrm6_selector_match()
219 (fl6->flowi6_proto == sel->proto || !sel->proto) && in __xfrm6_selector_match()
220 (fl6->flowi6_oif == sel->ifindex || !sel->ifindex); in __xfrm6_selector_match()
255 const struct xfrm_dst_lookup_params *params) in __xfrm_dst_lookup() argument
262 return ERR_PTR(-EAFNOSUPPORT); in __xfrm_dst_lookup()
264 dst = afinfo->dst_lookup(params); in __xfrm_dst_lookup()
278 struct xfrm_dst_lookup_params params; in xfrm_dst_lookup() local
280 xfrm_address_t *saddr = &x->props.saddr; in xfrm_dst_lookup()
281 xfrm_address_t *daddr = &x->id.daddr; in xfrm_dst_lookup()
284 if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) { in xfrm_dst_lookup()
285 saddr = x->coaddr; in xfrm_dst_lookup()
288 if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) { in xfrm_dst_lookup()
290 daddr = x->coaddr; in xfrm_dst_lookup()
293 params.net = net; in xfrm_dst_lookup()
294 params.saddr = saddr; in xfrm_dst_lookup()
295 params.daddr = daddr; in xfrm_dst_lookup()
296 params.tos = tos; in xfrm_dst_lookup()
297 params.oif = oif; in xfrm_dst_lookup()
298 params.mark = mark; in xfrm_dst_lookup()
299 params.ipproto = x->id.proto; in xfrm_dst_lookup()
300 if (x->encap) { in xfrm_dst_lookup()
301 switch (x->encap->encap_type) { in xfrm_dst_lookup()
303 params.ipproto = IPPROTO_UDP; in xfrm_dst_lookup()
304 params.uli.ports.sport = x->encap->encap_sport; in xfrm_dst_lookup()
305 params.uli.ports.dport = x->encap->encap_dport; in xfrm_dst_lookup()
308 params.ipproto = IPPROTO_TCP; in xfrm_dst_lookup()
309 params.uli.ports.sport = x->encap->encap_sport; in xfrm_dst_lookup()
310 params.uli.ports.dport = x->encap->encap_dport; in xfrm_dst_lookup()
315 dst = __xfrm_dst_lookup(family, ¶ms); in xfrm_dst_lookup()
329 if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ) in make_jiffies()
330 return MAX_SCHEDULE_TIMEOUT-1; in make_jiffies()
343 read_lock(&xp->lock); in xfrm_policy_timer()
345 if (unlikely(xp->walk.dead)) in xfrm_policy_timer()
348 dir = xfrm_policy_id2dir(xp->index); in xfrm_policy_timer()
350 if (xp->lft.hard_add_expires_seconds) { in xfrm_policy_timer()
351 time64_t tmo = xp->lft.hard_add_expires_seconds + in xfrm_policy_timer()
352 xp->curlft.add_time - now; in xfrm_policy_timer()
358 if (xp->lft.hard_use_expires_seconds) { in xfrm_policy_timer()
359 time64_t tmo = xp->lft.hard_use_expires_seconds + in xfrm_policy_timer()
360 (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now; in xfrm_policy_timer()
366 if (xp->lft.soft_add_expires_seconds) { in xfrm_policy_timer()
367 time64_t tmo = xp->lft.soft_add_expires_seconds + in xfrm_policy_timer()
368 xp->curlft.add_time - now; in xfrm_policy_timer()
376 if (xp->lft.soft_use_expires_seconds) { in xfrm_policy_timer()
377 time64_t tmo = xp->lft.soft_use_expires_seconds + in xfrm_policy_timer()
378 (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now; in xfrm_policy_timer()
390 !mod_timer(&xp->timer, jiffies + make_jiffies(next))) in xfrm_policy_timer()
394 read_unlock(&xp->lock); in xfrm_policy_timer()
399 read_unlock(&xp->lock); in xfrm_policy_timer()
416 write_pnet(&policy->xp_net, net); in xfrm_policy_alloc()
417 INIT_LIST_HEAD(&policy->walk.all); in xfrm_policy_alloc()
418 INIT_HLIST_NODE(&policy->bydst_inexact_list); in xfrm_policy_alloc()
419 INIT_HLIST_NODE(&policy->bydst); in xfrm_policy_alloc()
420 INIT_HLIST_NODE(&policy->byidx); in xfrm_policy_alloc()
421 rwlock_init(&policy->lock); in xfrm_policy_alloc()
422 refcount_set(&policy->refcnt, 1); in xfrm_policy_alloc()
423 skb_queue_head_init(&policy->polq.hold_queue); in xfrm_policy_alloc()
424 timer_setup(&policy->timer, xfrm_policy_timer, 0); in xfrm_policy_alloc()
425 timer_setup(&policy->polq.hold_timer, in xfrm_policy_alloc()
436 security_xfrm_policy_free(policy->security); in xfrm_policy_destroy_rcu()
444 BUG_ON(!policy->walk.dead); in xfrm_policy_destroy()
446 if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer)) in xfrm_policy_destroy()
450 call_rcu(&policy->rcu, xfrm_policy_destroy_rcu); in xfrm_policy_destroy()
462 write_lock_bh(&policy->lock); in xfrm_policy_kill()
463 policy->walk.dead = 1; in xfrm_policy_kill()
464 write_unlock_bh(&policy->lock); in xfrm_policy_kill()
466 atomic_inc(&policy->genid); in xfrm_policy_kill()
468 if (del_timer(&policy->polq.hold_timer)) in xfrm_policy_kill()
470 skb_queue_purge(&policy->polq.hold_queue); in xfrm_policy_kill()
472 if (del_timer(&policy->timer)) in xfrm_policy_kill()
482 return __idx_hash(index, net->xfrm.policy_idx_hmask); in idx_hash()
492 *dbits = net->xfrm.policy_bydst[dir].dbits4; in __get_hash_thresh()
493 *sbits = net->xfrm.policy_bydst[dir].sbits4; in __get_hash_thresh()
497 *dbits = net->xfrm.policy_bydst[dir].dbits6; in __get_hash_thresh()
498 *sbits = net->xfrm.policy_bydst[dir].sbits6; in __get_hash_thresh()
511 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; in policy_hash_bysel()
522 return rcu_dereference_check(net->xfrm.policy_bydst[dir].table, in policy_hash_bysel()
523 lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash; in policy_hash_bysel()
531 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; in policy_hash_direct()
539 return rcu_dereference_check(net->xfrm.policy_bydst[dir].table, in policy_hash_direct()
540 lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash; in policy_hash_direct()
559 __get_hash_thresh(net, pol->family, dir, &dbits, &sbits); in xfrm_dst_hash_transfer()
560 h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr, in xfrm_dst_hash_transfer()
561 pol->family, nhashmask, dbits, sbits); in xfrm_dst_hash_transfer()
562 if (!entry0 || pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { in xfrm_dst_hash_transfer()
563 hlist_del_rcu(&pol->bydst); in xfrm_dst_hash_transfer()
564 hlist_add_head_rcu(&pol->bydst, ndsttable + h); in xfrm_dst_hash_transfer()
569 hlist_del_rcu(&pol->bydst); in xfrm_dst_hash_transfer()
570 hlist_add_behind_rcu(&pol->bydst, entry0); in xfrm_dst_hash_transfer()
572 entry0 = &pol->bydst; in xfrm_dst_hash_transfer()
590 h = __idx_hash(pol->index, nhashmask); in xfrm_idx_hash_transfer()
591 hlist_add_head(&pol->byidx, nidxtable+h); in xfrm_idx_hash_transfer()
597 return ((old_hmask + 1) << 1) - 1; in xfrm_new_hash_mask()
602 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; in xfrm_bydst_resize()
612 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_bydst_resize()
613 write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); in xfrm_bydst_resize()
615 odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table, in xfrm_bydst_resize()
616 lockdep_is_held(&net->xfrm.xfrm_policy_lock)); in xfrm_bydst_resize()
618 for (i = hmask; i >= 0; i--) in xfrm_bydst_resize()
621 rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst); in xfrm_bydst_resize()
622 net->xfrm.policy_bydst[dir].hmask = nhashmask; in xfrm_bydst_resize()
624 write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation); in xfrm_bydst_resize()
625 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_bydst_resize()
634 unsigned int hmask = net->xfrm.policy_idx_hmask; in xfrm_byidx_resize()
637 struct hlist_head *oidx = net->xfrm.policy_byidx; in xfrm_byidx_resize()
644 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_byidx_resize()
646 for (i = hmask; i >= 0; i--) in xfrm_byidx_resize()
649 net->xfrm.policy_byidx = nidx; in xfrm_byidx_resize()
650 net->xfrm.policy_idx_hmask = nhashmask; in xfrm_byidx_resize()
652 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_byidx_resize()
659 unsigned int cnt = net->xfrm.policy_count[dir]; in xfrm_bydst_should_resize()
660 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; in xfrm_bydst_should_resize()
674 unsigned int hmask = net->xfrm.policy_idx_hmask; in xfrm_byidx_should_resize()
685 si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN]; in xfrm_spd_getinfo()
686 si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT]; in xfrm_spd_getinfo()
687 si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD]; in xfrm_spd_getinfo()
688 si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX]; in xfrm_spd_getinfo()
689 si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX]; in xfrm_spd_getinfo()
690 si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX]; in xfrm_spd_getinfo()
691 si->spdhcnt = net->xfrm.policy_idx_hmask; in xfrm_spd_getinfo()
692 si->spdhmcnt = xfrm_policy_hashmax; in xfrm_spd_getinfo()
724 .family = pol->family, in xfrm_policy_inexact_alloc_bin()
725 .type = pol->type, in xfrm_policy_inexact_alloc_bin()
727 .if_id = pol->if_id, in xfrm_policy_inexact_alloc_bin()
731 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_alloc_bin()
743 bin->k = k; in xfrm_policy_inexact_alloc_bin()
744 INIT_HLIST_HEAD(&bin->hhead); in xfrm_policy_inexact_alloc_bin()
745 bin->root_d = RB_ROOT; in xfrm_policy_inexact_alloc_bin()
746 bin->root_s = RB_ROOT; in xfrm_policy_inexact_alloc_bin()
747 seqcount_spinlock_init(&bin->count, &net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_alloc_bin()
750 &bin->k, &bin->head, in xfrm_policy_inexact_alloc_bin()
753 list_add(&bin->inexact_bins, &net->xfrm.inexact_bins); in xfrm_policy_inexact_alloc_bin()
784 addr = &policy->selector.saddr; in xfrm_policy_inexact_insert_use_any_list()
785 prefixlen = policy->selector.prefixlen_s; in xfrm_policy_inexact_insert_use_any_list()
788 policy->family, in xfrm_policy_inexact_insert_use_any_list()
790 addr = &policy->selector.daddr; in xfrm_policy_inexact_insert_use_any_list()
791 prefixlen = policy->selector.prefixlen_d; in xfrm_policy_inexact_insert_use_any_list()
793 policy->family, in xfrm_policy_inexact_insert_use_any_list()
801 node->addr = *addr; in xfrm_pol_inexact_node_init()
802 node->prefixlen = prefixlen; in xfrm_pol_inexact_node_init()
829 mask = ~0U << (32 - prefixlen); in xfrm_policy_addr_delta()
830 ma = ntohl(a->a4) & mask; in xfrm_policy_addr_delta()
831 mb = ntohl(b->a4) & mask; in xfrm_policy_addr_delta()
833 delta = -1; in xfrm_policy_addr_delta()
842 delta = memcmp(a->a6, b->a6, pdw << 2); in xfrm_policy_addr_delta()
847 mask = ~0U << (32 - pbi); in xfrm_policy_addr_delta()
848 ma = ntohl(a->a6[pdw]) & mask; in xfrm_policy_addr_delta()
849 mb = ntohl(b->a6[pdw]) & mask; in xfrm_policy_addr_delta()
851 delta = -1; in xfrm_policy_addr_delta()
873 list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) { in xfrm_policy_inexact_list_reinsert()
877 if (policy->walk.dead || !policy->bydst_reinsert) in xfrm_policy_inexact_list_reinsert()
880 WARN_ON_ONCE(policy->family != family); in xfrm_policy_inexact_list_reinsert()
882 policy->bydst_reinsert = false; in xfrm_policy_inexact_list_reinsert()
883 hlist_for_each_entry(p, &n->hhead, bydst) { in xfrm_policy_inexact_list_reinsert()
884 if (policy->priority > p->priority) in xfrm_policy_inexact_list_reinsert()
885 newpos = &p->bydst; in xfrm_policy_inexact_list_reinsert()
886 else if (policy->priority == p->priority && in xfrm_policy_inexact_list_reinsert()
887 policy->pos > p->pos) in xfrm_policy_inexact_list_reinsert()
888 newpos = &p->bydst; in xfrm_policy_inexact_list_reinsert()
893 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_policy_inexact_list_reinsert()
894 hlist_add_behind_rcu(&policy->bydst, newpos); in xfrm_policy_inexact_list_reinsert()
896 hlist_add_head_rcu(&policy->bydst, &n->hhead); in xfrm_policy_inexact_list_reinsert()
906 matches_s = xfrm_policy_addr_delta(&policy->selector.saddr, in xfrm_policy_inexact_list_reinsert()
907 &n->addr, in xfrm_policy_inexact_list_reinsert()
908 n->prefixlen, in xfrm_policy_inexact_list_reinsert()
910 matches_d = xfrm_policy_addr_delta(&policy->selector.daddr, in xfrm_policy_inexact_list_reinsert()
911 &n->addr, in xfrm_policy_inexact_list_reinsert()
912 n->prefixlen, in xfrm_policy_inexact_list_reinsert()
935 WARN_ON_ONCE(!RB_EMPTY_ROOT(&n->root)); in xfrm_policy_inexact_node_reinsert()
938 p = &new->rb_node; in xfrm_policy_inexact_node_reinsert()
946 prefixlen = min(node->prefixlen, n->prefixlen); in xfrm_policy_inexact_node_reinsert()
948 delta = xfrm_policy_addr_delta(&n->addr, &node->addr, in xfrm_policy_inexact_node_reinsert()
951 p = &parent->rb_left; in xfrm_policy_inexact_node_reinsert()
953 p = &parent->rb_right; in xfrm_policy_inexact_node_reinsert()
955 bool same_prefixlen = node->prefixlen == n->prefixlen; in xfrm_policy_inexact_node_reinsert()
958 hlist_for_each_entry(tmp, &n->hhead, bydst) { in xfrm_policy_inexact_node_reinsert()
959 tmp->bydst_reinsert = true; in xfrm_policy_inexact_node_reinsert()
960 hlist_del_rcu(&tmp->bydst); in xfrm_policy_inexact_node_reinsert()
963 node->prefixlen = prefixlen; in xfrm_policy_inexact_node_reinsert()
979 rb_link_node_rcu(&n->node, parent, p); in xfrm_policy_inexact_node_reinsert()
980 rb_insert_color(&n->node, new); in xfrm_policy_inexact_node_reinsert()
993 /* To-be-merged node v has a subtree. in xfrm_policy_inexact_node_merge()
995 * Dismantle it and insert its nodes to n->root. in xfrm_policy_inexact_node_merge()
997 while ((rnode = rb_first(&v->root)) != NULL) { in xfrm_policy_inexact_node_merge()
999 rb_erase(&node->node, &v->root); in xfrm_policy_inexact_node_merge()
1000 xfrm_policy_inexact_node_reinsert(net, node, &n->root, in xfrm_policy_inexact_node_merge()
1004 hlist_for_each_entry(tmp, &v->hhead, bydst) { in xfrm_policy_inexact_node_merge()
1005 tmp->bydst_reinsert = true; in xfrm_policy_inexact_node_merge()
1006 hlist_del_rcu(&tmp->bydst); in xfrm_policy_inexact_node_merge()
1022 p = &root->rb_node; in xfrm_policy_inexact_insert_node()
1029 delta = xfrm_policy_addr_delta(addr, &node->addr, in xfrm_policy_inexact_insert_node()
1030 node->prefixlen, in xfrm_policy_inexact_insert_node()
1032 if (delta == 0 && prefixlen >= node->prefixlen) { in xfrm_policy_inexact_insert_node()
1038 p = &parent->rb_left; in xfrm_policy_inexact_insert_node()
1040 p = &parent->rb_right; in xfrm_policy_inexact_insert_node()
1042 if (prefixlen < node->prefixlen) { in xfrm_policy_inexact_insert_node()
1043 delta = xfrm_policy_addr_delta(addr, &node->addr, in xfrm_policy_inexact_insert_node()
1050 * to be removed and re-inserted with the smaller in xfrm_policy_inexact_insert_node()
1054 rb_erase(&node->node, root); in xfrm_policy_inexact_insert_node()
1062 * prefixlen. Merge the to-be-reinserted in xfrm_policy_inexact_insert_node()
1071 p = &root->rb_node; in xfrm_policy_inexact_insert_node()
1083 rb_link_node_rcu(&node->node, parent, p); in xfrm_policy_inexact_insert_node()
1084 rb_insert_color(&node->node, root); in xfrm_policy_inexact_insert_node()
1097 xfrm_policy_inexact_gc_tree(&node->root, rm); in xfrm_policy_inexact_gc_tree()
1100 if (!hlist_empty(&node->hhead) || !RB_EMPTY_ROOT(&node->root)) { in xfrm_policy_inexact_gc_tree()
1105 rb_erase(&node->node, r); in xfrm_policy_inexact_gc_tree()
1112 write_seqcount_begin(&b->count); in __xfrm_policy_inexact_prune_bin()
1113 xfrm_policy_inexact_gc_tree(&b->root_d, net_exit); in __xfrm_policy_inexact_prune_bin()
1114 xfrm_policy_inexact_gc_tree(&b->root_s, net_exit); in __xfrm_policy_inexact_prune_bin()
1115 write_seqcount_end(&b->count); in __xfrm_policy_inexact_prune_bin()
1117 if (!RB_EMPTY_ROOT(&b->root_d) || !RB_EMPTY_ROOT(&b->root_s) || in __xfrm_policy_inexact_prune_bin()
1118 !hlist_empty(&b->hhead)) { in __xfrm_policy_inexact_prune_bin()
1123 if (rhashtable_remove_fast(&xfrm_policy_inexact_table, &b->head, in __xfrm_policy_inexact_prune_bin()
1125 list_del(&b->inexact_bins); in __xfrm_policy_inexact_prune_bin()
1132 struct net *net = read_pnet(&b->k.net); in xfrm_policy_inexact_prune_bin()
1134 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_prune_bin()
1136 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_prune_bin()
1143 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in __xfrm_policy_inexact_flush()
1145 list_for_each_entry_safe(bin, t, &net->xfrm.inexact_bins, inexact_bins) in __xfrm_policy_inexact_flush()
1157 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_alloc_chain()
1160 return &bin->hhead; in xfrm_policy_inexact_alloc_chain()
1162 if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.daddr, in xfrm_policy_inexact_alloc_chain()
1163 policy->family, in xfrm_policy_inexact_alloc_chain()
1164 policy->selector.prefixlen_d)) { in xfrm_policy_inexact_alloc_chain()
1165 write_seqcount_begin(&bin->count); in xfrm_policy_inexact_alloc_chain()
1167 &bin->root_s, in xfrm_policy_inexact_alloc_chain()
1168 &policy->selector.saddr, in xfrm_policy_inexact_alloc_chain()
1169 policy->family, in xfrm_policy_inexact_alloc_chain()
1170 policy->selector.prefixlen_s, in xfrm_policy_inexact_alloc_chain()
1172 write_seqcount_end(&bin->count); in xfrm_policy_inexact_alloc_chain()
1176 return &n->hhead; in xfrm_policy_inexact_alloc_chain()
1180 write_seqcount_begin(&bin->count); in xfrm_policy_inexact_alloc_chain()
1182 &bin->root_d, in xfrm_policy_inexact_alloc_chain()
1183 &policy->selector.daddr, in xfrm_policy_inexact_alloc_chain()
1184 policy->family, in xfrm_policy_inexact_alloc_chain()
1185 policy->selector.prefixlen_d, dir); in xfrm_policy_inexact_alloc_chain()
1186 write_seqcount_end(&bin->count); in xfrm_policy_inexact_alloc_chain()
1191 if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.saddr, in xfrm_policy_inexact_alloc_chain()
1192 policy->family, in xfrm_policy_inexact_alloc_chain()
1193 policy->selector.prefixlen_s)) in xfrm_policy_inexact_alloc_chain()
1194 return &n->hhead; in xfrm_policy_inexact_alloc_chain()
1196 write_seqcount_begin(&bin->count); in xfrm_policy_inexact_alloc_chain()
1198 &n->root, in xfrm_policy_inexact_alloc_chain()
1199 &policy->selector.saddr, in xfrm_policy_inexact_alloc_chain()
1200 policy->family, in xfrm_policy_inexact_alloc_chain()
1201 policy->selector.prefixlen_s, dir); in xfrm_policy_inexact_alloc_chain()
1202 write_seqcount_end(&bin->count); in xfrm_policy_inexact_alloc_chain()
1206 return &n->hhead; in xfrm_policy_inexact_alloc_chain()
1219 return ERR_PTR(-ENOMEM); in xfrm_policy_inexact_insert()
1222 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_insert()
1227 return ERR_PTR(-ENOMEM); in xfrm_policy_inexact_insert()
1233 return ERR_PTR(-EEXIST); in xfrm_policy_inexact_insert()
1236 chain = &net->xfrm.policy_inexact[dir]; in xfrm_policy_inexact_insert()
1264 seq = read_seqbegin(&net->xfrm.policy_hthresh.lock); in xfrm_hash_rebuild()
1266 lbits4 = net->xfrm.policy_hthresh.lbits4; in xfrm_hash_rebuild()
1267 rbits4 = net->xfrm.policy_hthresh.rbits4; in xfrm_hash_rebuild()
1268 lbits6 = net->xfrm.policy_hthresh.lbits6; in xfrm_hash_rebuild()
1269 rbits6 = net->xfrm.policy_hthresh.rbits6; in xfrm_hash_rebuild()
1270 } while (read_seqretry(&net->xfrm.policy_hthresh.lock, seq)); in xfrm_hash_rebuild()
1272 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_hash_rebuild()
1273 write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); in xfrm_hash_rebuild()
1278 list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) { in xfrm_hash_rebuild()
1282 if (policy->walk.dead) in xfrm_hash_rebuild()
1285 dir = xfrm_policy_id2dir(policy->index); in xfrm_hash_rebuild()
1290 if (policy->family == AF_INET) { in xfrm_hash_rebuild()
1298 if (policy->family == AF_INET) { in xfrm_hash_rebuild()
1307 if (policy->selector.prefixlen_d < dbits || in xfrm_hash_rebuild()
1308 policy->selector.prefixlen_s < sbits) in xfrm_hash_rebuild()
1324 &net->xfrm.policy_inexact[dir], in xfrm_hash_rebuild()
1326 hlist_del_rcu(&policy->bydst); in xfrm_hash_rebuild()
1327 hlist_del_init(&policy->bydst_inexact_list); in xfrm_hash_rebuild()
1330 hmask = net->xfrm.policy_bydst[dir].hmask; in xfrm_hash_rebuild()
1331 odst = net->xfrm.policy_bydst[dir].table; in xfrm_hash_rebuild()
1332 for (i = hmask; i >= 0; i--) { in xfrm_hash_rebuild()
1334 hlist_del_rcu(&policy->bydst); in xfrm_hash_rebuild()
1338 net->xfrm.policy_bydst[dir].dbits4 = rbits4; in xfrm_hash_rebuild()
1339 net->xfrm.policy_bydst[dir].sbits4 = lbits4; in xfrm_hash_rebuild()
1340 net->xfrm.policy_bydst[dir].dbits6 = rbits6; in xfrm_hash_rebuild()
1341 net->xfrm.policy_bydst[dir].sbits6 = lbits6; in xfrm_hash_rebuild()
1344 net->xfrm.policy_bydst[dir].dbits4 = lbits4; in xfrm_hash_rebuild()
1345 net->xfrm.policy_bydst[dir].sbits4 = rbits4; in xfrm_hash_rebuild()
1346 net->xfrm.policy_bydst[dir].dbits6 = lbits6; in xfrm_hash_rebuild()
1347 net->xfrm.policy_bydst[dir].sbits6 = rbits6; in xfrm_hash_rebuild()
1351 /* re-insert all policies by order of creation */ in xfrm_hash_rebuild()
1352 list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) { in xfrm_hash_rebuild()
1353 if (policy->walk.dead) in xfrm_hash_rebuild()
1355 dir = xfrm_policy_id2dir(policy->index); in xfrm_hash_rebuild()
1361 chain = policy_hash_bysel(net, &policy->selector, in xfrm_hash_rebuild()
1362 policy->family, dir); in xfrm_hash_rebuild()
1372 if (policy->priority >= pol->priority) in xfrm_hash_rebuild()
1373 newpos = &pol->bydst; in xfrm_hash_rebuild()
1377 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_hash_rebuild()
1378 hlist_add_behind_rcu(&policy->bydst, newpos); in xfrm_hash_rebuild()
1380 hlist_add_head_rcu(&policy->bydst, chain); in xfrm_hash_rebuild()
1385 write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation); in xfrm_hash_rebuild()
1386 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_hash_rebuild()
1393 schedule_work(&net->xfrm.policy_hthresh.work); in xfrm_policy_hash_rebuild()
1408 idx = (net->xfrm.idx_generator | dir); in xfrm_gen_index()
1409 net->xfrm.idx_generator += 8; in xfrm_gen_index()
1417 list = net->xfrm.policy_byidx + idx_hash(net, idx); in xfrm_gen_index()
1420 if (p->index == idx) { in xfrm_gen_index()
1432 u32 *p1 = (u32 *) s1; in selector_cmp() local
1438 if (p1[i] != p2[i]) in selector_cmp()
1448 struct xfrm_policy_queue *pq = &old->polq; in xfrm_policy_requeue()
1451 if (skb_queue_empty(&pq->hold_queue)) in xfrm_policy_requeue()
1456 spin_lock_bh(&pq->hold_queue.lock); in xfrm_policy_requeue()
1457 skb_queue_splice_init(&pq->hold_queue, &list); in xfrm_policy_requeue()
1458 if (del_timer(&pq->hold_timer)) in xfrm_policy_requeue()
1460 spin_unlock_bh(&pq->hold_queue.lock); in xfrm_policy_requeue()
1462 pq = &new->polq; in xfrm_policy_requeue()
1464 spin_lock_bh(&pq->hold_queue.lock); in xfrm_policy_requeue()
1465 skb_queue_splice(&list, &pq->hold_queue); in xfrm_policy_requeue()
1466 pq->timeout = XFRM_QUEUE_TMO_MIN; in xfrm_policy_requeue()
1467 if (!mod_timer(&pq->hold_timer, jiffies)) in xfrm_policy_requeue()
1469 spin_unlock_bh(&pq->hold_queue.lock); in xfrm_policy_requeue()
1475 return mark->v == pol->mark.v && mark->m == pol->mark.m; in xfrm_policy_mark_match()
1481 u32 a = k->type << 24 | k->dir << 16 | k->family; in xfrm_pol_bin_key()
1483 return jhash_3words(a, k->if_id, net_hash_mix(read_pnet(&k->net)), in xfrm_pol_bin_key()
1491 return xfrm_pol_bin_key(&b->k, 0, seed); in xfrm_pol_bin_obj()
1497 const struct xfrm_pol_inexact_key *key = arg->key; in xfrm_pol_bin_cmp()
1501 if (!net_eq(read_pnet(&b->k.net), read_pnet(&key->net))) in xfrm_pol_bin_cmp()
1502 return -1; in xfrm_pol_bin_cmp()
1504 ret = b->k.dir ^ key->dir; in xfrm_pol_bin_cmp()
1508 ret = b->k.type ^ key->type; in xfrm_pol_bin_cmp()
1512 ret = b->k.family ^ key->family; in xfrm_pol_bin_cmp()
1516 return b->k.if_id ^ key->if_id; in xfrm_pol_bin_cmp()
1535 if (pol->type == policy->type && in xfrm_policy_insert_inexact_list()
1536 pol->if_id == policy->if_id && in xfrm_policy_insert_inexact_list()
1537 !selector_cmp(&pol->selector, &policy->selector) && in xfrm_policy_insert_inexact_list()
1538 xfrm_policy_mark_match(&policy->mark, pol) && in xfrm_policy_insert_inexact_list()
1539 xfrm_sec_ctx_match(pol->security, policy->security) && in xfrm_policy_insert_inexact_list()
1542 if (policy->priority > pol->priority) in xfrm_policy_insert_inexact_list()
1544 } else if (policy->priority >= pol->priority) { in xfrm_policy_insert_inexact_list()
1545 newpos = &pol->bydst_inexact_list; in xfrm_policy_insert_inexact_list()
1552 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_policy_insert_inexact_list()
1553 hlist_add_behind_rcu(&policy->bydst_inexact_list, newpos); in xfrm_policy_insert_inexact_list()
1555 hlist_add_head_rcu(&policy->bydst_inexact_list, chain); in xfrm_policy_insert_inexact_list()
1558 pol->pos = i; in xfrm_policy_insert_inexact_list()
1570 if (pol->type == policy->type && in xfrm_policy_insert_list()
1571 pol->if_id == policy->if_id && in xfrm_policy_insert_list()
1572 !selector_cmp(&pol->selector, &policy->selector) && in xfrm_policy_insert_list()
1573 xfrm_policy_mark_match(&policy->mark, pol) && in xfrm_policy_insert_list()
1574 xfrm_sec_ctx_match(pol->security, policy->security) && in xfrm_policy_insert_list()
1577 return ERR_PTR(-EEXIST); in xfrm_policy_insert_list()
1579 if (policy->priority > pol->priority) in xfrm_policy_insert_list()
1581 } else if (policy->priority >= pol->priority) { in xfrm_policy_insert_list()
1589 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_policy_insert_list()
1590 hlist_add_behind_rcu(&policy->bydst, &newpos->bydst); in xfrm_policy_insert_list()
1593 * to speed-up lookups. in xfrm_policy_insert_list()
1595 hlist_add_head_rcu(&policy->bydst, chain); in xfrm_policy_insert_list()
1606 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_insert()
1607 chain = policy_hash_bysel(net, &policy->selector, policy->family, dir); in xfrm_policy_insert()
1614 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_insert()
1621 if (policy->family == AF_INET) in xfrm_policy_insert()
1630 policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, policy->index); in xfrm_policy_insert()
1631 hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index)); in xfrm_policy_insert()
1632 policy->curlft.add_time = ktime_get_real_seconds(); in xfrm_policy_insert()
1633 policy->curlft.use_time = 0; in xfrm_policy_insert()
1634 if (!mod_timer(&policy->timer, jiffies + HZ)) in xfrm_policy_insert()
1636 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_insert()
1641 schedule_work(&net->xfrm.policy_hash_work); in xfrm_policy_insert()
1658 if (pol->type == type && in __xfrm_policy_bysel_ctx()
1659 pol->if_id == if_id && in __xfrm_policy_bysel_ctx()
1661 !selector_cmp(sel, &pol->selector) && in __xfrm_policy_bysel_ctx()
1662 xfrm_sec_ctx_match(ctx, pol->security)) in __xfrm_policy_bysel_ctx()
1679 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1680 chain = policy_hash_bysel(net, sel, sel->family, dir); in xfrm_policy_bysel_ctx()
1686 sel->family, dir, if_id); in xfrm_policy_bysel_ctx()
1688 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1693 &sel->saddr, in xfrm_policy_bysel_ctx()
1694 &sel->daddr)) { in xfrm_policy_bysel_ctx()
1695 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1709 if (!pol || tmp->pos < pol->pos) in xfrm_policy_bysel_ctx()
1720 *err = security_xfrm_policy_delete(pol->security); in xfrm_policy_bysel_ctx()
1722 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1729 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1746 *err = -ENOENT; in xfrm_policy_byid()
1751 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_byid()
1752 chain = net->xfrm.policy_byidx + idx_hash(net, id); in xfrm_policy_byid()
1755 if (pol->type == type && pol->index == id && in xfrm_policy_byid()
1756 pol->if_id == if_id && xfrm_policy_mark_match(mark, pol)) { in xfrm_policy_byid()
1760 pol->security); in xfrm_policy_byid()
1762 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_byid()
1771 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_byid()
1786 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_policy_flush_secctx_check()
1787 if (pol->walk.dead || in xfrm_policy_flush_secctx_check()
1788 xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX || in xfrm_policy_flush_secctx_check()
1789 pol->type != type) in xfrm_policy_flush_secctx_check()
1792 err = security_xfrm_policy_delete(pol->security); in xfrm_policy_flush_secctx_check()
1808 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_dev_policy_flush_secctx_check()
1809 if (pol->walk.dead || in xfrm_dev_policy_flush_secctx_check()
1810 xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX || in xfrm_dev_policy_flush_secctx_check()
1811 pol->xdo.dev != dev) in xfrm_dev_policy_flush_secctx_check()
1814 err = security_xfrm_policy_delete(pol->security); in xfrm_dev_policy_flush_secctx_check()
1842 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_flush()
1849 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_policy_flush()
1850 if (pol->walk.dead) in xfrm_policy_flush()
1853 dir = xfrm_policy_id2dir(pol->index); in xfrm_policy_flush()
1855 pol->type != type) in xfrm_policy_flush()
1859 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_flush()
1863 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_flush()
1869 err = -ESRCH; in xfrm_policy_flush()
1871 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_flush()
1882 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_dev_policy_flush()
1889 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_dev_policy_flush()
1890 if (pol->walk.dead) in xfrm_dev_policy_flush()
1893 dir = xfrm_policy_id2dir(pol->index); in xfrm_dev_policy_flush()
1895 pol->xdo.dev != dev) in xfrm_dev_policy_flush()
1899 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_dev_policy_flush()
1903 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_dev_policy_flush()
1909 err = -ESRCH; in xfrm_dev_policy_flush()
1911 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_dev_policy_flush()
1924 if (walk->type >= XFRM_POLICY_TYPE_MAX && in xfrm_policy_walk()
1925 walk->type != XFRM_POLICY_TYPE_ANY) in xfrm_policy_walk()
1926 return -EINVAL; in xfrm_policy_walk()
1928 if (list_empty(&walk->walk.all) && walk->seq != 0) in xfrm_policy_walk()
1931 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_walk()
1932 if (list_empty(&walk->walk.all)) in xfrm_policy_walk()
1933 x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all); in xfrm_policy_walk()
1935 x = list_first_entry(&walk->walk.all, in xfrm_policy_walk()
1938 list_for_each_entry_from(x, &net->xfrm.policy_all, all) { in xfrm_policy_walk()
1939 if (x->dead) in xfrm_policy_walk()
1942 if (walk->type != XFRM_POLICY_TYPE_ANY && in xfrm_policy_walk()
1943 walk->type != pol->type) in xfrm_policy_walk()
1945 error = func(pol, xfrm_policy_id2dir(pol->index), in xfrm_policy_walk()
1946 walk->seq, data); in xfrm_policy_walk()
1948 list_move_tail(&walk->walk.all, &x->all); in xfrm_policy_walk()
1951 walk->seq++; in xfrm_policy_walk()
1953 if (walk->seq == 0) { in xfrm_policy_walk()
1954 error = -ENOENT; in xfrm_policy_walk()
1957 list_del_init(&walk->walk.all); in xfrm_policy_walk()
1959 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_walk()
1966 INIT_LIST_HEAD(&walk->walk.all); in xfrm_policy_walk_init()
1967 walk->walk.dead = 1; in xfrm_policy_walk_init()
1968 walk->type = type; in xfrm_policy_walk_init()
1969 walk->seq = 0; in xfrm_policy_walk_init()
1975 if (list_empty(&walk->walk.all)) in xfrm_policy_walk_done()
1978 spin_lock_bh(&net->xfrm.xfrm_policy_lock); /*FIXME where is net? */ in xfrm_policy_walk_done()
1979 list_del(&walk->walk.all); in xfrm_policy_walk_done()
1980 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_walk_done()
1987 * Returns 0 if policy found, else an -errno.
1993 const struct xfrm_selector *sel = &pol->selector; in xfrm_policy_match()
1994 int ret = -ESRCH; in xfrm_policy_match()
1997 if (pol->family != family || in xfrm_policy_match()
1998 pol->if_id != if_id || in xfrm_policy_match()
1999 (fl->flowi_mark & pol->mark.m) != pol->mark.v || in xfrm_policy_match()
2000 pol->type != type) in xfrm_policy_match()
2005 ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid); in xfrm_policy_match()
2020 parent = rcu_dereference_raw(r->rb_node); in xfrm_policy_lookup_inexact_addr()
2027 delta = xfrm_policy_addr_delta(addr, &node->addr, in xfrm_policy_lookup_inexact_addr()
2028 node->prefixlen, family); in xfrm_policy_lookup_inexact_addr()
2030 parent = rcu_dereference_raw(parent->rb_left); in xfrm_policy_lookup_inexact_addr()
2033 parent = rcu_dereference_raw(parent->rb_right); in xfrm_policy_lookup_inexact_addr()
2058 family = b->k.family; in xfrm_policy_find_inexact_candidates()
2060 cand->res[XFRM_POL_CAND_ANY] = &b->hhead; in xfrm_policy_find_inexact_candidates()
2062 n = xfrm_policy_lookup_inexact_addr(&b->root_d, &b->count, daddr, in xfrm_policy_find_inexact_candidates()
2065 cand->res[XFRM_POL_CAND_DADDR] = &n->hhead; in xfrm_policy_find_inexact_candidates()
2066 n = xfrm_policy_lookup_inexact_addr(&n->root, &b->count, saddr, in xfrm_policy_find_inexact_candidates()
2069 cand->res[XFRM_POL_CAND_BOTH] = &n->hhead; in xfrm_policy_find_inexact_candidates()
2072 n = xfrm_policy_lookup_inexact_addr(&b->root_s, &b->count, saddr, in xfrm_policy_find_inexact_candidates()
2075 cand->res[XFRM_POL_CAND_SADDR] = &n->hhead; in xfrm_policy_find_inexact_candidates()
2103 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_lookup()
2118 u32 priority = prefer ? prefer->priority : ~0u; in __xfrm_policy_eval_candidates()
2127 if (pol->priority > priority) in __xfrm_policy_eval_candidates()
2132 if (err != -ESRCH) in __xfrm_policy_eval_candidates()
2140 if (pol->priority == priority && in __xfrm_policy_eval_candidates()
2141 prefer->pos < pol->pos) in __xfrm_policy_eval_candidates()
2160 for (i = 0; i < ARRAY_SIZE(cand->res); i++) { in xfrm_policy_eval_candidates()
2161 tmp = __xfrm_policy_eval_candidates(cand->res[i], in xfrm_policy_eval_candidates()
2194 retry: in xfrm_policy_lookup_bytype()
2196 sequence = read_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); in xfrm_policy_lookup_bytype()
2198 } while (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence)); in xfrm_policy_lookup_bytype()
2204 if (err == -ESRCH) in xfrm_policy_lookup_bytype()
2215 if (ret && ret->xdo.type == XFRM_DEV_OFFLOAD_PACKET) in xfrm_policy_lookup_bytype()
2232 if (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence)) in xfrm_policy_lookup_bytype()
2233 goto retry; in xfrm_policy_lookup_bytype()
2236 goto retry; in xfrm_policy_lookup_bytype()
2267 pol = rcu_dereference(sk->sk_policy[dir]); in xfrm_sk_policy_lookup()
2272 if (pol->family != family) { in xfrm_sk_policy_lookup()
2277 match = xfrm_selector_match(&pol->selector, fl, family); in xfrm_sk_policy_lookup()
2279 if ((READ_ONCE(sk->sk_mark) & pol->mark.m) != pol->mark.v || in xfrm_sk_policy_lookup()
2280 pol->if_id != if_id) { in xfrm_sk_policy_lookup()
2284 err = security_xfrm_policy_lookup(pol->security, in xfrm_sk_policy_lookup()
2285 fl->flowi_secid); in xfrm_sk_policy_lookup()
2289 } else if (err == -ESRCH) { in xfrm_sk_policy_lookup()
2306 list_add(&pol->walk.all, &net->xfrm.policy_all); in __xfrm_policy_link()
2307 net->xfrm.policy_count[dir]++; in __xfrm_policy_link()
2316 if (list_empty(&pol->walk.all)) in __xfrm_policy_unlink()
2320 if (!hlist_unhashed(&pol->bydst)) { in __xfrm_policy_unlink()
2321 hlist_del_rcu(&pol->bydst); in __xfrm_policy_unlink()
2322 hlist_del_init(&pol->bydst_inexact_list); in __xfrm_policy_unlink()
2323 hlist_del(&pol->byidx); in __xfrm_policy_unlink()
2326 list_del_init(&pol->walk.all); in __xfrm_policy_unlink()
2327 net->xfrm.policy_count[dir]--; in __xfrm_policy_unlink()
2346 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_delete()
2348 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_delete()
2353 return -ENOENT; in xfrm_policy_delete()
2363 if (pol && pol->type != XFRM_POLICY_TYPE_MAIN) in xfrm_sk_policy_insert()
2364 return -EINVAL; in xfrm_sk_policy_insert()
2367 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_sk_policy_insert()
2368 old_pol = rcu_dereference_protected(sk->sk_policy[dir], in xfrm_sk_policy_insert()
2369 lockdep_is_held(&net->xfrm.xfrm_policy_lock)); in xfrm_sk_policy_insert()
2371 pol->curlft.add_time = ktime_get_real_seconds(); in xfrm_sk_policy_insert()
2372 pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0); in xfrm_sk_policy_insert()
2375 rcu_assign_pointer(sk->sk_policy[dir], pol); in xfrm_sk_policy_insert()
2385 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_sk_policy_insert()
2399 newp->selector = old->selector; in clone_policy()
2400 if (security_xfrm_policy_clone(old->security, in clone_policy()
2401 &newp->security)) { in clone_policy()
2405 newp->lft = old->lft; in clone_policy()
2406 newp->curlft = old->curlft; in clone_policy()
2407 newp->mark = old->mark; in clone_policy()
2408 newp->if_id = old->if_id; in clone_policy()
2409 newp->action = old->action; in clone_policy()
2410 newp->flags = old->flags; in clone_policy()
2411 newp->xfrm_nr = old->xfrm_nr; in clone_policy()
2412 newp->index = old->index; in clone_policy()
2413 newp->type = old->type; in clone_policy()
2414 newp->family = old->family; in clone_policy()
2415 memcpy(newp->xfrm_vec, old->xfrm_vec, in clone_policy()
2416 newp->xfrm_nr*sizeof(struct xfrm_tmpl)); in clone_policy()
2417 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in clone_policy()
2419 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in clone_policy()
2433 p = rcu_dereference(osk->sk_policy[i]); in __xfrm_sk_clone_policy()
2437 ret = -ENOMEM; in __xfrm_sk_clone_policy()
2440 rcu_assign_pointer(sk->sk_policy[i], np); in __xfrm_sk_clone_policy()
2449 const struct xfrm_dst_lookup_params *params) in xfrm_get_saddr() argument
2455 return -EINVAL; in xfrm_get_saddr()
2456 err = afinfo->get_saddr(saddr, params); in xfrm_get_saddr()
2474 for (nx = 0, i = 0; i < policy->xfrm_nr; i++) { in xfrm_tmpl_resolve_one()
2478 struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i]; in xfrm_tmpl_resolve_one()
2480 if (tmpl->mode == XFRM_MODE_TUNNEL || in xfrm_tmpl_resolve_one()
2481 tmpl->mode == XFRM_MODE_BEET) { in xfrm_tmpl_resolve_one()
2482 remote = &tmpl->id.daddr; in xfrm_tmpl_resolve_one()
2483 local = &tmpl->saddr; in xfrm_tmpl_resolve_one()
2484 if (xfrm_addr_any(local, tmpl->encap_family)) { in xfrm_tmpl_resolve_one()
2485 struct xfrm_dst_lookup_params params; in xfrm_tmpl_resolve_one() local
2487 memset(¶ms, 0, sizeof(params)); in xfrm_tmpl_resolve_one()
2488 params.net = net; in xfrm_tmpl_resolve_one()
2489 params.oif = fl->flowi_oif; in xfrm_tmpl_resolve_one()
2490 params.daddr = remote; in xfrm_tmpl_resolve_one()
2491 error = xfrm_get_saddr(tmpl->encap_family, &tmp, in xfrm_tmpl_resolve_one()
2492 ¶ms); in xfrm_tmpl_resolve_one()
2500 family, policy->if_id); in xfrm_tmpl_resolve_one()
2502 if (x && x->km.state == XFRM_STATE_VALID) { in xfrm_tmpl_resolve_one()
2509 error = (x->km.state == XFRM_STATE_ERROR ? in xfrm_tmpl_resolve_one()
2510 -EINVAL : -EAGAIN); in xfrm_tmpl_resolve_one()
2512 } else if (error == -ESRCH) { in xfrm_tmpl_resolve_one()
2513 error = -EAGAIN; in xfrm_tmpl_resolve_one()
2516 if (!tmpl->optional) in xfrm_tmpl_resolve_one()
2522 for (nx--; nx >= 0; nx--) in xfrm_tmpl_resolve_one()
2539 if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) { in xfrm_tmpl_resolve()
2540 error = -ENOBUFS; in xfrm_tmpl_resolve()
2559 for (cnx--; cnx >= 0; cnx--) in xfrm_tmpl_resolve()
2568 return IPTOS_RT_MASK & fl->u.ip4.flowi4_tos; in xfrm_get_tos()
2580 return ERR_PTR(-EINVAL); in xfrm_alloc_dst()
2584 dst_ops = &net->xfrm.xfrm4_dst_ops; in xfrm_alloc_dst()
2588 dst_ops = &net->xfrm.xfrm6_dst_ops; in xfrm_alloc_dst()
2599 xdst = ERR_PTR(-ENOBUFS); in xfrm_alloc_dst()
2609 if (dst->ops->family == AF_INET6) { in xfrm_init_path()
2610 path->path_cookie = rt6_get_cookie(dst_rt6_info(dst)); in xfrm_init_path()
2611 path->u.rt6.rt6i_nfheader_len = nfheader_len; in xfrm_init_path()
2619 xfrm_policy_get_afinfo(xdst->u.dst.ops->family); in xfrm_fill_dst()
2623 return -EINVAL; in xfrm_fill_dst()
2625 err = afinfo->fill_dst(xdst, dev, fl); in xfrm_fill_dst()
2657 int family = policy->selector.family; in xfrm_bundle_create()
2668 struct dst_entry *dst1 = &xdst->u.dst; in xfrm_bundle_create()
2683 xfrm_dst_set_child(xdst_prev, &xdst->u.dst); in xfrm_bundle_create()
2685 if (xfrm[i]->sel.family == AF_UNSPEC) { in xfrm_bundle_create()
2689 err = -EAFNOSUPPORT; in xfrm_bundle_create()
2694 inner_mode = &xfrm[i]->inner_mode; in xfrm_bundle_create()
2696 xdst->route = dst; in xfrm_bundle_create()
2699 if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) { in xfrm_bundle_create()
2703 if (xfrm[i]->props.smark.v || xfrm[i]->props.smark.m) in xfrm_bundle_create()
2704 mark = xfrm_smark_get(fl->flowi_mark, xfrm[i]); in xfrm_bundle_create()
2706 if (xfrm[i]->xso.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_bundle_create()
2707 family = xfrm[i]->props.family; in xfrm_bundle_create()
2709 oif = fl->flowi_oif ? : fl->flowi_l3mdev; in xfrm_bundle_create()
2718 dst1->xfrm = xfrm[i]; in xfrm_bundle_create()
2719 xdst->xfrm_genid = xfrm[i]->genid; in xfrm_bundle_create()
2721 dst1->obsolete = DST_OBSOLETE_FORCE_CHK; in xfrm_bundle_create()
2722 dst1->lastuse = now; in xfrm_bundle_create()
2724 dst1->input = dst_discard; in xfrm_bundle_create()
2727 afinfo = xfrm_state_afinfo_get_rcu(inner_mode->family); in xfrm_bundle_create()
2729 dst1->output = afinfo->output; in xfrm_bundle_create()
2731 dst1->output = dst_discard_out; in xfrm_bundle_create()
2736 header_len += xfrm[i]->props.header_len; in xfrm_bundle_create()
2737 if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT) in xfrm_bundle_create()
2738 nfheader_len += xfrm[i]->props.header_len; in xfrm_bundle_create()
2739 trailer_len += xfrm[i]->props.trailer_len; in xfrm_bundle_create()
2743 xdst0->path = dst; in xfrm_bundle_create()
2745 err = -ENODEV; in xfrm_bundle_create()
2746 dev = dst->dev; in xfrm_bundle_create()
2754 xdst_prev = (struct xfrm_dst *) xfrm_dst_child(&xdst_prev->u.dst)) { in xfrm_bundle_create()
2759 xdst_prev->u.dst.header_len = header_len; in xfrm_bundle_create()
2760 xdst_prev->u.dst.trailer_len = trailer_len; in xfrm_bundle_create()
2761 header_len -= xdst_prev->u.dst.xfrm->props.header_len; in xfrm_bundle_create()
2762 trailer_len -= xdst_prev->u.dst.xfrm->props.trailer_len; in xfrm_bundle_create()
2765 return &xdst0->u.dst; in xfrm_bundle_create()
2772 dst_release_immediate(&xdst0->u.dst); in xfrm_bundle_create()
2793 *num_xfrms = pols[0]->xfrm_nr; in xfrm_expand_policies()
2796 if (pols[0]->action == XFRM_POLICY_ALLOW && in xfrm_expand_policies()
2797 pols[0]->type != XFRM_POLICY_TYPE_MAIN) { in xfrm_expand_policies()
2802 pols[0]->if_id); in xfrm_expand_policies()
2810 (*num_xfrms) += pols[1]->xfrm_nr; in xfrm_expand_policies()
2815 if (pols[i]->action != XFRM_POLICY_ALLOW) { in xfrm_expand_policies()
2816 *num_xfrms = -1; in xfrm_expand_policies()
2843 if (err != -EAGAIN) in xfrm_resolve_and_create_bundle()
2855 xdst->num_xfrms = err; in xfrm_resolve_and_create_bundle()
2856 xdst->num_pols = num_pols; in xfrm_resolve_and_create_bundle()
2857 memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols); in xfrm_resolve_and_create_bundle()
2858 xdst->policy_genid = atomic_read(&pols[0]->genid); in xfrm_resolve_and_create_bundle()
2870 struct xfrm_policy_queue *pq = &pol->polq; in xfrm_policy_queue_process()
2875 spin_lock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2876 skb = skb_peek(&pq->hold_queue); in xfrm_policy_queue_process()
2878 spin_unlock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2882 sk = skb->sk; in xfrm_policy_queue_process()
2885 skb_mark = skb->mark; in xfrm_policy_queue_process()
2886 skb->mark = pol->mark.v; in xfrm_policy_queue_process()
2887 xfrm_decode_session(skb, &fl, dst->ops->family); in xfrm_policy_queue_process()
2888 skb->mark = skb_mark; in xfrm_policy_queue_process()
2889 spin_unlock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2896 if (dst->flags & DST_XFRM_QUEUE) { in xfrm_policy_queue_process()
2899 if (pq->timeout >= XFRM_QUEUE_TMO_MAX) in xfrm_policy_queue_process()
2902 pq->timeout = pq->timeout << 1; in xfrm_policy_queue_process()
2903 if (!mod_timer(&pq->hold_timer, jiffies + pq->timeout)) in xfrm_policy_queue_process()
2912 spin_lock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2913 pq->timeout = 0; in xfrm_policy_queue_process()
2914 skb_queue_splice_init(&pq->hold_queue, &list); in xfrm_policy_queue_process()
2915 spin_unlock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2921 skb_mark = skb->mark; in xfrm_policy_queue_process()
2922 skb->mark = pol->mark.v; in xfrm_policy_queue_process()
2923 xfrm_decode_session(skb, &fl, skb_dst(skb)->ops->family); in xfrm_policy_queue_process()
2924 skb->mark = skb_mark; in xfrm_policy_queue_process()
2927 dst = xfrm_lookup(net, xfrm_dst_path(skb_dst(skb)), &fl, skb->sk, 0); in xfrm_policy_queue_process()
2937 dst_output(net, skb->sk, skb); in xfrm_policy_queue_process()
2945 pq->timeout = 0; in xfrm_policy_queue_process()
2946 skb_queue_purge(&pq->hold_queue); in xfrm_policy_queue_process()
2955 struct xfrm_policy *pol = xdst->pols[0]; in xdst_queue_output()
2956 struct xfrm_policy_queue *pq = &pol->polq; in xdst_queue_output()
2963 if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) { in xdst_queue_output()
2965 return -EAGAIN; in xdst_queue_output()
2970 spin_lock_bh(&pq->hold_queue.lock); in xdst_queue_output()
2972 if (!pq->timeout) in xdst_queue_output()
2973 pq->timeout = XFRM_QUEUE_TMO_MIN; in xdst_queue_output()
2975 sched_next = jiffies + pq->timeout; in xdst_queue_output()
2977 if (del_timer(&pq->hold_timer)) { in xdst_queue_output()
2978 if (time_before(pq->hold_timer.expires, sched_next)) in xdst_queue_output()
2979 sched_next = pq->hold_timer.expires; in xdst_queue_output()
2983 __skb_queue_tail(&pq->hold_queue, skb); in xdst_queue_output()
2984 if (!mod_timer(&pq->hold_timer, sched_next)) in xdst_queue_output()
2987 spin_unlock_bh(&pq->hold_queue.lock); in xdst_queue_output()
3008 if (!(xflo->flags & XFRM_LOOKUP_QUEUE) || in xfrm_create_dummy_bundle()
3009 net->xfrm.sysctl_larval_drop || in xfrm_create_dummy_bundle()
3013 dst = xflo->dst_orig; in xfrm_create_dummy_bundle()
3014 dst1 = &xdst->u.dst; in xfrm_create_dummy_bundle()
3016 xdst->route = dst; in xfrm_create_dummy_bundle()
3020 dst1->obsolete = DST_OBSOLETE_FORCE_CHK; in xfrm_create_dummy_bundle()
3021 dst1->flags |= DST_XFRM_QUEUE; in xfrm_create_dummy_bundle()
3022 dst1->lastuse = jiffies; in xfrm_create_dummy_bundle()
3024 dst1->input = dst_discard; in xfrm_create_dummy_bundle()
3025 dst1->output = xdst_queue_output; in xfrm_create_dummy_bundle()
3029 xdst->path = dst; in xfrm_create_dummy_bundle()
3033 err = -ENODEV; in xfrm_create_dummy_bundle()
3034 dev = dst->dev; in xfrm_create_dummy_bundle()
3074 xflo->dst_orig); in xfrm_bundle_lookup()
3077 if (err == -EREMOTE) { in xfrm_bundle_lookup()
3082 if (err != -EAGAIN) in xfrm_bundle_lookup()
3101 xdst->num_pols = num_pols; in xfrm_bundle_lookup()
3102 xdst->num_xfrms = num_xfrms; in xfrm_bundle_lookup()
3103 memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols); in xfrm_bundle_lookup()
3122 return ERR_PTR(-EINVAL); in make_blackhole()
3124 ret = afinfo->blackhole_route(net, dst_orig); in make_blackhole()
3148 u16 family = dst_orig->ops->family; in xfrm_lookup_with_ifid()
3157 if (sk && sk->sk_policy[XFRM_POLICY_OUT]) { in xfrm_lookup_with_ifid()
3179 if (err == -EREMOTE) in xfrm_lookup_with_ifid()
3189 route = xdst->route; in xfrm_lookup_with_ifid()
3200 if (!if_id && ((dst_orig->flags & DST_NOXFRM) || in xfrm_lookup_with_ifid()
3201 !net->xfrm.policy_count[XFRM_POLICY_OUT])) in xfrm_lookup_with_ifid()
3212 num_pols = xdst->num_pols; in xfrm_lookup_with_ifid()
3213 num_xfrms = xdst->num_xfrms; in xfrm_lookup_with_ifid()
3214 memcpy(pols, xdst->pols, sizeof(struct xfrm_policy *) * num_pols); in xfrm_lookup_with_ifid()
3215 route = xdst->route; in xfrm_lookup_with_ifid()
3218 dst = &xdst->u.dst; in xfrm_lookup_with_ifid()
3226 if (net->xfrm.sysctl_larval_drop) { in xfrm_lookup_with_ifid()
3228 err = -EREMOTE; in xfrm_lookup_with_ifid()
3232 err = -EAGAIN; in xfrm_lookup_with_ifid()
3243 !(pols[0]->flags & XFRM_POLICY_ICMP)) { in xfrm_lookup_with_ifid()
3244 err = -ENOENT; in xfrm_lookup_with_ifid()
3249 WRITE_ONCE(pols[i]->curlft.use_time, ktime_get_real_seconds()); in xfrm_lookup_with_ifid()
3254 err = -EPERM; in xfrm_lookup_with_ifid()
3266 if (dst && dst->xfrm && in xfrm_lookup_with_ifid()
3267 dst->xfrm->props.mode == XFRM_MODE_TUNNEL) in xfrm_lookup_with_ifid()
3268 dst->flags |= DST_XFRM_TUNNEL; in xfrm_lookup_with_ifid()
3272 if ((!dst_orig->dev || !(dst_orig->dev->flags & IFF_LOOPBACK)) && in xfrm_lookup_with_ifid()
3273 net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { in xfrm_lookup_with_ifid()
3274 err = -EPERM; in xfrm_lookup_with_ifid()
3281 err = -ENOENT; in xfrm_lookup_with_ifid()
3316 if (PTR_ERR(dst) == -EREMOTE) in xfrm_lookup_route()
3317 return make_blackhole(net, dst_orig->ops->family, dst_orig); in xfrm_lookup_route()
3332 if (!sp || idx < 0 || idx >= sp->len) in xfrm_secpath_reject()
3334 x = sp->xvec[idx]; in xfrm_secpath_reject()
3335 if (!x->type->reject) in xfrm_secpath_reject()
3337 return x->type->reject(x, skb, fl); in xfrm_secpath_reject()
3342 * stupid way. Shame on me. :-) Of course, connected sockets must
3351 return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family); in xfrm_state_ok()
3352 return x->id.proto == tmpl->id.proto && in xfrm_state_ok()
3353 (x->id.spi == tmpl->id.spi || !tmpl->id.spi) && in xfrm_state_ok()
3354 (x->props.reqid == tmpl->reqid || !tmpl->reqid) && in xfrm_state_ok()
3355 x->props.mode == tmpl->mode && in xfrm_state_ok()
3356 (tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) || in xfrm_state_ok()
3357 !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) && in xfrm_state_ok()
3358 !(x->props.mode != XFRM_MODE_TRANSPORT && in xfrm_state_ok()
3360 (if_id == 0 || if_id == x->if_id); in xfrm_state_ok()
3367 * -1 is returned when no matching template is found.
3368 * Otherwise "-2 - errored_index" is returned.
3376 if (tmpl->optional) { in xfrm_policy_ok()
3377 if (tmpl->mode == XFRM_MODE_TRANSPORT) in xfrm_policy_ok()
3380 start = -1; in xfrm_policy_ok()
3381 for (; idx < sp->len; idx++) { in xfrm_policy_ok()
3382 if (xfrm_state_ok(tmpl, sp->xvec[idx], family, if_id)) in xfrm_policy_ok()
3384 if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) { in xfrm_policy_ok()
3385 if (idx < sp->verified_cnt) { in xfrm_policy_ok()
3392 if (start == -1) in xfrm_policy_ok()
3393 start = -2-idx; in xfrm_policy_ok()
3404 int ihl = iph->ihl; in decode_session4()
3406 struct flowi4 *fl4 = &fl->u.ip4; in decode_session4()
3409 if (skb_dst(skb) && skb_dst(skb)->dev) in decode_session4()
3410 oif = skb_dst(skb)->dev->ifindex; in decode_session4()
3413 fl4->flowi4_mark = skb->mark; in decode_session4()
3414 fl4->flowi4_oif = reverse ? skb->skb_iif : oif; in decode_session4()
3416 fl4->flowi4_proto = iph->protocol; in decode_session4()
3417 fl4->daddr = reverse ? iph->saddr : iph->daddr; in decode_session4()
3418 fl4->saddr = reverse ? iph->daddr : iph->saddr; in decode_session4()
3419 fl4->flowi4_tos = iph->tos & ~INET_ECN_MASK; in decode_session4()
3422 switch (iph->protocol) { in decode_session4()
3428 if (xprth + 4 < skb->data || in decode_session4()
3429 pskb_may_pull(skb, xprth + 4 - skb->data)) { in decode_session4()
3435 fl4->fl4_sport = ports[!!reverse]; in decode_session4()
3436 fl4->fl4_dport = ports[!reverse]; in decode_session4()
3440 if (xprth + 2 < skb->data || in decode_session4()
3441 pskb_may_pull(skb, xprth + 2 - skb->data)) { in decode_session4()
3447 fl4->fl4_icmp_type = icmp[0]; in decode_session4()
3448 fl4->fl4_icmp_code = icmp[1]; in decode_session4()
3452 if (xprth + 12 < skb->data || in decode_session4()
3453 pskb_may_pull(skb, xprth + 12 - skb->data)) { in decode_session4()
3464 fl4->fl4_gre_key = gre_hdr[1]; in decode_session4()
3478 struct flowi6 *fl6 = &fl->u.ip6; in decode_session6()
3484 u16 nhoff = IP6CB(skb)->nhoff; in decode_session6()
3493 if (skb_dst(skb) && skb_dst(skb)->dev) in decode_session6()
3494 oif = skb_dst(skb)->dev->ifindex; in decode_session6()
3497 fl6->flowi6_mark = skb->mark; in decode_session6()
3498 fl6->flowi6_oif = reverse ? skb->skb_iif : oif; in decode_session6()
3500 fl6->daddr = reverse ? hdr->saddr : hdr->daddr; in decode_session6()
3501 fl6->saddr = reverse ? hdr->daddr : hdr->saddr; in decode_session6()
3503 while (nh + offset + sizeof(*exthdr) < skb->data || in decode_session6()
3504 pskb_may_pull(skb, nh + offset + sizeof(*exthdr) - skb->data)) { in decode_session6()
3516 nexthdr = exthdr->nexthdr; in decode_session6()
3523 if (!onlyproto && (nh + offset + 4 < skb->data || in decode_session6()
3524 pskb_may_pull(skb, nh + offset + 4 - skb->data))) { in decode_session6()
3529 fl6->fl6_sport = ports[!!reverse]; in decode_session6()
3530 fl6->fl6_dport = ports[!reverse]; in decode_session6()
3532 fl6->flowi6_proto = nexthdr; in decode_session6()
3535 if (!onlyproto && (nh + offset + 2 < skb->data || in decode_session6()
3536 pskb_may_pull(skb, nh + offset + 2 - skb->data))) { in decode_session6()
3541 fl6->fl6_icmp_type = icmp[0]; in decode_session6()
3542 fl6->fl6_icmp_code = icmp[1]; in decode_session6()
3544 fl6->flowi6_proto = nexthdr; in decode_session6()
3548 (nh + offset + 12 < skb->data || in decode_session6()
3549 pskb_may_pull(skb, nh + offset + 12 - skb->data))) { in decode_session6()
3557 if (gre_hdr->flags & GRE_KEY) { in decode_session6()
3558 if (gre_hdr->flags & GRE_CSUM) in decode_session6()
3560 fl6->fl6_gre_key = *gre_key; in decode_session6()
3563 fl6->flowi6_proto = nexthdr; in decode_session6()
3569 if (!onlyproto && (nh + offset + 3 < skb->data || in decode_session6()
3570 pskb_may_pull(skb, nh + offset + 3 - skb->data))) { in decode_session6()
3575 fl6->fl6_mh_type = mh->ip6mh_type; in decode_session6()
3577 fl6->flowi6_proto = nexthdr; in decode_session6()
3581 fl6->flowi6_proto = nexthdr; in decode_session6()
3601 return -EAFNOSUPPORT; in __xfrm_decode_session()
3604 return security_xfrm_decode_session(skb, &fl->flowi_secid); in __xfrm_decode_session()
3610 for (; k < sp->len; k++) { in secpath_has_nontransport()
3611 if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) { in secpath_has_nontransport()
3623 struct net *net = dev_net(skb->dev); in __xfrm_policy_check()
3631 int xerr_idx = -1; in __xfrm_policy_check()
3642 if (ifcb->decode_session(skb, family, &r)) { in __xfrm_policy_check()
3664 for (i = sp->len - 1; i >= 0; i--) { in __xfrm_policy_check()
3665 struct xfrm_state *x = sp->xvec[i]; in __xfrm_policy_check()
3666 if (!xfrm_selector_match(&x->sel, &fl, family)) { in __xfrm_policy_check()
3675 if (sk && sk->sk_policy[dir]) { in __xfrm_policy_check()
3692 if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { in __xfrm_policy_check()
3706 WRITE_ONCE(pol->curlft.use_time, ktime_get_real_seconds()); in __xfrm_policy_check()
3711 if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) { in __xfrm_policy_check()
3722 WRITE_ONCE(pols[1]->curlft.use_time, in __xfrm_policy_check()
3729 if (pol->action == XFRM_POLICY_ALLOW) { in __xfrm_policy_check()
3743 pols[pi]->action != XFRM_POLICY_ALLOW) { in __xfrm_policy_check()
3747 if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) { in __xfrm_policy_check()
3751 for (i = 0; i < pols[pi]->xfrm_nr; i++) in __xfrm_policy_check()
3752 tpp[ti++] = &pols[pi]->xfrm_vec[i]; in __xfrm_policy_check()
3770 for (i = xfrm_nr-1, k = 0; i >= 0; i--) { in __xfrm_policy_check()
3773 if (k < -1) in __xfrm_policy_check()
3774 /* "-2 - errored_index" returned */ in __xfrm_policy_check()
3775 xerr_idx = -(2+k); in __xfrm_policy_check()
3787 sp->verified_cnt = k; in __xfrm_policy_check()
3803 struct net *net = dev_net(skb->dev); in __xfrm_route_forward()
3833 /* Code (such as __xfrm4_bundle_create()) sets dst->obsolete in xfrm_dst_check()
3835 * get validated by dst_ops->check on every use. We do this in xfrm_dst_check()
3842 * XFRM dst A --> IPv4 dst X in xfrm_dst_check()
3844 * X is the "xdst->route" of A (X is also the "dst->path" of A in xfrm_dst_check()
3854 if (dst->obsolete < 0 && !stale_bundle(dst)) in xfrm_dst_check()
3867 while ((dst = xfrm_dst_child(dst)) && dst->xfrm && dst->dev == dev) { in xfrm_dst_ifdown()
3868 dst->dev = blackhole_netdev; in xfrm_dst_ifdown()
3869 dev_hold(dst->dev); in xfrm_dst_ifdown()
3882 if (dst->obsolete) in xfrm_negative_advice()
3888 while (nr--) { in xfrm_init_pmtu()
3893 dst = &xdst->u.dst; in xfrm_init_pmtu()
3895 xdst->child_mtu_cached = pmtu; in xfrm_init_pmtu()
3897 pmtu = xfrm_state_mtu(dst->xfrm, pmtu); in xfrm_init_pmtu()
3899 route_mtu_cached = dst_mtu(xdst->route); in xfrm_init_pmtu()
3900 xdst->route_mtu_cached = route_mtu_cached; in xfrm_init_pmtu()
3916 struct dst_entry *dst = &first->u.dst; in xfrm_bundle_ok()
3921 if (!dst_check(xfrm_dst_path(dst), ((struct xfrm_dst *)dst)->path_cookie) || in xfrm_bundle_ok()
3922 (dst->dev && !netif_running(dst->dev))) in xfrm_bundle_ok()
3925 if (dst->flags & DST_XFRM_QUEUE) in xfrm_bundle_ok()
3932 if (dst->xfrm->km.state != XFRM_STATE_VALID) in xfrm_bundle_ok()
3934 if (xdst->xfrm_genid != dst->xfrm->genid) in xfrm_bundle_ok()
3936 if (xdst->num_pols > 0 && in xfrm_bundle_ok()
3937 xdst->policy_genid != atomic_read(&xdst->pols[0]->genid)) in xfrm_bundle_ok()
3943 if (xdst->child_mtu_cached != mtu) { in xfrm_bundle_ok()
3945 xdst->child_mtu_cached = mtu; in xfrm_bundle_ok()
3948 if (!dst_check(xdst->route, xdst->route_cookie)) in xfrm_bundle_ok()
3950 mtu = dst_mtu(xdst->route); in xfrm_bundle_ok()
3951 if (xdst->route_mtu_cached != mtu) { in xfrm_bundle_ok()
3953 xdst->route_mtu_cached = mtu; in xfrm_bundle_ok()
3957 } while (dst->xfrm); in xfrm_bundle_ok()
3962 xdst = bundle[start_from - 1]; in xfrm_bundle_ok()
3963 mtu = xdst->child_mtu_cached; in xfrm_bundle_ok()
3964 while (start_from--) { in xfrm_bundle_ok()
3965 dst = &xdst->u.dst; in xfrm_bundle_ok()
3967 mtu = xfrm_state_mtu(dst->xfrm, mtu); in xfrm_bundle_ok()
3968 if (mtu > xdst->route_mtu_cached) in xfrm_bundle_ok()
3969 mtu = xdst->route_mtu_cached; in xfrm_bundle_ok()
3974 xdst = bundle[start_from - 1]; in xfrm_bundle_ok()
3975 xdst->child_mtu_cached = mtu; in xfrm_bundle_ok()
3996 while (dst->xfrm) { in xfrm_get_dst_nexthop()
3997 const struct xfrm_state *xfrm = dst->xfrm; in xfrm_get_dst_nexthop()
4001 if (xfrm->props.mode == XFRM_MODE_TRANSPORT) in xfrm_get_dst_nexthop()
4003 if (xfrm->type->flags & XFRM_TYPE_REMOTE_COADDR) in xfrm_get_dst_nexthop()
4004 daddr = xfrm->coaddr; in xfrm_get_dst_nexthop()
4005 else if (!(xfrm->type->flags & XFRM_TYPE_LOCAL_COADDR)) in xfrm_get_dst_nexthop()
4006 daddr = &xfrm->id.daddr; in xfrm_get_dst_nexthop()
4019 return path->ops->neigh_lookup(path, skb, daddr); in xfrm_neigh_lookup()
4027 path->ops->confirm_neigh(path, daddr); in xfrm_confirm_neigh()
4035 return -EAFNOSUPPORT; in xfrm_policy_register_afinfo()
4039 err = -EEXIST; in xfrm_policy_register_afinfo()
4041 struct dst_ops *dst_ops = afinfo->dst_ops; in xfrm_policy_register_afinfo()
4042 if (likely(dst_ops->kmem_cachep == NULL)) in xfrm_policy_register_afinfo()
4043 dst_ops->kmem_cachep = xfrm_dst_cache; in xfrm_policy_register_afinfo()
4044 if (likely(dst_ops->check == NULL)) in xfrm_policy_register_afinfo()
4045 dst_ops->check = xfrm_dst_check; in xfrm_policy_register_afinfo()
4046 if (likely(dst_ops->default_advmss == NULL)) in xfrm_policy_register_afinfo()
4047 dst_ops->default_advmss = xfrm_default_advmss; in xfrm_policy_register_afinfo()
4048 if (likely(dst_ops->mtu == NULL)) in xfrm_policy_register_afinfo()
4049 dst_ops->mtu = xfrm_mtu; in xfrm_policy_register_afinfo()
4050 if (likely(dst_ops->negative_advice == NULL)) in xfrm_policy_register_afinfo()
4051 dst_ops->negative_advice = xfrm_negative_advice; in xfrm_policy_register_afinfo()
4052 if (likely(dst_ops->link_failure == NULL)) in xfrm_policy_register_afinfo()
4053 dst_ops->link_failure = xfrm_link_failure; in xfrm_policy_register_afinfo()
4054 if (likely(dst_ops->neigh_lookup == NULL)) in xfrm_policy_register_afinfo()
4055 dst_ops->neigh_lookup = xfrm_neigh_lookup; in xfrm_policy_register_afinfo()
4056 if (likely(!dst_ops->confirm_neigh)) in xfrm_policy_register_afinfo()
4057 dst_ops->confirm_neigh = xfrm_confirm_neigh; in xfrm_policy_register_afinfo()
4068 struct dst_ops *dst_ops = afinfo->dst_ops; in xfrm_policy_unregister_afinfo()
4080 dst_ops->kmem_cachep = NULL; in xfrm_policy_unregister_afinfo()
4081 dst_ops->check = NULL; in xfrm_policy_unregister_afinfo()
4082 dst_ops->negative_advice = NULL; in xfrm_policy_unregister_afinfo()
4083 dst_ops->link_failure = NULL; in xfrm_policy_unregister_afinfo()
4106 net->mib.xfrm_statistics = alloc_percpu(struct linux_xfrm_mib); in xfrm_statistics_init()
4107 if (!net->mib.xfrm_statistics) in xfrm_statistics_init()
4108 return -ENOMEM; in xfrm_statistics_init()
4111 free_percpu(net->mib.xfrm_statistics); in xfrm_statistics_init()
4118 free_percpu(net->mib.xfrm_statistics); in xfrm_statistics_fini()
4146 hmask = 8 - 1; in xfrm_policy_init()
4149 net->xfrm.policy_byidx = xfrm_hash_alloc(sz); in xfrm_policy_init()
4150 if (!net->xfrm.policy_byidx) in xfrm_policy_init()
4152 net->xfrm.policy_idx_hmask = hmask; in xfrm_policy_init()
4157 net->xfrm.policy_count[dir] = 0; in xfrm_policy_init()
4158 net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0; in xfrm_policy_init()
4159 INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]); in xfrm_policy_init()
4161 htab = &net->xfrm.policy_bydst[dir]; in xfrm_policy_init()
4162 htab->table = xfrm_hash_alloc(sz); in xfrm_policy_init()
4163 if (!htab->table) in xfrm_policy_init()
4165 htab->hmask = hmask; in xfrm_policy_init()
4166 htab->dbits4 = 32; in xfrm_policy_init()
4167 htab->sbits4 = 32; in xfrm_policy_init()
4168 htab->dbits6 = 128; in xfrm_policy_init()
4169 htab->sbits6 = 128; in xfrm_policy_init()
4171 net->xfrm.policy_hthresh.lbits4 = 32; in xfrm_policy_init()
4172 net->xfrm.policy_hthresh.rbits4 = 32; in xfrm_policy_init()
4173 net->xfrm.policy_hthresh.lbits6 = 128; in xfrm_policy_init()
4174 net->xfrm.policy_hthresh.rbits6 = 128; in xfrm_policy_init()
4176 seqlock_init(&net->xfrm.policy_hthresh.lock); in xfrm_policy_init()
4178 INIT_LIST_HEAD(&net->xfrm.policy_all); in xfrm_policy_init()
4179 INIT_LIST_HEAD(&net->xfrm.inexact_bins); in xfrm_policy_init()
4180 INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize); in xfrm_policy_init()
4181 INIT_WORK(&net->xfrm.policy_hthresh.work, xfrm_hash_rebuild); in xfrm_policy_init()
4185 for (dir--; dir >= 0; dir--) { in xfrm_policy_init()
4188 htab = &net->xfrm.policy_bydst[dir]; in xfrm_policy_init()
4189 xfrm_hash_free(htab->table, sz); in xfrm_policy_init()
4191 xfrm_hash_free(net->xfrm.policy_byidx, sz); in xfrm_policy_init()
4193 return -ENOMEM; in xfrm_policy_init()
4202 flush_work(&net->xfrm.policy_hash_work); in xfrm_policy_fini()
4208 WARN_ON(!list_empty(&net->xfrm.policy_all)); in xfrm_policy_fini()
4213 WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir])); in xfrm_policy_fini()
4215 htab = &net->xfrm.policy_bydst[dir]; in xfrm_policy_fini()
4216 sz = (htab->hmask + 1) * sizeof(struct hlist_head); in xfrm_policy_fini()
4217 WARN_ON(!hlist_empty(htab->table)); in xfrm_policy_fini()
4218 xfrm_hash_free(htab->table, sz); in xfrm_policy_fini()
4221 sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head); in xfrm_policy_fini()
4222 WARN_ON(!hlist_empty(net->xfrm.policy_byidx)); in xfrm_policy_fini()
4223 xfrm_hash_free(net->xfrm.policy_byidx, sz); in xfrm_policy_fini()
4225 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_fini()
4226 list_for_each_entry_safe(b, t, &net->xfrm.inexact_bins, inexact_bins) in xfrm_policy_fini()
4228 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_fini()
4235 /* Initialize the per-net locks here */ in xfrm_net_init()
4236 spin_lock_init(&net->xfrm.xfrm_state_lock); in xfrm_net_init()
4237 spin_lock_init(&net->xfrm.xfrm_policy_lock); in xfrm_net_init()
4238 seqcount_spinlock_init(&net->xfrm.xfrm_policy_hash_generation, &net->xfrm.xfrm_policy_lock); in xfrm_net_init()
4239 mutex_init(&net->xfrm.xfrm_cfg_mutex); in xfrm_net_init()
4240 net->xfrm.policy_default[XFRM_POLICY_IN] = XFRM_USERPOLICY_ACCEPT; in xfrm_net_init()
4241 net->xfrm.policy_default[XFRM_POLICY_FWD] = XFRM_USERPOLICY_ACCEPT; in xfrm_net_init()
4242 net->xfrm.policy_default[XFRM_POLICY_OUT] = XFRM_USERPOLICY_ACCEPT; in xfrm_net_init()
4297 struct xfrm_sec_ctx *ctx = xp->security; in xfrm_audit_common_policyinfo()
4298 struct xfrm_selector *sel = &xp->selector; in xfrm_audit_common_policyinfo()
4302 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str); in xfrm_audit_common_policyinfo()
4304 switch (sel->family) { in xfrm_audit_common_policyinfo()
4306 audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4); in xfrm_audit_common_policyinfo()
4307 if (sel->prefixlen_s != 32) in xfrm_audit_common_policyinfo()
4309 sel->prefixlen_s); in xfrm_audit_common_policyinfo()
4310 audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4); in xfrm_audit_common_policyinfo()
4311 if (sel->prefixlen_d != 32) in xfrm_audit_common_policyinfo()
4313 sel->prefixlen_d); in xfrm_audit_common_policyinfo()
4316 audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6); in xfrm_audit_common_policyinfo()
4317 if (sel->prefixlen_s != 128) in xfrm_audit_common_policyinfo()
4319 sel->prefixlen_s); in xfrm_audit_common_policyinfo()
4320 audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6); in xfrm_audit_common_policyinfo()
4321 if (sel->prefixlen_d != 128) in xfrm_audit_common_policyinfo()
4323 sel->prefixlen_d); in xfrm_audit_common_policyinfo()
4332 audit_buf = xfrm_audit_start("SPD-add"); in xfrm_audit_policy_add()
4347 audit_buf = xfrm_audit_start("SPD-delete"); in xfrm_audit_policy_delete()
4362 if (sel_cmp->proto == IPSEC_ULPROTO_ANY) { in xfrm_migrate_selector_match()
4363 if (sel_tgt->family == sel_cmp->family && in xfrm_migrate_selector_match()
4364 xfrm_addr_equal(&sel_tgt->daddr, &sel_cmp->daddr, in xfrm_migrate_selector_match()
4365 sel_cmp->family) && in xfrm_migrate_selector_match()
4366 xfrm_addr_equal(&sel_tgt->saddr, &sel_cmp->saddr, in xfrm_migrate_selector_match()
4367 sel_cmp->family) && in xfrm_migrate_selector_match()
4368 sel_tgt->prefixlen_d == sel_cmp->prefixlen_d && in xfrm_migrate_selector_match()
4369 sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) { in xfrm_migrate_selector_match()
4387 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_migrate_policy_find()
4388 chain = policy_hash_direct(net, &sel->daddr, &sel->saddr, sel->family, dir); in xfrm_migrate_policy_find()
4390 if ((if_id == 0 || pol->if_id == if_id) && in xfrm_migrate_policy_find()
4391 xfrm_migrate_selector_match(sel, &pol->selector) && in xfrm_migrate_policy_find()
4392 pol->type == type) { in xfrm_migrate_policy_find()
4394 priority = ret->priority; in xfrm_migrate_policy_find()
4398 chain = &net->xfrm.policy_inexact[dir]; in xfrm_migrate_policy_find()
4400 if ((pol->priority >= priority) && ret) in xfrm_migrate_policy_find()
4403 if ((if_id == 0 || pol->if_id == if_id) && in xfrm_migrate_policy_find()
4404 xfrm_migrate_selector_match(sel, &pol->selector) && in xfrm_migrate_policy_find()
4405 pol->type == type) { in xfrm_migrate_policy_find()
4413 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_migrate_policy_find()
4422 if (t->mode == m->mode && t->id.proto == m->proto && in migrate_tmpl_match()
4423 (m->reqid == 0 || t->reqid == m->reqid)) { in migrate_tmpl_match()
4424 switch (t->mode) { in migrate_tmpl_match()
4427 if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr, in migrate_tmpl_match()
4428 m->old_family) && in migrate_tmpl_match()
4429 xfrm_addr_equal(&t->saddr, &m->old_saddr, in migrate_tmpl_match()
4430 m->old_family)) { in migrate_tmpl_match()
4455 write_lock_bh(&pol->lock); in xfrm_policy_migrate()
4456 if (unlikely(pol->walk.dead)) { in xfrm_policy_migrate()
4459 write_unlock_bh(&pol->lock); in xfrm_policy_migrate()
4460 return -ENOENT; in xfrm_policy_migrate()
4463 for (i = 0; i < pol->xfrm_nr; i++) { in xfrm_policy_migrate()
4465 if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i])) in xfrm_policy_migrate()
4468 if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL && in xfrm_policy_migrate()
4469 pol->xfrm_vec[i].mode != XFRM_MODE_BEET) in xfrm_policy_migrate()
4472 memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr, in xfrm_policy_migrate()
4473 sizeof(pol->xfrm_vec[i].id.daddr)); in xfrm_policy_migrate()
4474 memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr, in xfrm_policy_migrate()
4475 sizeof(pol->xfrm_vec[i].saddr)); in xfrm_policy_migrate()
4476 pol->xfrm_vec[i].encap_family = mp->new_family; in xfrm_policy_migrate()
4478 atomic_inc(&pol->genid); in xfrm_policy_migrate()
4482 write_unlock_bh(&pol->lock); in xfrm_policy_migrate()
4485 return -ENODATA; in xfrm_policy_migrate()
4497 return -EINVAL; in xfrm_migrate_check()
4504 return -EINVAL; in xfrm_migrate_check()
4518 return -EINVAL; in xfrm_migrate_check()
4539 /* Stage 0 - sanity checks */ in xfrm_migrate()
4546 err = -EINVAL; in xfrm_migrate()
4550 /* Stage 1 - find policy */ in xfrm_migrate()
4554 err = -ENOENT; in xfrm_migrate()
4558 /* Stage 2 - find and update state(s) */ in xfrm_migrate()
4568 err = -ENODATA; in xfrm_migrate()
4574 /* Stage 3 - update policy */ in xfrm_migrate()
4579 /* Stage 4 - delete old state(s) */ in xfrm_migrate()
4585 /* Stage 5 - announce */ in xfrm_migrate()