Lines Matching +full:p1 +full:- +full:retry +full:- +full:params
1 // SPDX-License-Identifier: GPL-2.0-only
12 * Split up af-specific portion
80 * +---- root_d: sorted by daddr:prefix
84 * | +- root: sorted by saddr/prefix
92 * | +- coarse policies and all any:daddr policies
94 * +---- root_s: sorted by saddr:prefix
102 * +---- coarse policies and all any:any policies
105 * 1. any:any list from top-level xfrm_pol_inexact_bin
194 return refcount_inc_not_zero(&policy->refcnt); in xfrm_pol_hold_rcu()
200 const struct flowi4 *fl4 = &fl->u.ip4; in __xfrm4_selector_match()
202 return addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) && in __xfrm4_selector_match()
203 addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) && in __xfrm4_selector_match()
204 !((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) && in __xfrm4_selector_match()
205 !((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) && in __xfrm4_selector_match()
206 (fl4->flowi4_proto == sel->proto || !sel->proto) && in __xfrm4_selector_match()
207 (fl4->flowi4_oif == sel->ifindex || !sel->ifindex); in __xfrm4_selector_match()
213 const struct flowi6 *fl6 = &fl->u.ip6; in __xfrm6_selector_match()
215 return addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) && in __xfrm6_selector_match()
216 addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) && in __xfrm6_selector_match()
217 !((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) && in __xfrm6_selector_match()
218 !((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) && in __xfrm6_selector_match()
219 (fl6->flowi6_proto == sel->proto || !sel->proto) && in __xfrm6_selector_match()
220 (fl6->flowi6_oif == sel->ifindex || !sel->ifindex); in __xfrm6_selector_match()
255 const struct xfrm_dst_lookup_params *params) in __xfrm_dst_lookup() argument
262 return ERR_PTR(-EAFNOSUPPORT); in __xfrm_dst_lookup()
264 dst = afinfo->dst_lookup(params); in __xfrm_dst_lookup()
278 struct xfrm_dst_lookup_params params; in xfrm_dst_lookup() local
280 xfrm_address_t *saddr = &x->props.saddr; in xfrm_dst_lookup()
281 xfrm_address_t *daddr = &x->id.daddr; in xfrm_dst_lookup()
284 if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) { in xfrm_dst_lookup()
285 saddr = x->coaddr; in xfrm_dst_lookup()
288 if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) { in xfrm_dst_lookup()
290 daddr = x->coaddr; in xfrm_dst_lookup()
293 params.net = net; in xfrm_dst_lookup()
294 params.saddr = saddr; in xfrm_dst_lookup()
295 params.daddr = daddr; in xfrm_dst_lookup()
296 params.tos = tos; in xfrm_dst_lookup()
297 params.oif = oif; in xfrm_dst_lookup()
298 params.mark = mark; in xfrm_dst_lookup()
299 params.ipproto = x->id.proto; in xfrm_dst_lookup()
300 if (x->encap) { in xfrm_dst_lookup()
301 switch (x->encap->encap_type) { in xfrm_dst_lookup()
303 params.ipproto = IPPROTO_UDP; in xfrm_dst_lookup()
304 params.uli.ports.sport = x->encap->encap_sport; in xfrm_dst_lookup()
305 params.uli.ports.dport = x->encap->encap_dport; in xfrm_dst_lookup()
308 params.ipproto = IPPROTO_TCP; in xfrm_dst_lookup()
309 params.uli.ports.sport = x->encap->encap_sport; in xfrm_dst_lookup()
310 params.uli.ports.dport = x->encap->encap_dport; in xfrm_dst_lookup()
315 dst = __xfrm_dst_lookup(family, ¶ms); in xfrm_dst_lookup()
329 if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ) in make_jiffies()
330 return MAX_SCHEDULE_TIMEOUT-1; in make_jiffies()
343 read_lock(&xp->lock); in xfrm_policy_timer()
345 if (unlikely(xp->walk.dead)) in xfrm_policy_timer()
348 dir = xfrm_policy_id2dir(xp->index); in xfrm_policy_timer()
350 if (xp->lft.hard_add_expires_seconds) { in xfrm_policy_timer()
351 time64_t tmo = xp->lft.hard_add_expires_seconds + in xfrm_policy_timer()
352 xp->curlft.add_time - now; in xfrm_policy_timer()
358 if (xp->lft.hard_use_expires_seconds) { in xfrm_policy_timer()
359 time64_t tmo = xp->lft.hard_use_expires_seconds + in xfrm_policy_timer()
360 (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now; in xfrm_policy_timer()
366 if (xp->lft.soft_add_expires_seconds) { in xfrm_policy_timer()
367 time64_t tmo = xp->lft.soft_add_expires_seconds + in xfrm_policy_timer()
368 xp->curlft.add_time - now; in xfrm_policy_timer()
376 if (xp->lft.soft_use_expires_seconds) { in xfrm_policy_timer()
377 time64_t tmo = xp->lft.soft_use_expires_seconds + in xfrm_policy_timer()
378 (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now; in xfrm_policy_timer()
390 !mod_timer(&xp->timer, jiffies + make_jiffies(next))) in xfrm_policy_timer()
394 read_unlock(&xp->lock); in xfrm_policy_timer()
399 read_unlock(&xp->lock); in xfrm_policy_timer()
416 write_pnet(&policy->xp_net, net); in xfrm_policy_alloc()
417 INIT_LIST_HEAD(&policy->walk.all); in xfrm_policy_alloc()
418 INIT_HLIST_NODE(&policy->bydst_inexact_list); in xfrm_policy_alloc()
419 INIT_HLIST_NODE(&policy->bydst); in xfrm_policy_alloc()
420 INIT_HLIST_NODE(&policy->byidx); in xfrm_policy_alloc()
421 rwlock_init(&policy->lock); in xfrm_policy_alloc()
422 refcount_set(&policy->refcnt, 1); in xfrm_policy_alloc()
423 skb_queue_head_init(&policy->polq.hold_queue); in xfrm_policy_alloc()
424 timer_setup(&policy->timer, xfrm_policy_timer, 0); in xfrm_policy_alloc()
425 timer_setup(&policy->polq.hold_timer, in xfrm_policy_alloc()
436 security_xfrm_policy_free(policy->security); in xfrm_policy_destroy_rcu()
444 BUG_ON(!policy->walk.dead); in xfrm_policy_destroy()
446 if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer)) in xfrm_policy_destroy()
450 call_rcu(&policy->rcu, xfrm_policy_destroy_rcu); in xfrm_policy_destroy()
462 write_lock_bh(&policy->lock); in xfrm_policy_kill()
463 policy->walk.dead = 1; in xfrm_policy_kill()
464 write_unlock_bh(&policy->lock); in xfrm_policy_kill()
466 atomic_inc(&policy->genid); in xfrm_policy_kill()
468 if (del_timer(&policy->polq.hold_timer)) in xfrm_policy_kill()
470 skb_queue_purge(&policy->polq.hold_queue); in xfrm_policy_kill()
472 if (del_timer(&policy->timer)) in xfrm_policy_kill()
482 return __idx_hash(index, net->xfrm.policy_idx_hmask); in idx_hash()
492 *dbits = net->xfrm.policy_bydst[dir].dbits4; in __get_hash_thresh()
493 *sbits = net->xfrm.policy_bydst[dir].sbits4; in __get_hash_thresh()
497 *dbits = net->xfrm.policy_bydst[dir].dbits6; in __get_hash_thresh()
498 *sbits = net->xfrm.policy_bydst[dir].sbits6; in __get_hash_thresh()
511 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; in policy_hash_bysel()
522 return rcu_dereference_check(net->xfrm.policy_bydst[dir].table, in policy_hash_bysel()
523 lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash; in policy_hash_bysel()
531 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; in policy_hash_direct()
539 return rcu_dereference_check(net->xfrm.policy_bydst[dir].table, in policy_hash_direct()
540 lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash; in policy_hash_direct()
559 __get_hash_thresh(net, pol->family, dir, &dbits, &sbits); in xfrm_dst_hash_transfer()
560 h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr, in xfrm_dst_hash_transfer()
561 pol->family, nhashmask, dbits, sbits); in xfrm_dst_hash_transfer()
562 if (!entry0 || pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { in xfrm_dst_hash_transfer()
563 hlist_del_rcu(&pol->bydst); in xfrm_dst_hash_transfer()
564 hlist_add_head_rcu(&pol->bydst, ndsttable + h); in xfrm_dst_hash_transfer()
569 hlist_del_rcu(&pol->bydst); in xfrm_dst_hash_transfer()
570 hlist_add_behind_rcu(&pol->bydst, entry0); in xfrm_dst_hash_transfer()
572 entry0 = &pol->bydst; in xfrm_dst_hash_transfer()
590 h = __idx_hash(pol->index, nhashmask); in xfrm_idx_hash_transfer()
591 hlist_add_head(&pol->byidx, nidxtable+h); in xfrm_idx_hash_transfer()
597 return ((old_hmask + 1) << 1) - 1; in xfrm_new_hash_mask()
602 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; in xfrm_bydst_resize()
612 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_bydst_resize()
613 write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); in xfrm_bydst_resize()
615 odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table, in xfrm_bydst_resize()
616 lockdep_is_held(&net->xfrm.xfrm_policy_lock)); in xfrm_bydst_resize()
618 for (i = hmask; i >= 0; i--) in xfrm_bydst_resize()
621 rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst); in xfrm_bydst_resize()
622 net->xfrm.policy_bydst[dir].hmask = nhashmask; in xfrm_bydst_resize()
624 write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation); in xfrm_bydst_resize()
625 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_bydst_resize()
634 unsigned int hmask = net->xfrm.policy_idx_hmask; in xfrm_byidx_resize()
637 struct hlist_head *oidx = net->xfrm.policy_byidx; in xfrm_byidx_resize()
644 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_byidx_resize()
646 for (i = hmask; i >= 0; i--) in xfrm_byidx_resize()
649 net->xfrm.policy_byidx = nidx; in xfrm_byidx_resize()
650 net->xfrm.policy_idx_hmask = nhashmask; in xfrm_byidx_resize()
652 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_byidx_resize()
659 unsigned int cnt = net->xfrm.policy_count[dir]; in xfrm_bydst_should_resize()
660 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; in xfrm_bydst_should_resize()
674 unsigned int hmask = net->xfrm.policy_idx_hmask; in xfrm_byidx_should_resize()
685 si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN]; in xfrm_spd_getinfo()
686 si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT]; in xfrm_spd_getinfo()
687 si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD]; in xfrm_spd_getinfo()
688 si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX]; in xfrm_spd_getinfo()
689 si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX]; in xfrm_spd_getinfo()
690 si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX]; in xfrm_spd_getinfo()
691 si->spdhcnt = net->xfrm.policy_idx_hmask; in xfrm_spd_getinfo()
692 si->spdhmcnt = xfrm_policy_hashmax; in xfrm_spd_getinfo()
724 .family = pol->family, in xfrm_policy_inexact_alloc_bin()
725 .type = pol->type, in xfrm_policy_inexact_alloc_bin()
727 .if_id = pol->if_id, in xfrm_policy_inexact_alloc_bin()
731 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_alloc_bin()
743 bin->k = k; in xfrm_policy_inexact_alloc_bin()
744 INIT_HLIST_HEAD(&bin->hhead); in xfrm_policy_inexact_alloc_bin()
745 bin->root_d = RB_ROOT; in xfrm_policy_inexact_alloc_bin()
746 bin->root_s = RB_ROOT; in xfrm_policy_inexact_alloc_bin()
747 seqcount_spinlock_init(&bin->count, &net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_alloc_bin()
750 &bin->k, &bin->head, in xfrm_policy_inexact_alloc_bin()
753 list_add(&bin->inexact_bins, &net->xfrm.inexact_bins); in xfrm_policy_inexact_alloc_bin()
784 addr = &policy->selector.saddr; in xfrm_policy_inexact_insert_use_any_list()
785 prefixlen = policy->selector.prefixlen_s; in xfrm_policy_inexact_insert_use_any_list()
788 policy->family, in xfrm_policy_inexact_insert_use_any_list()
790 addr = &policy->selector.daddr; in xfrm_policy_inexact_insert_use_any_list()
791 prefixlen = policy->selector.prefixlen_d; in xfrm_policy_inexact_insert_use_any_list()
793 policy->family, in xfrm_policy_inexact_insert_use_any_list()
801 node->addr = *addr; in xfrm_pol_inexact_node_init()
802 node->prefixlen = prefixlen; in xfrm_pol_inexact_node_init()
829 mask = ~0U << (32 - prefixlen); in xfrm_policy_addr_delta()
830 ma = ntohl(a->a4) & mask; in xfrm_policy_addr_delta()
831 mb = ntohl(b->a4) & mask; in xfrm_policy_addr_delta()
833 delta = -1; in xfrm_policy_addr_delta()
842 delta = memcmp(a->a6, b->a6, pdw << 2); in xfrm_policy_addr_delta()
847 mask = ~0U << (32 - pbi); in xfrm_policy_addr_delta()
848 ma = ntohl(a->a6[pdw]) & mask; in xfrm_policy_addr_delta()
849 mb = ntohl(b->a6[pdw]) & mask; in xfrm_policy_addr_delta()
851 delta = -1; in xfrm_policy_addr_delta()
873 list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) { in xfrm_policy_inexact_list_reinsert()
877 if (policy->walk.dead || !policy->bydst_reinsert) in xfrm_policy_inexact_list_reinsert()
880 WARN_ON_ONCE(policy->family != family); in xfrm_policy_inexact_list_reinsert()
882 policy->bydst_reinsert = false; in xfrm_policy_inexact_list_reinsert()
883 hlist_for_each_entry(p, &n->hhead, bydst) { in xfrm_policy_inexact_list_reinsert()
884 if (policy->priority > p->priority) in xfrm_policy_inexact_list_reinsert()
885 newpos = &p->bydst; in xfrm_policy_inexact_list_reinsert()
886 else if (policy->priority == p->priority && in xfrm_policy_inexact_list_reinsert()
887 policy->pos > p->pos) in xfrm_policy_inexact_list_reinsert()
888 newpos = &p->bydst; in xfrm_policy_inexact_list_reinsert()
893 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_policy_inexact_list_reinsert()
894 hlist_add_behind_rcu(&policy->bydst, newpos); in xfrm_policy_inexact_list_reinsert()
896 hlist_add_head_rcu(&policy->bydst, &n->hhead); in xfrm_policy_inexact_list_reinsert()
906 matches_s = xfrm_policy_addr_delta(&policy->selector.saddr, in xfrm_policy_inexact_list_reinsert()
907 &n->addr, in xfrm_policy_inexact_list_reinsert()
908 n->prefixlen, in xfrm_policy_inexact_list_reinsert()
910 matches_d = xfrm_policy_addr_delta(&policy->selector.daddr, in xfrm_policy_inexact_list_reinsert()
911 &n->addr, in xfrm_policy_inexact_list_reinsert()
912 n->prefixlen, in xfrm_policy_inexact_list_reinsert()
935 WARN_ON_ONCE(!RB_EMPTY_ROOT(&n->root)); in xfrm_policy_inexact_node_reinsert()
938 p = &new->rb_node; in xfrm_policy_inexact_node_reinsert()
946 prefixlen = min(node->prefixlen, n->prefixlen); in xfrm_policy_inexact_node_reinsert()
948 delta = xfrm_policy_addr_delta(&n->addr, &node->addr, in xfrm_policy_inexact_node_reinsert()
951 p = &parent->rb_left; in xfrm_policy_inexact_node_reinsert()
953 p = &parent->rb_right; in xfrm_policy_inexact_node_reinsert()
955 bool same_prefixlen = node->prefixlen == n->prefixlen; in xfrm_policy_inexact_node_reinsert()
958 hlist_for_each_entry(tmp, &n->hhead, bydst) { in xfrm_policy_inexact_node_reinsert()
959 tmp->bydst_reinsert = true; in xfrm_policy_inexact_node_reinsert()
960 hlist_del_rcu(&tmp->bydst); in xfrm_policy_inexact_node_reinsert()
963 node->prefixlen = prefixlen; in xfrm_policy_inexact_node_reinsert()
979 rb_link_node_rcu(&n->node, parent, p); in xfrm_policy_inexact_node_reinsert()
980 rb_insert_color(&n->node, new); in xfrm_policy_inexact_node_reinsert()
993 /* To-be-merged node v has a subtree. in xfrm_policy_inexact_node_merge()
995 * Dismantle it and insert its nodes to n->root. in xfrm_policy_inexact_node_merge()
997 while ((rnode = rb_first(&v->root)) != NULL) { in xfrm_policy_inexact_node_merge()
999 rb_erase(&node->node, &v->root); in xfrm_policy_inexact_node_merge()
1000 xfrm_policy_inexact_node_reinsert(net, node, &n->root, in xfrm_policy_inexact_node_merge()
1004 hlist_for_each_entry(tmp, &v->hhead, bydst) { in xfrm_policy_inexact_node_merge()
1005 tmp->bydst_reinsert = true; in xfrm_policy_inexact_node_merge()
1006 hlist_del_rcu(&tmp->bydst); in xfrm_policy_inexact_node_merge()
1022 p = &root->rb_node; in xfrm_policy_inexact_insert_node()
1029 delta = xfrm_policy_addr_delta(addr, &node->addr, in xfrm_policy_inexact_insert_node()
1030 node->prefixlen, in xfrm_policy_inexact_insert_node()
1032 if (delta == 0 && prefixlen >= node->prefixlen) { in xfrm_policy_inexact_insert_node()
1038 p = &parent->rb_left; in xfrm_policy_inexact_insert_node()
1040 p = &parent->rb_right; in xfrm_policy_inexact_insert_node()
1042 if (prefixlen < node->prefixlen) { in xfrm_policy_inexact_insert_node()
1043 delta = xfrm_policy_addr_delta(addr, &node->addr, in xfrm_policy_inexact_insert_node()
1050 * to be removed and re-inserted with the smaller in xfrm_policy_inexact_insert_node()
1054 rb_erase(&node->node, root); in xfrm_policy_inexact_insert_node()
1062 * prefixlen. Merge the to-be-reinserted in xfrm_policy_inexact_insert_node()
1071 p = &root->rb_node; in xfrm_policy_inexact_insert_node()
1083 rb_link_node_rcu(&node->node, parent, p); in xfrm_policy_inexact_insert_node()
1084 rb_insert_color(&node->node, root); in xfrm_policy_inexact_insert_node()
1097 xfrm_policy_inexact_gc_tree(&node->root, rm); in xfrm_policy_inexact_gc_tree()
1100 if (!hlist_empty(&node->hhead) || !RB_EMPTY_ROOT(&node->root)) { in xfrm_policy_inexact_gc_tree()
1105 rb_erase(&node->node, r); in xfrm_policy_inexact_gc_tree()
1112 write_seqcount_begin(&b->count); in __xfrm_policy_inexact_prune_bin()
1113 xfrm_policy_inexact_gc_tree(&b->root_d, net_exit); in __xfrm_policy_inexact_prune_bin()
1114 xfrm_policy_inexact_gc_tree(&b->root_s, net_exit); in __xfrm_policy_inexact_prune_bin()
1115 write_seqcount_end(&b->count); in __xfrm_policy_inexact_prune_bin()
1117 if (!RB_EMPTY_ROOT(&b->root_d) || !RB_EMPTY_ROOT(&b->root_s) || in __xfrm_policy_inexact_prune_bin()
1118 !hlist_empty(&b->hhead)) { in __xfrm_policy_inexact_prune_bin()
1123 if (rhashtable_remove_fast(&xfrm_policy_inexact_table, &b->head, in __xfrm_policy_inexact_prune_bin()
1125 list_del(&b->inexact_bins); in __xfrm_policy_inexact_prune_bin()
1132 struct net *net = read_pnet(&b->k.net); in xfrm_policy_inexact_prune_bin()
1134 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_prune_bin()
1136 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_prune_bin()
1143 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in __xfrm_policy_inexact_flush()
1145 list_for_each_entry_safe(bin, t, &net->xfrm.inexact_bins, inexact_bins) in __xfrm_policy_inexact_flush()
1157 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_alloc_chain()
1160 return &bin->hhead; in xfrm_policy_inexact_alloc_chain()
1162 if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.daddr, in xfrm_policy_inexact_alloc_chain()
1163 policy->family, in xfrm_policy_inexact_alloc_chain()
1164 policy->selector.prefixlen_d)) { in xfrm_policy_inexact_alloc_chain()
1165 write_seqcount_begin(&bin->count); in xfrm_policy_inexact_alloc_chain()
1167 &bin->root_s, in xfrm_policy_inexact_alloc_chain()
1168 &policy->selector.saddr, in xfrm_policy_inexact_alloc_chain()
1169 policy->family, in xfrm_policy_inexact_alloc_chain()
1170 policy->selector.prefixlen_s, in xfrm_policy_inexact_alloc_chain()
1172 write_seqcount_end(&bin->count); in xfrm_policy_inexact_alloc_chain()
1176 return &n->hhead; in xfrm_policy_inexact_alloc_chain()
1180 write_seqcount_begin(&bin->count); in xfrm_policy_inexact_alloc_chain()
1182 &bin->root_d, in xfrm_policy_inexact_alloc_chain()
1183 &policy->selector.daddr, in xfrm_policy_inexact_alloc_chain()
1184 policy->family, in xfrm_policy_inexact_alloc_chain()
1185 policy->selector.prefixlen_d, dir); in xfrm_policy_inexact_alloc_chain()
1186 write_seqcount_end(&bin->count); in xfrm_policy_inexact_alloc_chain()
1191 if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.saddr, in xfrm_policy_inexact_alloc_chain()
1192 policy->family, in xfrm_policy_inexact_alloc_chain()
1193 policy->selector.prefixlen_s)) in xfrm_policy_inexact_alloc_chain()
1194 return &n->hhead; in xfrm_policy_inexact_alloc_chain()
1196 write_seqcount_begin(&bin->count); in xfrm_policy_inexact_alloc_chain()
1198 &n->root, in xfrm_policy_inexact_alloc_chain()
1199 &policy->selector.saddr, in xfrm_policy_inexact_alloc_chain()
1200 policy->family, in xfrm_policy_inexact_alloc_chain()
1201 policy->selector.prefixlen_s, dir); in xfrm_policy_inexact_alloc_chain()
1202 write_seqcount_end(&bin->count); in xfrm_policy_inexact_alloc_chain()
1206 return &n->hhead; in xfrm_policy_inexact_alloc_chain()
1219 return ERR_PTR(-ENOMEM); in xfrm_policy_inexact_insert()
1222 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_insert()
1227 return ERR_PTR(-ENOMEM); in xfrm_policy_inexact_insert()
1233 return ERR_PTR(-EEXIST); in xfrm_policy_inexact_insert()
1236 chain = &net->xfrm.policy_inexact[dir]; in xfrm_policy_inexact_insert()
1264 seq = read_seqbegin(&net->xfrm.policy_hthresh.lock); in xfrm_hash_rebuild()
1266 lbits4 = net->xfrm.policy_hthresh.lbits4; in xfrm_hash_rebuild()
1267 rbits4 = net->xfrm.policy_hthresh.rbits4; in xfrm_hash_rebuild()
1268 lbits6 = net->xfrm.policy_hthresh.lbits6; in xfrm_hash_rebuild()
1269 rbits6 = net->xfrm.policy_hthresh.rbits6; in xfrm_hash_rebuild()
1270 } while (read_seqretry(&net->xfrm.policy_hthresh.lock, seq)); in xfrm_hash_rebuild()
1272 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_hash_rebuild()
1273 write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); in xfrm_hash_rebuild()
1278 list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) { in xfrm_hash_rebuild()
1282 if (policy->walk.dead) in xfrm_hash_rebuild()
1285 dir = xfrm_policy_id2dir(policy->index); in xfrm_hash_rebuild()
1290 if (policy->family == AF_INET) { in xfrm_hash_rebuild()
1298 if (policy->family == AF_INET) { in xfrm_hash_rebuild()
1307 if (policy->selector.prefixlen_d < dbits || in xfrm_hash_rebuild()
1308 policy->selector.prefixlen_s < sbits) in xfrm_hash_rebuild()
1324 &net->xfrm.policy_inexact[dir], in xfrm_hash_rebuild()
1326 hlist_del_rcu(&policy->bydst); in xfrm_hash_rebuild()
1327 hlist_del_init(&policy->bydst_inexact_list); in xfrm_hash_rebuild()
1330 hmask = net->xfrm.policy_bydst[dir].hmask; in xfrm_hash_rebuild()
1331 odst = net->xfrm.policy_bydst[dir].table; in xfrm_hash_rebuild()
1332 for (i = hmask; i >= 0; i--) { in xfrm_hash_rebuild()
1334 hlist_del_rcu(&policy->bydst); in xfrm_hash_rebuild()
1338 net->xfrm.policy_bydst[dir].dbits4 = rbits4; in xfrm_hash_rebuild()
1339 net->xfrm.policy_bydst[dir].sbits4 = lbits4; in xfrm_hash_rebuild()
1340 net->xfrm.policy_bydst[dir].dbits6 = rbits6; in xfrm_hash_rebuild()
1341 net->xfrm.policy_bydst[dir].sbits6 = lbits6; in xfrm_hash_rebuild()
1344 net->xfrm.policy_bydst[dir].dbits4 = lbits4; in xfrm_hash_rebuild()
1345 net->xfrm.policy_bydst[dir].sbits4 = rbits4; in xfrm_hash_rebuild()
1346 net->xfrm.policy_bydst[dir].dbits6 = lbits6; in xfrm_hash_rebuild()
1347 net->xfrm.policy_bydst[dir].sbits6 = rbits6; in xfrm_hash_rebuild()
1351 /* re-insert all policies by order of creation */ in xfrm_hash_rebuild()
1352 list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) { in xfrm_hash_rebuild()
1353 if (policy->walk.dead) in xfrm_hash_rebuild()
1355 dir = xfrm_policy_id2dir(policy->index); in xfrm_hash_rebuild()
1361 chain = policy_hash_bysel(net, &policy->selector, in xfrm_hash_rebuild()
1362 policy->family, dir); in xfrm_hash_rebuild()
1372 if (policy->priority >= pol->priority) in xfrm_hash_rebuild()
1373 newpos = &pol->bydst; in xfrm_hash_rebuild()
1377 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_hash_rebuild()
1378 hlist_add_behind_rcu(&policy->bydst, newpos); in xfrm_hash_rebuild()
1380 hlist_add_head_rcu(&policy->bydst, chain); in xfrm_hash_rebuild()
1385 write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation); in xfrm_hash_rebuild()
1386 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_hash_rebuild()
1393 schedule_work(&net->xfrm.policy_hthresh.work); in xfrm_policy_hash_rebuild()
1408 idx = (net->xfrm.idx_generator | dir); in xfrm_gen_index()
1409 net->xfrm.idx_generator += 8; in xfrm_gen_index()
1417 list = net->xfrm.policy_byidx + idx_hash(net, idx); in xfrm_gen_index()
1420 if (p->index == idx) { in xfrm_gen_index()
1432 u32 *p1 = (u32 *) s1; in selector_cmp() local
1438 if (p1[i] != p2[i]) in selector_cmp()
1448 struct xfrm_policy_queue *pq = &old->polq; in xfrm_policy_requeue()
1451 if (skb_queue_empty(&pq->hold_queue)) in xfrm_policy_requeue()
1456 spin_lock_bh(&pq->hold_queue.lock); in xfrm_policy_requeue()
1457 skb_queue_splice_init(&pq->hold_queue, &list); in xfrm_policy_requeue()
1458 if (del_timer(&pq->hold_timer)) in xfrm_policy_requeue()
1460 spin_unlock_bh(&pq->hold_queue.lock); in xfrm_policy_requeue()
1462 pq = &new->polq; in xfrm_policy_requeue()
1464 spin_lock_bh(&pq->hold_queue.lock); in xfrm_policy_requeue()
1465 skb_queue_splice(&list, &pq->hold_queue); in xfrm_policy_requeue()
1466 pq->timeout = XFRM_QUEUE_TMO_MIN; in xfrm_policy_requeue()
1467 if (!mod_timer(&pq->hold_timer, jiffies)) in xfrm_policy_requeue()
1469 spin_unlock_bh(&pq->hold_queue.lock); in xfrm_policy_requeue()
1475 return mark->v == pol->mark.v && mark->m == pol->mark.m; in xfrm_policy_mark_match()
1481 u32 a = k->type << 24 | k->dir << 16 | k->family; in xfrm_pol_bin_key()
1483 return jhash_3words(a, k->if_id, net_hash_mix(read_pnet(&k->net)), in xfrm_pol_bin_key()
1491 return xfrm_pol_bin_key(&b->k, 0, seed); in xfrm_pol_bin_obj()
1497 const struct xfrm_pol_inexact_key *key = arg->key; in xfrm_pol_bin_cmp()
1501 if (!net_eq(read_pnet(&b->k.net), read_pnet(&key->net))) in xfrm_pol_bin_cmp()
1502 return -1; in xfrm_pol_bin_cmp()
1504 ret = b->k.dir ^ key->dir; in xfrm_pol_bin_cmp()
1508 ret = b->k.type ^ key->type; in xfrm_pol_bin_cmp()
1512 ret = b->k.family ^ key->family; in xfrm_pol_bin_cmp()
1516 return b->k.if_id ^ key->if_id; in xfrm_pol_bin_cmp()
1535 if (pol->type == policy->type && in xfrm_policy_insert_inexact_list()
1536 pol->if_id == policy->if_id && in xfrm_policy_insert_inexact_list()
1537 !selector_cmp(&pol->selector, &policy->selector) && in xfrm_policy_insert_inexact_list()
1538 xfrm_policy_mark_match(&policy->mark, pol) && in xfrm_policy_insert_inexact_list()
1539 xfrm_sec_ctx_match(pol->security, policy->security) && in xfrm_policy_insert_inexact_list()
1542 if (policy->priority > pol->priority) in xfrm_policy_insert_inexact_list()
1544 } else if (policy->priority >= pol->priority) { in xfrm_policy_insert_inexact_list()
1545 newpos = &pol->bydst_inexact_list; in xfrm_policy_insert_inexact_list()
1552 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_policy_insert_inexact_list()
1553 hlist_add_behind_rcu(&policy->bydst_inexact_list, newpos); in xfrm_policy_insert_inexact_list()
1555 hlist_add_head_rcu(&policy->bydst_inexact_list, chain); in xfrm_policy_insert_inexact_list()
1558 pol->pos = i; in xfrm_policy_insert_inexact_list()
1570 if (pol->type == policy->type && in xfrm_policy_insert_list()
1571 pol->if_id == policy->if_id && in xfrm_policy_insert_list()
1572 !selector_cmp(&pol->selector, &policy->selector) && in xfrm_policy_insert_list()
1573 xfrm_policy_mark_match(&policy->mark, pol) && in xfrm_policy_insert_list()
1574 xfrm_sec_ctx_match(pol->security, policy->security) && in xfrm_policy_insert_list()
1577 return ERR_PTR(-EEXIST); in xfrm_policy_insert_list()
1579 if (policy->priority > pol->priority) in xfrm_policy_insert_list()
1581 } else if (policy->priority >= pol->priority) { in xfrm_policy_insert_list()
1589 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_policy_insert_list()
1590 hlist_add_behind_rcu(&policy->bydst, &newpos->bydst); in xfrm_policy_insert_list()
1593 * to speed-up lookups. in xfrm_policy_insert_list()
1595 hlist_add_head_rcu(&policy->bydst, chain); in xfrm_policy_insert_list()
1607 policy->mark.v &= policy->mark.m; in xfrm_policy_insert()
1609 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_insert()
1610 chain = policy_hash_bysel(net, &policy->selector, policy->family, dir); in xfrm_policy_insert()
1617 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_insert()
1624 if (policy->family == AF_INET) in xfrm_policy_insert()
1633 policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, policy->index); in xfrm_policy_insert()
1634 hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index)); in xfrm_policy_insert()
1635 policy->curlft.add_time = ktime_get_real_seconds(); in xfrm_policy_insert()
1636 policy->curlft.use_time = 0; in xfrm_policy_insert()
1637 if (!mod_timer(&policy->timer, jiffies + HZ)) in xfrm_policy_insert()
1639 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_insert()
1644 schedule_work(&net->xfrm.policy_hash_work); in xfrm_policy_insert()
1661 if (pol->type == type && in __xfrm_policy_bysel_ctx()
1662 pol->if_id == if_id && in __xfrm_policy_bysel_ctx()
1664 !selector_cmp(sel, &pol->selector) && in __xfrm_policy_bysel_ctx()
1665 xfrm_sec_ctx_match(ctx, pol->security)) in __xfrm_policy_bysel_ctx()
1682 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1683 chain = policy_hash_bysel(net, sel, sel->family, dir); in xfrm_policy_bysel_ctx()
1689 sel->family, dir, if_id); in xfrm_policy_bysel_ctx()
1691 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1696 &sel->saddr, in xfrm_policy_bysel_ctx()
1697 &sel->daddr)) { in xfrm_policy_bysel_ctx()
1698 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1712 if (!pol || tmp->pos < pol->pos) in xfrm_policy_bysel_ctx()
1723 *err = security_xfrm_policy_delete(pol->security); in xfrm_policy_bysel_ctx()
1725 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1732 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_bysel_ctx()
1749 *err = -ENOENT; in xfrm_policy_byid()
1754 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_byid()
1755 chain = net->xfrm.policy_byidx + idx_hash(net, id); in xfrm_policy_byid()
1758 if (pol->type == type && pol->index == id && in xfrm_policy_byid()
1759 pol->if_id == if_id && xfrm_policy_mark_match(mark, pol)) { in xfrm_policy_byid()
1763 pol->security); in xfrm_policy_byid()
1765 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_byid()
1774 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_byid()
1789 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_policy_flush_secctx_check()
1790 if (pol->walk.dead || in xfrm_policy_flush_secctx_check()
1791 xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX || in xfrm_policy_flush_secctx_check()
1792 pol->type != type) in xfrm_policy_flush_secctx_check()
1795 err = security_xfrm_policy_delete(pol->security); in xfrm_policy_flush_secctx_check()
1811 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_dev_policy_flush_secctx_check()
1812 if (pol->walk.dead || in xfrm_dev_policy_flush_secctx_check()
1813 xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX || in xfrm_dev_policy_flush_secctx_check()
1814 pol->xdo.dev != dev) in xfrm_dev_policy_flush_secctx_check()
1817 err = security_xfrm_policy_delete(pol->security); in xfrm_dev_policy_flush_secctx_check()
1845 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_flush()
1852 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_policy_flush()
1853 if (pol->walk.dead) in xfrm_policy_flush()
1856 dir = xfrm_policy_id2dir(pol->index); in xfrm_policy_flush()
1858 pol->type != type) in xfrm_policy_flush()
1862 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_flush()
1866 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_flush()
1872 err = -ESRCH; in xfrm_policy_flush()
1874 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_flush()
1885 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_dev_policy_flush()
1892 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { in xfrm_dev_policy_flush()
1893 if (pol->walk.dead) in xfrm_dev_policy_flush()
1896 dir = xfrm_policy_id2dir(pol->index); in xfrm_dev_policy_flush()
1898 pol->xdo.dev != dev) in xfrm_dev_policy_flush()
1902 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_dev_policy_flush()
1906 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_dev_policy_flush()
1912 err = -ESRCH; in xfrm_dev_policy_flush()
1914 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_dev_policy_flush()
1927 if (walk->type >= XFRM_POLICY_TYPE_MAX && in xfrm_policy_walk()
1928 walk->type != XFRM_POLICY_TYPE_ANY) in xfrm_policy_walk()
1929 return -EINVAL; in xfrm_policy_walk()
1931 if (list_empty(&walk->walk.all) && walk->seq != 0) in xfrm_policy_walk()
1934 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_walk()
1935 if (list_empty(&walk->walk.all)) in xfrm_policy_walk()
1936 x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all); in xfrm_policy_walk()
1938 x = list_first_entry(&walk->walk.all, in xfrm_policy_walk()
1941 list_for_each_entry_from(x, &net->xfrm.policy_all, all) { in xfrm_policy_walk()
1942 if (x->dead) in xfrm_policy_walk()
1945 if (walk->type != XFRM_POLICY_TYPE_ANY && in xfrm_policy_walk()
1946 walk->type != pol->type) in xfrm_policy_walk()
1948 error = func(pol, xfrm_policy_id2dir(pol->index), in xfrm_policy_walk()
1949 walk->seq, data); in xfrm_policy_walk()
1951 list_move_tail(&walk->walk.all, &x->all); in xfrm_policy_walk()
1954 walk->seq++; in xfrm_policy_walk()
1956 if (walk->seq == 0) { in xfrm_policy_walk()
1957 error = -ENOENT; in xfrm_policy_walk()
1960 list_del_init(&walk->walk.all); in xfrm_policy_walk()
1962 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_walk()
1969 INIT_LIST_HEAD(&walk->walk.all); in xfrm_policy_walk_init()
1970 walk->walk.dead = 1; in xfrm_policy_walk_init()
1971 walk->type = type; in xfrm_policy_walk_init()
1972 walk->seq = 0; in xfrm_policy_walk_init()
1978 if (list_empty(&walk->walk.all)) in xfrm_policy_walk_done()
1981 spin_lock_bh(&net->xfrm.xfrm_policy_lock); /*FIXME where is net? */ in xfrm_policy_walk_done()
1982 list_del(&walk->walk.all); in xfrm_policy_walk_done()
1983 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_walk_done()
1990 * Returns 0 if policy found, else an -errno.
1996 const struct xfrm_selector *sel = &pol->selector; in xfrm_policy_match()
1997 int ret = -ESRCH; in xfrm_policy_match()
2000 if (pol->family != family || in xfrm_policy_match()
2001 pol->if_id != if_id || in xfrm_policy_match()
2002 (fl->flowi_mark & pol->mark.m) != pol->mark.v || in xfrm_policy_match()
2003 pol->type != type) in xfrm_policy_match()
2008 ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid); in xfrm_policy_match()
2023 parent = rcu_dereference_raw(r->rb_node); in xfrm_policy_lookup_inexact_addr()
2030 delta = xfrm_policy_addr_delta(addr, &node->addr, in xfrm_policy_lookup_inexact_addr()
2031 node->prefixlen, family); in xfrm_policy_lookup_inexact_addr()
2033 parent = rcu_dereference_raw(parent->rb_left); in xfrm_policy_lookup_inexact_addr()
2036 parent = rcu_dereference_raw(parent->rb_right); in xfrm_policy_lookup_inexact_addr()
2061 family = b->k.family; in xfrm_policy_find_inexact_candidates()
2063 cand->res[XFRM_POL_CAND_ANY] = &b->hhead; in xfrm_policy_find_inexact_candidates()
2065 n = xfrm_policy_lookup_inexact_addr(&b->root_d, &b->count, daddr, in xfrm_policy_find_inexact_candidates()
2068 cand->res[XFRM_POL_CAND_DADDR] = &n->hhead; in xfrm_policy_find_inexact_candidates()
2069 n = xfrm_policy_lookup_inexact_addr(&n->root, &b->count, saddr, in xfrm_policy_find_inexact_candidates()
2072 cand->res[XFRM_POL_CAND_BOTH] = &n->hhead; in xfrm_policy_find_inexact_candidates()
2075 n = xfrm_policy_lookup_inexact_addr(&b->root_s, &b->count, saddr, in xfrm_policy_find_inexact_candidates()
2078 cand->res[XFRM_POL_CAND_SADDR] = &n->hhead; in xfrm_policy_find_inexact_candidates()
2106 lockdep_assert_held(&net->xfrm.xfrm_policy_lock); in xfrm_policy_inexact_lookup()
2121 u32 priority = prefer ? prefer->priority : ~0u; in __xfrm_policy_eval_candidates()
2130 if (pol->priority > priority) in __xfrm_policy_eval_candidates()
2135 if (err != -ESRCH) in __xfrm_policy_eval_candidates()
2143 if (pol->priority == priority && in __xfrm_policy_eval_candidates()
2144 prefer->pos < pol->pos) in __xfrm_policy_eval_candidates()
2163 for (i = 0; i < ARRAY_SIZE(cand->res); i++) { in xfrm_policy_eval_candidates()
2164 tmp = __xfrm_policy_eval_candidates(cand->res[i], in xfrm_policy_eval_candidates()
2197 retry: in xfrm_policy_lookup_bytype()
2199 sequence = read_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); in xfrm_policy_lookup_bytype()
2201 } while (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence)); in xfrm_policy_lookup_bytype()
2207 if (err == -ESRCH) in xfrm_policy_lookup_bytype()
2218 if (ret && ret->xdo.type == XFRM_DEV_OFFLOAD_PACKET) in xfrm_policy_lookup_bytype()
2235 if (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence)) in xfrm_policy_lookup_bytype()
2236 goto retry; in xfrm_policy_lookup_bytype()
2239 goto retry; in xfrm_policy_lookup_bytype()
2270 pol = rcu_dereference(sk->sk_policy[dir]); in xfrm_sk_policy_lookup()
2275 if (pol->family != family) { in xfrm_sk_policy_lookup()
2280 match = xfrm_selector_match(&pol->selector, fl, family); in xfrm_sk_policy_lookup()
2282 if ((READ_ONCE(sk->sk_mark) & pol->mark.m) != pol->mark.v || in xfrm_sk_policy_lookup()
2283 pol->if_id != if_id) { in xfrm_sk_policy_lookup()
2287 err = security_xfrm_policy_lookup(pol->security, in xfrm_sk_policy_lookup()
2288 fl->flowi_secid); in xfrm_sk_policy_lookup()
2292 } else if (err == -ESRCH) { in xfrm_sk_policy_lookup()
2309 list_add(&pol->walk.all, &net->xfrm.policy_all); in __xfrm_policy_link()
2310 net->xfrm.policy_count[dir]++; in __xfrm_policy_link()
2319 if (list_empty(&pol->walk.all)) in __xfrm_policy_unlink()
2323 if (!hlist_unhashed(&pol->bydst)) { in __xfrm_policy_unlink()
2324 hlist_del_rcu(&pol->bydst); in __xfrm_policy_unlink()
2325 hlist_del_init(&pol->bydst_inexact_list); in __xfrm_policy_unlink()
2326 hlist_del(&pol->byidx); in __xfrm_policy_unlink()
2329 list_del_init(&pol->walk.all); in __xfrm_policy_unlink()
2330 net->xfrm.policy_count[dir]--; in __xfrm_policy_unlink()
2349 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_delete()
2351 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_delete()
2356 return -ENOENT; in xfrm_policy_delete()
2366 if (pol && pol->type != XFRM_POLICY_TYPE_MAIN) in xfrm_sk_policy_insert()
2367 return -EINVAL; in xfrm_sk_policy_insert()
2370 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_sk_policy_insert()
2371 old_pol = rcu_dereference_protected(sk->sk_policy[dir], in xfrm_sk_policy_insert()
2372 lockdep_is_held(&net->xfrm.xfrm_policy_lock)); in xfrm_sk_policy_insert()
2374 pol->curlft.add_time = ktime_get_real_seconds(); in xfrm_sk_policy_insert()
2375 pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0); in xfrm_sk_policy_insert()
2378 rcu_assign_pointer(sk->sk_policy[dir], pol); in xfrm_sk_policy_insert()
2388 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_sk_policy_insert()
2402 newp->selector = old->selector; in clone_policy()
2403 if (security_xfrm_policy_clone(old->security, in clone_policy()
2404 &newp->security)) { in clone_policy()
2408 newp->lft = old->lft; in clone_policy()
2409 newp->curlft = old->curlft; in clone_policy()
2410 newp->mark = old->mark; in clone_policy()
2411 newp->if_id = old->if_id; in clone_policy()
2412 newp->action = old->action; in clone_policy()
2413 newp->flags = old->flags; in clone_policy()
2414 newp->xfrm_nr = old->xfrm_nr; in clone_policy()
2415 newp->index = old->index; in clone_policy()
2416 newp->type = old->type; in clone_policy()
2417 newp->family = old->family; in clone_policy()
2418 memcpy(newp->xfrm_vec, old->xfrm_vec, in clone_policy()
2419 newp->xfrm_nr*sizeof(struct xfrm_tmpl)); in clone_policy()
2420 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in clone_policy()
2422 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in clone_policy()
2436 p = rcu_dereference(osk->sk_policy[i]); in __xfrm_sk_clone_policy()
2440 ret = -ENOMEM; in __xfrm_sk_clone_policy()
2443 rcu_assign_pointer(sk->sk_policy[i], np); in __xfrm_sk_clone_policy()
2452 const struct xfrm_dst_lookup_params *params) in xfrm_get_saddr() argument
2458 return -EINVAL; in xfrm_get_saddr()
2459 err = afinfo->get_saddr(saddr, params); in xfrm_get_saddr()
2477 for (nx = 0, i = 0; i < policy->xfrm_nr; i++) { in xfrm_tmpl_resolve_one()
2481 struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i]; in xfrm_tmpl_resolve_one()
2483 if (tmpl->mode == XFRM_MODE_TUNNEL || in xfrm_tmpl_resolve_one()
2484 tmpl->mode == XFRM_MODE_BEET) { in xfrm_tmpl_resolve_one()
2485 remote = &tmpl->id.daddr; in xfrm_tmpl_resolve_one()
2486 local = &tmpl->saddr; in xfrm_tmpl_resolve_one()
2487 if (xfrm_addr_any(local, tmpl->encap_family)) { in xfrm_tmpl_resolve_one()
2488 struct xfrm_dst_lookup_params params; in xfrm_tmpl_resolve_one() local
2490 memset(¶ms, 0, sizeof(params)); in xfrm_tmpl_resolve_one()
2491 params.net = net; in xfrm_tmpl_resolve_one()
2492 params.oif = fl->flowi_oif; in xfrm_tmpl_resolve_one()
2493 params.daddr = remote; in xfrm_tmpl_resolve_one()
2494 error = xfrm_get_saddr(tmpl->encap_family, &tmp, in xfrm_tmpl_resolve_one()
2495 ¶ms); in xfrm_tmpl_resolve_one()
2503 family, policy->if_id); in xfrm_tmpl_resolve_one()
2505 if (x && x->km.state == XFRM_STATE_VALID) { in xfrm_tmpl_resolve_one()
2512 error = (x->km.state == XFRM_STATE_ERROR ? in xfrm_tmpl_resolve_one()
2513 -EINVAL : -EAGAIN); in xfrm_tmpl_resolve_one()
2515 } else if (error == -ESRCH) { in xfrm_tmpl_resolve_one()
2516 error = -EAGAIN; in xfrm_tmpl_resolve_one()
2519 if (!tmpl->optional) in xfrm_tmpl_resolve_one()
2525 for (nx--; nx >= 0; nx--) in xfrm_tmpl_resolve_one()
2542 if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) { in xfrm_tmpl_resolve()
2543 error = -ENOBUFS; in xfrm_tmpl_resolve()
2562 for (cnx--; cnx >= 0; cnx--) in xfrm_tmpl_resolve()
2571 return IPTOS_RT_MASK & fl->u.ip4.flowi4_tos; in xfrm_get_tos()
2583 return ERR_PTR(-EINVAL); in xfrm_alloc_dst()
2587 dst_ops = &net->xfrm.xfrm4_dst_ops; in xfrm_alloc_dst()
2591 dst_ops = &net->xfrm.xfrm6_dst_ops; in xfrm_alloc_dst()
2602 xdst = ERR_PTR(-ENOBUFS); in xfrm_alloc_dst()
2612 if (dst->ops->family == AF_INET6) { in xfrm_init_path()
2613 path->path_cookie = rt6_get_cookie(dst_rt6_info(dst)); in xfrm_init_path()
2614 path->u.rt6.rt6i_nfheader_len = nfheader_len; in xfrm_init_path()
2622 xfrm_policy_get_afinfo(xdst->u.dst.ops->family); in xfrm_fill_dst()
2626 return -EINVAL; in xfrm_fill_dst()
2628 err = afinfo->fill_dst(xdst, dev, fl); in xfrm_fill_dst()
2660 int family = policy->selector.family; in xfrm_bundle_create()
2671 struct dst_entry *dst1 = &xdst->u.dst; in xfrm_bundle_create()
2686 xfrm_dst_set_child(xdst_prev, &xdst->u.dst); in xfrm_bundle_create()
2688 if (xfrm[i]->sel.family == AF_UNSPEC) { in xfrm_bundle_create()
2692 err = -EAFNOSUPPORT; in xfrm_bundle_create()
2697 inner_mode = &xfrm[i]->inner_mode; in xfrm_bundle_create()
2699 xdst->route = dst; in xfrm_bundle_create()
2702 if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) { in xfrm_bundle_create()
2706 if (xfrm[i]->props.smark.v || xfrm[i]->props.smark.m) in xfrm_bundle_create()
2707 mark = xfrm_smark_get(fl->flowi_mark, xfrm[i]); in xfrm_bundle_create()
2709 if (xfrm[i]->xso.type != XFRM_DEV_OFFLOAD_PACKET) in xfrm_bundle_create()
2710 family = xfrm[i]->props.family; in xfrm_bundle_create()
2712 oif = fl->flowi_oif ? : fl->flowi_l3mdev; in xfrm_bundle_create()
2721 dst1->xfrm = xfrm[i]; in xfrm_bundle_create()
2722 xdst->xfrm_genid = xfrm[i]->genid; in xfrm_bundle_create()
2724 dst1->obsolete = DST_OBSOLETE_FORCE_CHK; in xfrm_bundle_create()
2725 dst1->lastuse = now; in xfrm_bundle_create()
2727 dst1->input = dst_discard; in xfrm_bundle_create()
2730 afinfo = xfrm_state_afinfo_get_rcu(inner_mode->family); in xfrm_bundle_create()
2732 dst1->output = afinfo->output; in xfrm_bundle_create()
2734 dst1->output = dst_discard_out; in xfrm_bundle_create()
2739 header_len += xfrm[i]->props.header_len; in xfrm_bundle_create()
2740 if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT) in xfrm_bundle_create()
2741 nfheader_len += xfrm[i]->props.header_len; in xfrm_bundle_create()
2742 trailer_len += xfrm[i]->props.trailer_len; in xfrm_bundle_create()
2746 xdst0->path = dst; in xfrm_bundle_create()
2748 err = -ENODEV; in xfrm_bundle_create()
2749 dev = dst->dev; in xfrm_bundle_create()
2757 xdst_prev = (struct xfrm_dst *) xfrm_dst_child(&xdst_prev->u.dst)) { in xfrm_bundle_create()
2762 xdst_prev->u.dst.header_len = header_len; in xfrm_bundle_create()
2763 xdst_prev->u.dst.trailer_len = trailer_len; in xfrm_bundle_create()
2764 header_len -= xdst_prev->u.dst.xfrm->props.header_len; in xfrm_bundle_create()
2765 trailer_len -= xdst_prev->u.dst.xfrm->props.trailer_len; in xfrm_bundle_create()
2768 return &xdst0->u.dst; in xfrm_bundle_create()
2775 dst_release_immediate(&xdst0->u.dst); in xfrm_bundle_create()
2796 *num_xfrms = pols[0]->xfrm_nr; in xfrm_expand_policies()
2799 if (pols[0]->action == XFRM_POLICY_ALLOW && in xfrm_expand_policies()
2800 pols[0]->type != XFRM_POLICY_TYPE_MAIN) { in xfrm_expand_policies()
2805 pols[0]->if_id); in xfrm_expand_policies()
2813 (*num_xfrms) += pols[1]->xfrm_nr; in xfrm_expand_policies()
2818 if (pols[i]->action != XFRM_POLICY_ALLOW) { in xfrm_expand_policies()
2819 *num_xfrms = -1; in xfrm_expand_policies()
2846 if (err != -EAGAIN) in xfrm_resolve_and_create_bundle()
2858 xdst->num_xfrms = err; in xfrm_resolve_and_create_bundle()
2859 xdst->num_pols = num_pols; in xfrm_resolve_and_create_bundle()
2860 memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols); in xfrm_resolve_and_create_bundle()
2861 xdst->policy_genid = atomic_read(&pols[0]->genid); in xfrm_resolve_and_create_bundle()
2873 struct xfrm_policy_queue *pq = &pol->polq; in xfrm_policy_queue_process()
2878 spin_lock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2879 skb = skb_peek(&pq->hold_queue); in xfrm_policy_queue_process()
2881 spin_unlock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2885 sk = skb->sk; in xfrm_policy_queue_process()
2888 skb_mark = skb->mark; in xfrm_policy_queue_process()
2889 skb->mark = pol->mark.v; in xfrm_policy_queue_process()
2890 xfrm_decode_session(skb, &fl, dst->ops->family); in xfrm_policy_queue_process()
2891 skb->mark = skb_mark; in xfrm_policy_queue_process()
2892 spin_unlock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2899 if (dst->flags & DST_XFRM_QUEUE) { in xfrm_policy_queue_process()
2902 if (pq->timeout >= XFRM_QUEUE_TMO_MAX) in xfrm_policy_queue_process()
2905 pq->timeout = pq->timeout << 1; in xfrm_policy_queue_process()
2906 if (!mod_timer(&pq->hold_timer, jiffies + pq->timeout)) in xfrm_policy_queue_process()
2915 spin_lock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2916 pq->timeout = 0; in xfrm_policy_queue_process()
2917 skb_queue_splice_init(&pq->hold_queue, &list); in xfrm_policy_queue_process()
2918 spin_unlock(&pq->hold_queue.lock); in xfrm_policy_queue_process()
2924 skb_mark = skb->mark; in xfrm_policy_queue_process()
2925 skb->mark = pol->mark.v; in xfrm_policy_queue_process()
2926 xfrm_decode_session(skb, &fl, skb_dst(skb)->ops->family); in xfrm_policy_queue_process()
2927 skb->mark = skb_mark; in xfrm_policy_queue_process()
2930 dst = xfrm_lookup(net, xfrm_dst_path(skb_dst(skb)), &fl, skb->sk, 0); in xfrm_policy_queue_process()
2940 dst_output(net, skb->sk, skb); in xfrm_policy_queue_process()
2948 pq->timeout = 0; in xfrm_policy_queue_process()
2949 skb_queue_purge(&pq->hold_queue); in xfrm_policy_queue_process()
2958 struct xfrm_policy *pol = xdst->pols[0]; in xdst_queue_output()
2959 struct xfrm_policy_queue *pq = &pol->polq; in xdst_queue_output()
2966 if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) { in xdst_queue_output()
2968 return -EAGAIN; in xdst_queue_output()
2973 spin_lock_bh(&pq->hold_queue.lock); in xdst_queue_output()
2975 if (!pq->timeout) in xdst_queue_output()
2976 pq->timeout = XFRM_QUEUE_TMO_MIN; in xdst_queue_output()
2978 sched_next = jiffies + pq->timeout; in xdst_queue_output()
2980 if (del_timer(&pq->hold_timer)) { in xdst_queue_output()
2981 if (time_before(pq->hold_timer.expires, sched_next)) in xdst_queue_output()
2982 sched_next = pq->hold_timer.expires; in xdst_queue_output()
2986 __skb_queue_tail(&pq->hold_queue, skb); in xdst_queue_output()
2987 if (!mod_timer(&pq->hold_timer, sched_next)) in xdst_queue_output()
2990 spin_unlock_bh(&pq->hold_queue.lock); in xdst_queue_output()
3011 if (!(xflo->flags & XFRM_LOOKUP_QUEUE) || in xfrm_create_dummy_bundle()
3012 net->xfrm.sysctl_larval_drop || in xfrm_create_dummy_bundle()
3016 dst = xflo->dst_orig; in xfrm_create_dummy_bundle()
3017 dst1 = &xdst->u.dst; in xfrm_create_dummy_bundle()
3019 xdst->route = dst; in xfrm_create_dummy_bundle()
3023 dst1->obsolete = DST_OBSOLETE_FORCE_CHK; in xfrm_create_dummy_bundle()
3024 dst1->flags |= DST_XFRM_QUEUE; in xfrm_create_dummy_bundle()
3025 dst1->lastuse = jiffies; in xfrm_create_dummy_bundle()
3027 dst1->input = dst_discard; in xfrm_create_dummy_bundle()
3028 dst1->output = xdst_queue_output; in xfrm_create_dummy_bundle()
3032 xdst->path = dst; in xfrm_create_dummy_bundle()
3036 err = -ENODEV; in xfrm_create_dummy_bundle()
3037 dev = dst->dev; in xfrm_create_dummy_bundle()
3077 xflo->dst_orig); in xfrm_bundle_lookup()
3080 if (err == -EREMOTE) { in xfrm_bundle_lookup()
3085 if (err != -EAGAIN) in xfrm_bundle_lookup()
3104 xdst->num_pols = num_pols; in xfrm_bundle_lookup()
3105 xdst->num_xfrms = num_xfrms; in xfrm_bundle_lookup()
3106 memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols); in xfrm_bundle_lookup()
3125 return ERR_PTR(-EINVAL); in make_blackhole()
3127 ret = afinfo->blackhole_route(net, dst_orig); in make_blackhole()
3151 u16 family = dst_orig->ops->family; in xfrm_lookup_with_ifid()
3160 if (sk && sk->sk_policy[XFRM_POLICY_OUT]) { in xfrm_lookup_with_ifid()
3182 if (err == -EREMOTE) in xfrm_lookup_with_ifid()
3192 route = xdst->route; in xfrm_lookup_with_ifid()
3203 if (!if_id && ((dst_orig->flags & DST_NOXFRM) || in xfrm_lookup_with_ifid()
3204 !net->xfrm.policy_count[XFRM_POLICY_OUT])) in xfrm_lookup_with_ifid()
3215 num_pols = xdst->num_pols; in xfrm_lookup_with_ifid()
3216 num_xfrms = xdst->num_xfrms; in xfrm_lookup_with_ifid()
3217 memcpy(pols, xdst->pols, sizeof(struct xfrm_policy *) * num_pols); in xfrm_lookup_with_ifid()
3218 route = xdst->route; in xfrm_lookup_with_ifid()
3221 dst = &xdst->u.dst; in xfrm_lookup_with_ifid()
3229 if (net->xfrm.sysctl_larval_drop) { in xfrm_lookup_with_ifid()
3231 err = -EREMOTE; in xfrm_lookup_with_ifid()
3235 err = -EAGAIN; in xfrm_lookup_with_ifid()
3246 !(pols[0]->flags & XFRM_POLICY_ICMP)) { in xfrm_lookup_with_ifid()
3247 err = -ENOENT; in xfrm_lookup_with_ifid()
3252 WRITE_ONCE(pols[i]->curlft.use_time, ktime_get_real_seconds()); in xfrm_lookup_with_ifid()
3257 err = -EPERM; in xfrm_lookup_with_ifid()
3269 if (dst && dst->xfrm && in xfrm_lookup_with_ifid()
3270 dst->xfrm->props.mode == XFRM_MODE_TUNNEL) in xfrm_lookup_with_ifid()
3271 dst->flags |= DST_XFRM_TUNNEL; in xfrm_lookup_with_ifid()
3275 if ((!dst_orig->dev || !(dst_orig->dev->flags & IFF_LOOPBACK)) && in xfrm_lookup_with_ifid()
3276 net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { in xfrm_lookup_with_ifid()
3277 err = -EPERM; in xfrm_lookup_with_ifid()
3284 err = -ENOENT; in xfrm_lookup_with_ifid()
3319 if (PTR_ERR(dst) == -EREMOTE) in xfrm_lookup_route()
3320 return make_blackhole(net, dst_orig->ops->family, dst_orig); in xfrm_lookup_route()
3335 if (!sp || idx < 0 || idx >= sp->len) in xfrm_secpath_reject()
3337 x = sp->xvec[idx]; in xfrm_secpath_reject()
3338 if (!x->type->reject) in xfrm_secpath_reject()
3340 return x->type->reject(x, skb, fl); in xfrm_secpath_reject()
3345 * stupid way. Shame on me. :-) Of course, connected sockets must
3354 return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family); in xfrm_state_ok()
3355 return x->id.proto == tmpl->id.proto && in xfrm_state_ok()
3356 (x->id.spi == tmpl->id.spi || !tmpl->id.spi) && in xfrm_state_ok()
3357 (x->props.reqid == tmpl->reqid || !tmpl->reqid) && in xfrm_state_ok()
3358 x->props.mode == tmpl->mode && in xfrm_state_ok()
3359 (tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) || in xfrm_state_ok()
3360 !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) && in xfrm_state_ok()
3361 !(x->props.mode != XFRM_MODE_TRANSPORT && in xfrm_state_ok()
3363 (if_id == 0 || if_id == x->if_id); in xfrm_state_ok()
3370 * -1 is returned when no matching template is found.
3371 * Otherwise "-2 - errored_index" is returned.
3379 if (tmpl->optional) { in xfrm_policy_ok()
3380 if (tmpl->mode == XFRM_MODE_TRANSPORT) in xfrm_policy_ok()
3383 start = -1; in xfrm_policy_ok()
3384 for (; idx < sp->len; idx++) { in xfrm_policy_ok()
3385 if (xfrm_state_ok(tmpl, sp->xvec[idx], family, if_id)) in xfrm_policy_ok()
3387 if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) { in xfrm_policy_ok()
3388 if (idx < sp->verified_cnt) { in xfrm_policy_ok()
3395 if (start == -1) in xfrm_policy_ok()
3396 start = -2-idx; in xfrm_policy_ok()
3407 int ihl = iph->ihl; in decode_session4()
3409 struct flowi4 *fl4 = &fl->u.ip4; in decode_session4()
3412 if (skb_dst(skb) && skb_dst(skb)->dev) in decode_session4()
3413 oif = skb_dst(skb)->dev->ifindex; in decode_session4()
3416 fl4->flowi4_mark = skb->mark; in decode_session4()
3417 fl4->flowi4_oif = reverse ? skb->skb_iif : oif; in decode_session4()
3419 fl4->flowi4_proto = iph->protocol; in decode_session4()
3420 fl4->daddr = reverse ? iph->saddr : iph->daddr; in decode_session4()
3421 fl4->saddr = reverse ? iph->daddr : iph->saddr; in decode_session4()
3422 fl4->flowi4_tos = iph->tos & ~INET_ECN_MASK; in decode_session4()
3425 switch (iph->protocol) { in decode_session4()
3431 if (xprth + 4 < skb->data || in decode_session4()
3432 pskb_may_pull(skb, xprth + 4 - skb->data)) { in decode_session4()
3438 fl4->fl4_sport = ports[!!reverse]; in decode_session4()
3439 fl4->fl4_dport = ports[!reverse]; in decode_session4()
3443 if (xprth + 2 < skb->data || in decode_session4()
3444 pskb_may_pull(skb, xprth + 2 - skb->data)) { in decode_session4()
3450 fl4->fl4_icmp_type = icmp[0]; in decode_session4()
3451 fl4->fl4_icmp_code = icmp[1]; in decode_session4()
3455 if (xprth + 12 < skb->data || in decode_session4()
3456 pskb_may_pull(skb, xprth + 12 - skb->data)) { in decode_session4()
3467 fl4->fl4_gre_key = gre_hdr[1]; in decode_session4()
3481 struct flowi6 *fl6 = &fl->u.ip6; in decode_session6()
3487 u16 nhoff = IP6CB(skb)->nhoff; in decode_session6()
3496 if (skb_dst(skb) && skb_dst(skb)->dev) in decode_session6()
3497 oif = skb_dst(skb)->dev->ifindex; in decode_session6()
3500 fl6->flowi6_mark = skb->mark; in decode_session6()
3501 fl6->flowi6_oif = reverse ? skb->skb_iif : oif; in decode_session6()
3503 fl6->daddr = reverse ? hdr->saddr : hdr->daddr; in decode_session6()
3504 fl6->saddr = reverse ? hdr->daddr : hdr->saddr; in decode_session6()
3506 while (nh + offset + sizeof(*exthdr) < skb->data || in decode_session6()
3507 pskb_may_pull(skb, nh + offset + sizeof(*exthdr) - skb->data)) { in decode_session6()
3519 nexthdr = exthdr->nexthdr; in decode_session6()
3526 if (!onlyproto && (nh + offset + 4 < skb->data || in decode_session6()
3527 pskb_may_pull(skb, nh + offset + 4 - skb->data))) { in decode_session6()
3532 fl6->fl6_sport = ports[!!reverse]; in decode_session6()
3533 fl6->fl6_dport = ports[!reverse]; in decode_session6()
3535 fl6->flowi6_proto = nexthdr; in decode_session6()
3538 if (!onlyproto && (nh + offset + 2 < skb->data || in decode_session6()
3539 pskb_may_pull(skb, nh + offset + 2 - skb->data))) { in decode_session6()
3544 fl6->fl6_icmp_type = icmp[0]; in decode_session6()
3545 fl6->fl6_icmp_code = icmp[1]; in decode_session6()
3547 fl6->flowi6_proto = nexthdr; in decode_session6()
3551 (nh + offset + 12 < skb->data || in decode_session6()
3552 pskb_may_pull(skb, nh + offset + 12 - skb->data))) { in decode_session6()
3560 if (gre_hdr->flags & GRE_KEY) { in decode_session6()
3561 if (gre_hdr->flags & GRE_CSUM) in decode_session6()
3563 fl6->fl6_gre_key = *gre_key; in decode_session6()
3566 fl6->flowi6_proto = nexthdr; in decode_session6()
3572 if (!onlyproto && (nh + offset + 3 < skb->data || in decode_session6()
3573 pskb_may_pull(skb, nh + offset + 3 - skb->data))) { in decode_session6()
3578 fl6->fl6_mh_type = mh->ip6mh_type; in decode_session6()
3580 fl6->flowi6_proto = nexthdr; in decode_session6()
3584 fl6->flowi6_proto = nexthdr; in decode_session6()
3604 return -EAFNOSUPPORT; in __xfrm_decode_session()
3607 return security_xfrm_decode_session(skb, &fl->flowi_secid); in __xfrm_decode_session()
3613 for (; k < sp->len; k++) { in secpath_has_nontransport()
3614 if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) { in secpath_has_nontransport()
3626 struct net *net = dev_net(skb->dev); in __xfrm_policy_check()
3634 int xerr_idx = -1; in __xfrm_policy_check()
3645 if (ifcb->decode_session(skb, family, &r)) { in __xfrm_policy_check()
3667 for (i = sp->len - 1; i >= 0; i--) { in __xfrm_policy_check()
3668 struct xfrm_state *x = sp->xvec[i]; in __xfrm_policy_check()
3669 if (!xfrm_selector_match(&x->sel, &fl, family)) { in __xfrm_policy_check()
3678 if (sk && sk->sk_policy[dir]) { in __xfrm_policy_check()
3695 if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { in __xfrm_policy_check()
3709 WRITE_ONCE(pol->curlft.use_time, ktime_get_real_seconds()); in __xfrm_policy_check()
3714 if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) { in __xfrm_policy_check()
3725 WRITE_ONCE(pols[1]->curlft.use_time, in __xfrm_policy_check()
3732 if (pol->action == XFRM_POLICY_ALLOW) { in __xfrm_policy_check()
3746 pols[pi]->action != XFRM_POLICY_ALLOW) { in __xfrm_policy_check()
3750 if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) { in __xfrm_policy_check()
3754 for (i = 0; i < pols[pi]->xfrm_nr; i++) in __xfrm_policy_check()
3755 tpp[ti++] = &pols[pi]->xfrm_vec[i]; in __xfrm_policy_check()
3773 for (i = xfrm_nr-1, k = 0; i >= 0; i--) { in __xfrm_policy_check()
3776 if (k < -1) in __xfrm_policy_check()
3777 /* "-2 - errored_index" returned */ in __xfrm_policy_check()
3778 xerr_idx = -(2+k); in __xfrm_policy_check()
3790 sp->verified_cnt = k; in __xfrm_policy_check()
3806 struct net *net = dev_net(skb->dev); in __xfrm_route_forward()
3836 /* Code (such as __xfrm4_bundle_create()) sets dst->obsolete in xfrm_dst_check()
3838 * get validated by dst_ops->check on every use. We do this in xfrm_dst_check()
3845 * XFRM dst A --> IPv4 dst X in xfrm_dst_check()
3847 * X is the "xdst->route" of A (X is also the "dst->path" of A in xfrm_dst_check()
3857 if (dst->obsolete < 0 && !stale_bundle(dst)) in xfrm_dst_check()
3870 while ((dst = xfrm_dst_child(dst)) && dst->xfrm && dst->dev == dev) { in xfrm_dst_ifdown()
3871 dst->dev = blackhole_netdev; in xfrm_dst_ifdown()
3872 dev_hold(dst->dev); in xfrm_dst_ifdown()
3885 if (dst->obsolete) in xfrm_negative_advice()
3891 while (nr--) { in xfrm_init_pmtu()
3896 dst = &xdst->u.dst; in xfrm_init_pmtu()
3898 xdst->child_mtu_cached = pmtu; in xfrm_init_pmtu()
3900 pmtu = xfrm_state_mtu(dst->xfrm, pmtu); in xfrm_init_pmtu()
3902 route_mtu_cached = dst_mtu(xdst->route); in xfrm_init_pmtu()
3903 xdst->route_mtu_cached = route_mtu_cached; in xfrm_init_pmtu()
3919 struct dst_entry *dst = &first->u.dst; in xfrm_bundle_ok()
3924 if (!dst_check(xfrm_dst_path(dst), ((struct xfrm_dst *)dst)->path_cookie) || in xfrm_bundle_ok()
3925 (dst->dev && !netif_running(dst->dev))) in xfrm_bundle_ok()
3928 if (dst->flags & DST_XFRM_QUEUE) in xfrm_bundle_ok()
3935 if (dst->xfrm->km.state != XFRM_STATE_VALID) in xfrm_bundle_ok()
3937 if (xdst->xfrm_genid != dst->xfrm->genid) in xfrm_bundle_ok()
3939 if (xdst->num_pols > 0 && in xfrm_bundle_ok()
3940 xdst->policy_genid != atomic_read(&xdst->pols[0]->genid)) in xfrm_bundle_ok()
3946 if (xdst->child_mtu_cached != mtu) { in xfrm_bundle_ok()
3948 xdst->child_mtu_cached = mtu; in xfrm_bundle_ok()
3951 if (!dst_check(xdst->route, xdst->route_cookie)) in xfrm_bundle_ok()
3953 mtu = dst_mtu(xdst->route); in xfrm_bundle_ok()
3954 if (xdst->route_mtu_cached != mtu) { in xfrm_bundle_ok()
3956 xdst->route_mtu_cached = mtu; in xfrm_bundle_ok()
3960 } while (dst->xfrm); in xfrm_bundle_ok()
3965 xdst = bundle[start_from - 1]; in xfrm_bundle_ok()
3966 mtu = xdst->child_mtu_cached; in xfrm_bundle_ok()
3967 while (start_from--) { in xfrm_bundle_ok()
3968 dst = &xdst->u.dst; in xfrm_bundle_ok()
3970 mtu = xfrm_state_mtu(dst->xfrm, mtu); in xfrm_bundle_ok()
3971 if (mtu > xdst->route_mtu_cached) in xfrm_bundle_ok()
3972 mtu = xdst->route_mtu_cached; in xfrm_bundle_ok()
3977 xdst = bundle[start_from - 1]; in xfrm_bundle_ok()
3978 xdst->child_mtu_cached = mtu; in xfrm_bundle_ok()
3999 while (dst->xfrm) { in xfrm_get_dst_nexthop()
4000 const struct xfrm_state *xfrm = dst->xfrm; in xfrm_get_dst_nexthop()
4004 if (xfrm->props.mode == XFRM_MODE_TRANSPORT) in xfrm_get_dst_nexthop()
4006 if (xfrm->type->flags & XFRM_TYPE_REMOTE_COADDR) in xfrm_get_dst_nexthop()
4007 daddr = xfrm->coaddr; in xfrm_get_dst_nexthop()
4008 else if (!(xfrm->type->flags & XFRM_TYPE_LOCAL_COADDR)) in xfrm_get_dst_nexthop()
4009 daddr = &xfrm->id.daddr; in xfrm_get_dst_nexthop()
4022 return path->ops->neigh_lookup(path, skb, daddr); in xfrm_neigh_lookup()
4030 path->ops->confirm_neigh(path, daddr); in xfrm_confirm_neigh()
4038 return -EAFNOSUPPORT; in xfrm_policy_register_afinfo()
4042 err = -EEXIST; in xfrm_policy_register_afinfo()
4044 struct dst_ops *dst_ops = afinfo->dst_ops; in xfrm_policy_register_afinfo()
4045 if (likely(dst_ops->kmem_cachep == NULL)) in xfrm_policy_register_afinfo()
4046 dst_ops->kmem_cachep = xfrm_dst_cache; in xfrm_policy_register_afinfo()
4047 if (likely(dst_ops->check == NULL)) in xfrm_policy_register_afinfo()
4048 dst_ops->check = xfrm_dst_check; in xfrm_policy_register_afinfo()
4049 if (likely(dst_ops->default_advmss == NULL)) in xfrm_policy_register_afinfo()
4050 dst_ops->default_advmss = xfrm_default_advmss; in xfrm_policy_register_afinfo()
4051 if (likely(dst_ops->mtu == NULL)) in xfrm_policy_register_afinfo()
4052 dst_ops->mtu = xfrm_mtu; in xfrm_policy_register_afinfo()
4053 if (likely(dst_ops->negative_advice == NULL)) in xfrm_policy_register_afinfo()
4054 dst_ops->negative_advice = xfrm_negative_advice; in xfrm_policy_register_afinfo()
4055 if (likely(dst_ops->link_failure == NULL)) in xfrm_policy_register_afinfo()
4056 dst_ops->link_failure = xfrm_link_failure; in xfrm_policy_register_afinfo()
4057 if (likely(dst_ops->neigh_lookup == NULL)) in xfrm_policy_register_afinfo()
4058 dst_ops->neigh_lookup = xfrm_neigh_lookup; in xfrm_policy_register_afinfo()
4059 if (likely(!dst_ops->confirm_neigh)) in xfrm_policy_register_afinfo()
4060 dst_ops->confirm_neigh = xfrm_confirm_neigh; in xfrm_policy_register_afinfo()
4071 struct dst_ops *dst_ops = afinfo->dst_ops; in xfrm_policy_unregister_afinfo()
4083 dst_ops->kmem_cachep = NULL; in xfrm_policy_unregister_afinfo()
4084 dst_ops->check = NULL; in xfrm_policy_unregister_afinfo()
4085 dst_ops->negative_advice = NULL; in xfrm_policy_unregister_afinfo()
4086 dst_ops->link_failure = NULL; in xfrm_policy_unregister_afinfo()
4109 net->mib.xfrm_statistics = alloc_percpu(struct linux_xfrm_mib); in xfrm_statistics_init()
4110 if (!net->mib.xfrm_statistics) in xfrm_statistics_init()
4111 return -ENOMEM; in xfrm_statistics_init()
4114 free_percpu(net->mib.xfrm_statistics); in xfrm_statistics_init()
4121 free_percpu(net->mib.xfrm_statistics); in xfrm_statistics_fini()
4149 hmask = 8 - 1; in xfrm_policy_init()
4152 net->xfrm.policy_byidx = xfrm_hash_alloc(sz); in xfrm_policy_init()
4153 if (!net->xfrm.policy_byidx) in xfrm_policy_init()
4155 net->xfrm.policy_idx_hmask = hmask; in xfrm_policy_init()
4160 net->xfrm.policy_count[dir] = 0; in xfrm_policy_init()
4161 net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0; in xfrm_policy_init()
4162 INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]); in xfrm_policy_init()
4164 htab = &net->xfrm.policy_bydst[dir]; in xfrm_policy_init()
4165 htab->table = xfrm_hash_alloc(sz); in xfrm_policy_init()
4166 if (!htab->table) in xfrm_policy_init()
4168 htab->hmask = hmask; in xfrm_policy_init()
4169 htab->dbits4 = 32; in xfrm_policy_init()
4170 htab->sbits4 = 32; in xfrm_policy_init()
4171 htab->dbits6 = 128; in xfrm_policy_init()
4172 htab->sbits6 = 128; in xfrm_policy_init()
4174 net->xfrm.policy_hthresh.lbits4 = 32; in xfrm_policy_init()
4175 net->xfrm.policy_hthresh.rbits4 = 32; in xfrm_policy_init()
4176 net->xfrm.policy_hthresh.lbits6 = 128; in xfrm_policy_init()
4177 net->xfrm.policy_hthresh.rbits6 = 128; in xfrm_policy_init()
4179 seqlock_init(&net->xfrm.policy_hthresh.lock); in xfrm_policy_init()
4181 INIT_LIST_HEAD(&net->xfrm.policy_all); in xfrm_policy_init()
4182 INIT_LIST_HEAD(&net->xfrm.inexact_bins); in xfrm_policy_init()
4183 INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize); in xfrm_policy_init()
4184 INIT_WORK(&net->xfrm.policy_hthresh.work, xfrm_hash_rebuild); in xfrm_policy_init()
4188 for (dir--; dir >= 0; dir--) { in xfrm_policy_init()
4191 htab = &net->xfrm.policy_bydst[dir]; in xfrm_policy_init()
4192 xfrm_hash_free(htab->table, sz); in xfrm_policy_init()
4194 xfrm_hash_free(net->xfrm.policy_byidx, sz); in xfrm_policy_init()
4196 return -ENOMEM; in xfrm_policy_init()
4205 flush_work(&net->xfrm.policy_hash_work); in xfrm_policy_fini()
4211 WARN_ON(!list_empty(&net->xfrm.policy_all)); in xfrm_policy_fini()
4216 WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir])); in xfrm_policy_fini()
4218 htab = &net->xfrm.policy_bydst[dir]; in xfrm_policy_fini()
4219 sz = (htab->hmask + 1) * sizeof(struct hlist_head); in xfrm_policy_fini()
4220 WARN_ON(!hlist_empty(htab->table)); in xfrm_policy_fini()
4221 xfrm_hash_free(htab->table, sz); in xfrm_policy_fini()
4224 sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head); in xfrm_policy_fini()
4225 WARN_ON(!hlist_empty(net->xfrm.policy_byidx)); in xfrm_policy_fini()
4226 xfrm_hash_free(net->xfrm.policy_byidx, sz); in xfrm_policy_fini()
4228 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_fini()
4229 list_for_each_entry_safe(b, t, &net->xfrm.inexact_bins, inexact_bins) in xfrm_policy_fini()
4231 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_policy_fini()
4238 /* Initialize the per-net locks here */ in xfrm_net_init()
4239 spin_lock_init(&net->xfrm.xfrm_state_lock); in xfrm_net_init()
4240 spin_lock_init(&net->xfrm.xfrm_policy_lock); in xfrm_net_init()
4241 seqcount_spinlock_init(&net->xfrm.xfrm_policy_hash_generation, &net->xfrm.xfrm_policy_lock); in xfrm_net_init()
4242 mutex_init(&net->xfrm.xfrm_cfg_mutex); in xfrm_net_init()
4243 net->xfrm.policy_default[XFRM_POLICY_IN] = XFRM_USERPOLICY_ACCEPT; in xfrm_net_init()
4244 net->xfrm.policy_default[XFRM_POLICY_FWD] = XFRM_USERPOLICY_ACCEPT; in xfrm_net_init()
4245 net->xfrm.policy_default[XFRM_POLICY_OUT] = XFRM_USERPOLICY_ACCEPT; in xfrm_net_init()
4300 struct xfrm_sec_ctx *ctx = xp->security; in xfrm_audit_common_policyinfo()
4301 struct xfrm_selector *sel = &xp->selector; in xfrm_audit_common_policyinfo()
4305 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str); in xfrm_audit_common_policyinfo()
4307 switch (sel->family) { in xfrm_audit_common_policyinfo()
4309 audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4); in xfrm_audit_common_policyinfo()
4310 if (sel->prefixlen_s != 32) in xfrm_audit_common_policyinfo()
4312 sel->prefixlen_s); in xfrm_audit_common_policyinfo()
4313 audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4); in xfrm_audit_common_policyinfo()
4314 if (sel->prefixlen_d != 32) in xfrm_audit_common_policyinfo()
4316 sel->prefixlen_d); in xfrm_audit_common_policyinfo()
4319 audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6); in xfrm_audit_common_policyinfo()
4320 if (sel->prefixlen_s != 128) in xfrm_audit_common_policyinfo()
4322 sel->prefixlen_s); in xfrm_audit_common_policyinfo()
4323 audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6); in xfrm_audit_common_policyinfo()
4324 if (sel->prefixlen_d != 128) in xfrm_audit_common_policyinfo()
4326 sel->prefixlen_d); in xfrm_audit_common_policyinfo()
4335 audit_buf = xfrm_audit_start("SPD-add"); in xfrm_audit_policy_add()
4350 audit_buf = xfrm_audit_start("SPD-delete"); in xfrm_audit_policy_delete()
4365 if (sel_cmp->proto == IPSEC_ULPROTO_ANY) { in xfrm_migrate_selector_match()
4366 if (sel_tgt->family == sel_cmp->family && in xfrm_migrate_selector_match()
4367 xfrm_addr_equal(&sel_tgt->daddr, &sel_cmp->daddr, in xfrm_migrate_selector_match()
4368 sel_cmp->family) && in xfrm_migrate_selector_match()
4369 xfrm_addr_equal(&sel_tgt->saddr, &sel_cmp->saddr, in xfrm_migrate_selector_match()
4370 sel_cmp->family) && in xfrm_migrate_selector_match()
4371 sel_tgt->prefixlen_d == sel_cmp->prefixlen_d && in xfrm_migrate_selector_match()
4372 sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) { in xfrm_migrate_selector_match()
4390 spin_lock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_migrate_policy_find()
4391 chain = policy_hash_direct(net, &sel->daddr, &sel->saddr, sel->family, dir); in xfrm_migrate_policy_find()
4393 if ((if_id == 0 || pol->if_id == if_id) && in xfrm_migrate_policy_find()
4394 xfrm_migrate_selector_match(sel, &pol->selector) && in xfrm_migrate_policy_find()
4395 pol->type == type) { in xfrm_migrate_policy_find()
4397 priority = ret->priority; in xfrm_migrate_policy_find()
4401 chain = &net->xfrm.policy_inexact[dir]; in xfrm_migrate_policy_find()
4403 if ((pol->priority >= priority) && ret) in xfrm_migrate_policy_find()
4406 if ((if_id == 0 || pol->if_id == if_id) && in xfrm_migrate_policy_find()
4407 xfrm_migrate_selector_match(sel, &pol->selector) && in xfrm_migrate_policy_find()
4408 pol->type == type) { in xfrm_migrate_policy_find()
4416 spin_unlock_bh(&net->xfrm.xfrm_policy_lock); in xfrm_migrate_policy_find()
4425 if (t->mode == m->mode && t->id.proto == m->proto && in migrate_tmpl_match()
4426 (m->reqid == 0 || t->reqid == m->reqid)) { in migrate_tmpl_match()
4427 switch (t->mode) { in migrate_tmpl_match()
4430 if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr, in migrate_tmpl_match()
4431 m->old_family) && in migrate_tmpl_match()
4432 xfrm_addr_equal(&t->saddr, &m->old_saddr, in migrate_tmpl_match()
4433 m->old_family)) { in migrate_tmpl_match()
4458 write_lock_bh(&pol->lock); in xfrm_policy_migrate()
4459 if (unlikely(pol->walk.dead)) { in xfrm_policy_migrate()
4462 write_unlock_bh(&pol->lock); in xfrm_policy_migrate()
4463 return -ENOENT; in xfrm_policy_migrate()
4466 for (i = 0; i < pol->xfrm_nr; i++) { in xfrm_policy_migrate()
4468 if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i])) in xfrm_policy_migrate()
4471 if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL && in xfrm_policy_migrate()
4472 pol->xfrm_vec[i].mode != XFRM_MODE_BEET) in xfrm_policy_migrate()
4475 memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr, in xfrm_policy_migrate()
4476 sizeof(pol->xfrm_vec[i].id.daddr)); in xfrm_policy_migrate()
4477 memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr, in xfrm_policy_migrate()
4478 sizeof(pol->xfrm_vec[i].saddr)); in xfrm_policy_migrate()
4479 pol->xfrm_vec[i].encap_family = mp->new_family; in xfrm_policy_migrate()
4481 atomic_inc(&pol->genid); in xfrm_policy_migrate()
4485 write_unlock_bh(&pol->lock); in xfrm_policy_migrate()
4488 return -ENODATA; in xfrm_policy_migrate()
4500 return -EINVAL; in xfrm_migrate_check()
4507 return -EINVAL; in xfrm_migrate_check()
4521 return -EINVAL; in xfrm_migrate_check()
4542 /* Stage 0 - sanity checks */ in xfrm_migrate()
4549 err = -EINVAL; in xfrm_migrate()
4553 /* Stage 1 - find policy */ in xfrm_migrate()
4557 err = -ENOENT; in xfrm_migrate()
4561 /* Stage 2 - find and update state(s) */ in xfrm_migrate()
4571 err = -ENODATA; in xfrm_migrate()
4577 /* Stage 3 - update policy */ in xfrm_migrate()
4582 /* Stage 4 - delete old state(s) */ in xfrm_migrate()
4588 /* Stage 5 - announce */ in xfrm_migrate()