1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
3
4 #include <linux/bpf.h>
5 #include <linux/btf_ids.h>
6 #include <linux/filter.h>
7 #include <linux/errno.h>
8 #include <linux/file.h>
9 #include <linux/net.h>
10 #include <linux/workqueue.h>
11 #include <linux/skmsg.h>
12 #include <linux/list.h>
13 #include <linux/jhash.h>
14 #include <linux/sock_diag.h>
15 #include <net/udp.h>
16
17 struct bpf_stab {
18 struct bpf_map map;
19 struct sock **sks;
20 struct sk_psock_progs progs;
21 raw_spinlock_t lock;
22 };
23
24 #define SOCK_CREATE_FLAG_MASK \
25 (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)
26
sock_map_alloc(union bpf_attr * attr)27 static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
28 {
29 struct bpf_stab *stab;
30 u64 cost;
31 int err;
32
33 if (!capable(CAP_NET_ADMIN))
34 return ERR_PTR(-EPERM);
35 if (attr->max_entries == 0 ||
36 attr->key_size != 4 ||
37 (attr->value_size != sizeof(u32) &&
38 attr->value_size != sizeof(u64)) ||
39 attr->map_flags & ~SOCK_CREATE_FLAG_MASK)
40 return ERR_PTR(-EINVAL);
41
42 stab = kzalloc(sizeof(*stab), GFP_USER);
43 if (!stab)
44 return ERR_PTR(-ENOMEM);
45
46 bpf_map_init_from_attr(&stab->map, attr);
47 raw_spin_lock_init(&stab->lock);
48
49 /* Make sure page count doesn't overflow. */
50 cost = (u64) stab->map.max_entries * sizeof(struct sock *);
51 err = bpf_map_charge_init(&stab->map.memory, cost);
52 if (err)
53 goto free_stab;
54
55 stab->sks = bpf_map_area_alloc((u64) stab->map.max_entries *
56 sizeof(struct sock *),
57 stab->map.numa_node);
58 if (stab->sks)
59 return &stab->map;
60 err = -ENOMEM;
61 bpf_map_charge_finish(&stab->map.memory);
62 free_stab:
63 kfree(stab);
64 return ERR_PTR(err);
65 }
66
sock_map_get_from_fd(const union bpf_attr * attr,struct bpf_prog * prog)67 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog)
68 {
69 u32 ufd = attr->target_fd;
70 struct bpf_map *map;
71 struct fd f;
72 int ret;
73
74 if (attr->attach_flags || attr->replace_bpf_fd)
75 return -EINVAL;
76
77 f = fdget(ufd);
78 map = __bpf_map_get(f);
79 if (IS_ERR(map))
80 return PTR_ERR(map);
81 ret = sock_map_prog_update(map, prog, NULL, attr->attach_type);
82 fdput(f);
83 return ret;
84 }
85
sock_map_prog_detach(const union bpf_attr * attr,enum bpf_prog_type ptype)86 int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype)
87 {
88 u32 ufd = attr->target_fd;
89 struct bpf_prog *prog;
90 struct bpf_map *map;
91 struct fd f;
92 int ret;
93
94 if (attr->attach_flags || attr->replace_bpf_fd)
95 return -EINVAL;
96
97 f = fdget(ufd);
98 map = __bpf_map_get(f);
99 if (IS_ERR(map))
100 return PTR_ERR(map);
101
102 prog = bpf_prog_get(attr->attach_bpf_fd);
103 if (IS_ERR(prog)) {
104 ret = PTR_ERR(prog);
105 goto put_map;
106 }
107
108 if (prog->type != ptype) {
109 ret = -EINVAL;
110 goto put_prog;
111 }
112
113 ret = sock_map_prog_update(map, NULL, prog, attr->attach_type);
114 put_prog:
115 bpf_prog_put(prog);
116 put_map:
117 fdput(f);
118 return ret;
119 }
120
sock_map_sk_acquire(struct sock * sk)121 static void sock_map_sk_acquire(struct sock *sk)
122 __acquires(&sk->sk_lock.slock)
123 {
124 lock_sock(sk);
125 preempt_disable();
126 rcu_read_lock();
127 }
128
sock_map_sk_release(struct sock * sk)129 static void sock_map_sk_release(struct sock *sk)
130 __releases(&sk->sk_lock.slock)
131 {
132 rcu_read_unlock();
133 preempt_enable();
134 release_sock(sk);
135 }
136
sock_map_add_link(struct sk_psock * psock,struct sk_psock_link * link,struct bpf_map * map,void * link_raw)137 static void sock_map_add_link(struct sk_psock *psock,
138 struct sk_psock_link *link,
139 struct bpf_map *map, void *link_raw)
140 {
141 link->link_raw = link_raw;
142 link->map = map;
143 spin_lock_bh(&psock->link_lock);
144 list_add_tail(&link->list, &psock->link);
145 spin_unlock_bh(&psock->link_lock);
146 }
147
sock_map_del_link(struct sock * sk,struct sk_psock * psock,void * link_raw)148 static void sock_map_del_link(struct sock *sk,
149 struct sk_psock *psock, void *link_raw)
150 {
151 bool strp_stop = false, verdict_stop = false;
152 struct sk_psock_link *link, *tmp;
153
154 spin_lock_bh(&psock->link_lock);
155 list_for_each_entry_safe(link, tmp, &psock->link, list) {
156 if (link->link_raw == link_raw) {
157 struct bpf_map *map = link->map;
158 struct bpf_stab *stab = container_of(map, struct bpf_stab,
159 map);
160 if (psock->parser.enabled && stab->progs.skb_parser)
161 strp_stop = true;
162 if (psock->parser.enabled && stab->progs.skb_verdict)
163 verdict_stop = true;
164 list_del(&link->list);
165 sk_psock_free_link(link);
166 }
167 }
168 spin_unlock_bh(&psock->link_lock);
169 if (strp_stop || verdict_stop) {
170 write_lock_bh(&sk->sk_callback_lock);
171 if (strp_stop)
172 sk_psock_stop_strp(sk, psock);
173 else
174 sk_psock_stop_verdict(sk, psock);
175 write_unlock_bh(&sk->sk_callback_lock);
176 }
177 }
178
sock_map_unref(struct sock * sk,void * link_raw)179 static void sock_map_unref(struct sock *sk, void *link_raw)
180 {
181 struct sk_psock *psock = sk_psock(sk);
182
183 if (likely(psock)) {
184 sock_map_del_link(sk, psock, link_raw);
185 sk_psock_put(sk, psock);
186 }
187 }
188
sock_map_init_proto(struct sock * sk,struct sk_psock * psock)189 static int sock_map_init_proto(struct sock *sk, struct sk_psock *psock)
190 {
191 struct proto *prot;
192
193 switch (sk->sk_type) {
194 case SOCK_STREAM:
195 prot = tcp_bpf_get_proto(sk, psock);
196 break;
197
198 case SOCK_DGRAM:
199 prot = udp_bpf_get_proto(sk, psock);
200 break;
201
202 default:
203 return -EINVAL;
204 }
205
206 if (IS_ERR(prot))
207 return PTR_ERR(prot);
208
209 sk_psock_update_proto(sk, psock, prot);
210 return 0;
211 }
212
sock_map_psock_get_checked(struct sock * sk)213 static struct sk_psock *sock_map_psock_get_checked(struct sock *sk)
214 {
215 struct sk_psock *psock;
216
217 rcu_read_lock();
218 psock = sk_psock(sk);
219 if (psock) {
220 if (sk->sk_prot->close != sock_map_close) {
221 psock = ERR_PTR(-EBUSY);
222 goto out;
223 }
224
225 if (!refcount_inc_not_zero(&psock->refcnt))
226 psock = ERR_PTR(-EBUSY);
227 }
228 out:
229 rcu_read_unlock();
230 return psock;
231 }
232
sock_map_link(struct bpf_map * map,struct sk_psock_progs * progs,struct sock * sk)233 static int sock_map_link(struct bpf_map *map, struct sk_psock_progs *progs,
234 struct sock *sk)
235 {
236 struct bpf_prog *msg_parser, *skb_parser, *skb_verdict;
237 struct sk_psock *psock;
238 int ret;
239
240 skb_verdict = READ_ONCE(progs->skb_verdict);
241 if (skb_verdict) {
242 skb_verdict = bpf_prog_inc_not_zero(skb_verdict);
243 if (IS_ERR(skb_verdict))
244 return PTR_ERR(skb_verdict);
245 }
246
247 skb_parser = READ_ONCE(progs->skb_parser);
248 if (skb_parser) {
249 skb_parser = bpf_prog_inc_not_zero(skb_parser);
250 if (IS_ERR(skb_parser)) {
251 ret = PTR_ERR(skb_parser);
252 goto out_put_skb_verdict;
253 }
254 }
255
256 msg_parser = READ_ONCE(progs->msg_parser);
257 if (msg_parser) {
258 msg_parser = bpf_prog_inc_not_zero(msg_parser);
259 if (IS_ERR(msg_parser)) {
260 ret = PTR_ERR(msg_parser);
261 goto out_put_skb_parser;
262 }
263 }
264
265 psock = sock_map_psock_get_checked(sk);
266 if (IS_ERR(psock)) {
267 ret = PTR_ERR(psock);
268 goto out_progs;
269 }
270
271 if (psock) {
272 if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) ||
273 (skb_parser && READ_ONCE(psock->progs.skb_parser)) ||
274 (skb_verdict && READ_ONCE(psock->progs.skb_verdict))) {
275 sk_psock_put(sk, psock);
276 ret = -EBUSY;
277 goto out_progs;
278 }
279 } else {
280 psock = sk_psock_init(sk, map->numa_node);
281 if (IS_ERR(psock)) {
282 ret = PTR_ERR(psock);
283 goto out_progs;
284 }
285 }
286
287 if (msg_parser)
288 psock_set_prog(&psock->progs.msg_parser, msg_parser);
289
290 ret = sock_map_init_proto(sk, psock);
291 if (ret < 0)
292 goto out_drop;
293
294 write_lock_bh(&sk->sk_callback_lock);
295 if (skb_parser && skb_verdict && !psock->parser.enabled) {
296 ret = sk_psock_init_strp(sk, psock);
297 if (ret)
298 goto out_unlock_drop;
299 psock_set_prog(&psock->progs.skb_verdict, skb_verdict);
300 psock_set_prog(&psock->progs.skb_parser, skb_parser);
301 sk_psock_start_strp(sk, psock);
302 } else if (!skb_parser && skb_verdict && !psock->parser.enabled) {
303 psock_set_prog(&psock->progs.skb_verdict, skb_verdict);
304 sk_psock_start_verdict(sk,psock);
305 }
306 write_unlock_bh(&sk->sk_callback_lock);
307 return 0;
308 out_unlock_drop:
309 write_unlock_bh(&sk->sk_callback_lock);
310 out_drop:
311 sk_psock_put(sk, psock);
312 out_progs:
313 if (msg_parser)
314 bpf_prog_put(msg_parser);
315 out_put_skb_parser:
316 if (skb_parser)
317 bpf_prog_put(skb_parser);
318 out_put_skb_verdict:
319 if (skb_verdict)
320 bpf_prog_put(skb_verdict);
321 return ret;
322 }
323
sock_map_link_no_progs(struct bpf_map * map,struct sock * sk)324 static int sock_map_link_no_progs(struct bpf_map *map, struct sock *sk)
325 {
326 struct sk_psock *psock;
327 int ret;
328
329 psock = sock_map_psock_get_checked(sk);
330 if (IS_ERR(psock))
331 return PTR_ERR(psock);
332
333 if (!psock) {
334 psock = sk_psock_init(sk, map->numa_node);
335 if (IS_ERR(psock))
336 return PTR_ERR(psock);
337 }
338
339 ret = sock_map_init_proto(sk, psock);
340 if (ret < 0)
341 sk_psock_put(sk, psock);
342 return ret;
343 }
344
sock_map_free(struct bpf_map * map)345 static void sock_map_free(struct bpf_map *map)
346 {
347 struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
348 int i;
349
350 /* After the sync no updates or deletes will be in-flight so it
351 * is safe to walk map and remove entries without risking a race
352 * in EEXIST update case.
353 */
354 synchronize_rcu();
355 for (i = 0; i < stab->map.max_entries; i++) {
356 struct sock **psk = &stab->sks[i];
357 struct sock *sk;
358
359 sk = xchg(psk, NULL);
360 if (sk) {
361 sock_hold(sk);
362 lock_sock(sk);
363 rcu_read_lock();
364 sock_map_unref(sk, psk);
365 rcu_read_unlock();
366 release_sock(sk);
367 sock_put(sk);
368 }
369 }
370
371 /* wait for psock readers accessing its map link */
372 synchronize_rcu();
373
374 bpf_map_area_free(stab->sks);
375 kfree(stab);
376 }
377
sock_map_release_progs(struct bpf_map * map)378 static void sock_map_release_progs(struct bpf_map *map)
379 {
380 psock_progs_drop(&container_of(map, struct bpf_stab, map)->progs);
381 }
382
__sock_map_lookup_elem(struct bpf_map * map,u32 key)383 static struct sock *__sock_map_lookup_elem(struct bpf_map *map, u32 key)
384 {
385 struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
386
387 WARN_ON_ONCE(!rcu_read_lock_held());
388
389 if (unlikely(key >= map->max_entries))
390 return NULL;
391 return READ_ONCE(stab->sks[key]);
392 }
393
sock_map_lookup(struct bpf_map * map,void * key)394 static void *sock_map_lookup(struct bpf_map *map, void *key)
395 {
396 struct sock *sk;
397
398 sk = __sock_map_lookup_elem(map, *(u32 *)key);
399 if (!sk)
400 return NULL;
401 if (sk_is_refcounted(sk) && !refcount_inc_not_zero(&sk->sk_refcnt))
402 return NULL;
403 return sk;
404 }
405
sock_map_lookup_sys(struct bpf_map * map,void * key)406 static void *sock_map_lookup_sys(struct bpf_map *map, void *key)
407 {
408 struct sock *sk;
409
410 if (map->value_size != sizeof(u64))
411 return ERR_PTR(-ENOSPC);
412
413 sk = __sock_map_lookup_elem(map, *(u32 *)key);
414 if (!sk)
415 return ERR_PTR(-ENOENT);
416
417 __sock_gen_cookie(sk);
418 return &sk->sk_cookie;
419 }
420
__sock_map_delete(struct bpf_stab * stab,struct sock * sk_test,struct sock ** psk)421 static int __sock_map_delete(struct bpf_stab *stab, struct sock *sk_test,
422 struct sock **psk)
423 {
424 struct sock *sk;
425 int err = 0;
426
427 raw_spin_lock_bh(&stab->lock);
428 sk = *psk;
429 if (!sk_test || sk_test == sk)
430 sk = xchg(psk, NULL);
431
432 if (likely(sk))
433 sock_map_unref(sk, psk);
434 else
435 err = -EINVAL;
436
437 raw_spin_unlock_bh(&stab->lock);
438 return err;
439 }
440
sock_map_delete_from_link(struct bpf_map * map,struct sock * sk,void * link_raw)441 static void sock_map_delete_from_link(struct bpf_map *map, struct sock *sk,
442 void *link_raw)
443 {
444 struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
445
446 __sock_map_delete(stab, sk, link_raw);
447 }
448
sock_map_delete_elem(struct bpf_map * map,void * key)449 static int sock_map_delete_elem(struct bpf_map *map, void *key)
450 {
451 struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
452 u32 i = *(u32 *)key;
453 struct sock **psk;
454
455 if (unlikely(i >= map->max_entries))
456 return -EINVAL;
457
458 psk = &stab->sks[i];
459 return __sock_map_delete(stab, NULL, psk);
460 }
461
sock_map_get_next_key(struct bpf_map * map,void * key,void * next)462 static int sock_map_get_next_key(struct bpf_map *map, void *key, void *next)
463 {
464 struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
465 u32 i = key ? *(u32 *)key : U32_MAX;
466 u32 *key_next = next;
467
468 if (i == stab->map.max_entries - 1)
469 return -ENOENT;
470 if (i >= stab->map.max_entries)
471 *key_next = 0;
472 else
473 *key_next = i + 1;
474 return 0;
475 }
476
477 static bool sock_map_redirect_allowed(const struct sock *sk);
478
sock_map_update_common(struct bpf_map * map,u32 idx,struct sock * sk,u64 flags)479 static int sock_map_update_common(struct bpf_map *map, u32 idx,
480 struct sock *sk, u64 flags)
481 {
482 struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
483 struct sk_psock_link *link;
484 struct sk_psock *psock;
485 struct sock *osk;
486 int ret;
487
488 WARN_ON_ONCE(!rcu_read_lock_held());
489 if (unlikely(flags > BPF_EXIST))
490 return -EINVAL;
491 if (unlikely(idx >= map->max_entries))
492 return -E2BIG;
493
494 link = sk_psock_init_link();
495 if (!link)
496 return -ENOMEM;
497
498 /* Only sockets we can redirect into/from in BPF need to hold
499 * refs to parser/verdict progs and have their sk_data_ready
500 * and sk_write_space callbacks overridden.
501 */
502 if (sock_map_redirect_allowed(sk))
503 ret = sock_map_link(map, &stab->progs, sk);
504 else
505 ret = sock_map_link_no_progs(map, sk);
506 if (ret < 0)
507 goto out_free;
508
509 psock = sk_psock(sk);
510 WARN_ON_ONCE(!psock);
511
512 raw_spin_lock_bh(&stab->lock);
513 osk = stab->sks[idx];
514 if (osk && flags == BPF_NOEXIST) {
515 ret = -EEXIST;
516 goto out_unlock;
517 } else if (!osk && flags == BPF_EXIST) {
518 ret = -ENOENT;
519 goto out_unlock;
520 }
521
522 sock_map_add_link(psock, link, map, &stab->sks[idx]);
523 stab->sks[idx] = sk;
524 if (osk)
525 sock_map_unref(osk, &stab->sks[idx]);
526 raw_spin_unlock_bh(&stab->lock);
527 return 0;
528 out_unlock:
529 raw_spin_unlock_bh(&stab->lock);
530 if (psock)
531 sk_psock_put(sk, psock);
532 out_free:
533 sk_psock_free_link(link);
534 return ret;
535 }
536
sock_map_op_okay(const struct bpf_sock_ops_kern * ops)537 static bool sock_map_op_okay(const struct bpf_sock_ops_kern *ops)
538 {
539 return ops->op == BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB ||
540 ops->op == BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB ||
541 ops->op == BPF_SOCK_OPS_TCP_LISTEN_CB;
542 }
543
sk_is_tcp(const struct sock * sk)544 static bool sk_is_tcp(const struct sock *sk)
545 {
546 return sk->sk_type == SOCK_STREAM &&
547 sk->sk_protocol == IPPROTO_TCP;
548 }
549
sk_is_udp(const struct sock * sk)550 static bool sk_is_udp(const struct sock *sk)
551 {
552 return sk->sk_type == SOCK_DGRAM &&
553 sk->sk_protocol == IPPROTO_UDP;
554 }
555
sock_map_redirect_allowed(const struct sock * sk)556 static bool sock_map_redirect_allowed(const struct sock *sk)
557 {
558 return sk_is_tcp(sk) && sk->sk_state != TCP_LISTEN;
559 }
560
sock_map_sk_is_suitable(const struct sock * sk)561 static bool sock_map_sk_is_suitable(const struct sock *sk)
562 {
563 return sk_is_tcp(sk) || sk_is_udp(sk);
564 }
565
sock_map_sk_state_allowed(const struct sock * sk)566 static bool sock_map_sk_state_allowed(const struct sock *sk)
567 {
568 if (sk_is_tcp(sk))
569 return (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_LISTEN);
570 else if (sk_is_udp(sk))
571 return sk_hashed(sk);
572
573 return false;
574 }
575
576 static int sock_hash_update_common(struct bpf_map *map, void *key,
577 struct sock *sk, u64 flags);
578
sock_map_update_elem_sys(struct bpf_map * map,void * key,void * value,u64 flags)579 int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value,
580 u64 flags)
581 {
582 struct socket *sock;
583 struct sock *sk;
584 int ret;
585 u64 ufd;
586
587 if (map->value_size == sizeof(u64))
588 ufd = *(u64 *)value;
589 else
590 ufd = *(u32 *)value;
591 if (ufd > S32_MAX)
592 return -EINVAL;
593
594 sock = sockfd_lookup(ufd, &ret);
595 if (!sock)
596 return ret;
597 sk = sock->sk;
598 if (!sk) {
599 ret = -EINVAL;
600 goto out;
601 }
602 if (!sock_map_sk_is_suitable(sk)) {
603 ret = -EOPNOTSUPP;
604 goto out;
605 }
606
607 sock_map_sk_acquire(sk);
608 if (!sock_map_sk_state_allowed(sk))
609 ret = -EOPNOTSUPP;
610 else if (map->map_type == BPF_MAP_TYPE_SOCKMAP)
611 ret = sock_map_update_common(map, *(u32 *)key, sk, flags);
612 else
613 ret = sock_hash_update_common(map, key, sk, flags);
614 sock_map_sk_release(sk);
615 out:
616 fput(sock->file);
617 return ret;
618 }
619
sock_map_update_elem(struct bpf_map * map,void * key,void * value,u64 flags)620 static int sock_map_update_elem(struct bpf_map *map, void *key,
621 void *value, u64 flags)
622 {
623 struct sock *sk = (struct sock *)value;
624 int ret;
625
626 if (unlikely(!sk || !sk_fullsock(sk)))
627 return -EINVAL;
628
629 if (!sock_map_sk_is_suitable(sk))
630 return -EOPNOTSUPP;
631
632 local_bh_disable();
633 bh_lock_sock(sk);
634 if (!sock_map_sk_state_allowed(sk))
635 ret = -EOPNOTSUPP;
636 else if (map->map_type == BPF_MAP_TYPE_SOCKMAP)
637 ret = sock_map_update_common(map, *(u32 *)key, sk, flags);
638 else
639 ret = sock_hash_update_common(map, key, sk, flags);
640 bh_unlock_sock(sk);
641 local_bh_enable();
642 return ret;
643 }
644
BPF_CALL_4(bpf_sock_map_update,struct bpf_sock_ops_kern *,sops,struct bpf_map *,map,void *,key,u64,flags)645 BPF_CALL_4(bpf_sock_map_update, struct bpf_sock_ops_kern *, sops,
646 struct bpf_map *, map, void *, key, u64, flags)
647 {
648 WARN_ON_ONCE(!rcu_read_lock_held());
649
650 if (likely(sock_map_sk_is_suitable(sops->sk) &&
651 sock_map_op_okay(sops)))
652 return sock_map_update_common(map, *(u32 *)key, sops->sk,
653 flags);
654 return -EOPNOTSUPP;
655 }
656
657 const struct bpf_func_proto bpf_sock_map_update_proto = {
658 .func = bpf_sock_map_update,
659 .gpl_only = false,
660 .pkt_access = true,
661 .ret_type = RET_INTEGER,
662 .arg1_type = ARG_PTR_TO_CTX,
663 .arg2_type = ARG_CONST_MAP_PTR,
664 .arg3_type = ARG_PTR_TO_MAP_KEY,
665 .arg4_type = ARG_ANYTHING,
666 };
667
BPF_CALL_4(bpf_sk_redirect_map,struct sk_buff *,skb,struct bpf_map *,map,u32,key,u64,flags)668 BPF_CALL_4(bpf_sk_redirect_map, struct sk_buff *, skb,
669 struct bpf_map *, map, u32, key, u64, flags)
670 {
671 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
672 struct sock *sk;
673
674 if (unlikely(flags & ~(BPF_F_INGRESS)))
675 return SK_DROP;
676
677 sk = __sock_map_lookup_elem(map, key);
678 if (unlikely(!sk || !sock_map_redirect_allowed(sk)))
679 return SK_DROP;
680
681 tcb->bpf.flags = flags;
682 tcb->bpf.sk_redir = sk;
683 return SK_PASS;
684 }
685
686 const struct bpf_func_proto bpf_sk_redirect_map_proto = {
687 .func = bpf_sk_redirect_map,
688 .gpl_only = false,
689 .ret_type = RET_INTEGER,
690 .arg1_type = ARG_PTR_TO_CTX,
691 .arg2_type = ARG_CONST_MAP_PTR,
692 .arg3_type = ARG_ANYTHING,
693 .arg4_type = ARG_ANYTHING,
694 };
695
BPF_CALL_4(bpf_msg_redirect_map,struct sk_msg *,msg,struct bpf_map *,map,u32,key,u64,flags)696 BPF_CALL_4(bpf_msg_redirect_map, struct sk_msg *, msg,
697 struct bpf_map *, map, u32, key, u64, flags)
698 {
699 struct sock *sk;
700
701 if (unlikely(flags & ~(BPF_F_INGRESS)))
702 return SK_DROP;
703
704 sk = __sock_map_lookup_elem(map, key);
705 if (unlikely(!sk || !sock_map_redirect_allowed(sk)))
706 return SK_DROP;
707
708 msg->flags = flags;
709 msg->sk_redir = sk;
710 return SK_PASS;
711 }
712
713 const struct bpf_func_proto bpf_msg_redirect_map_proto = {
714 .func = bpf_msg_redirect_map,
715 .gpl_only = false,
716 .ret_type = RET_INTEGER,
717 .arg1_type = ARG_PTR_TO_CTX,
718 .arg2_type = ARG_CONST_MAP_PTR,
719 .arg3_type = ARG_ANYTHING,
720 .arg4_type = ARG_ANYTHING,
721 };
722
723 struct sock_map_seq_info {
724 struct bpf_map *map;
725 struct sock *sk;
726 u32 index;
727 };
728
729 struct bpf_iter__sockmap {
730 __bpf_md_ptr(struct bpf_iter_meta *, meta);
731 __bpf_md_ptr(struct bpf_map *, map);
732 __bpf_md_ptr(void *, key);
733 __bpf_md_ptr(struct sock *, sk);
734 };
735
DEFINE_BPF_ITER_FUNC(sockmap,struct bpf_iter_meta * meta,struct bpf_map * map,void * key,struct sock * sk)736 DEFINE_BPF_ITER_FUNC(sockmap, struct bpf_iter_meta *meta,
737 struct bpf_map *map, void *key,
738 struct sock *sk)
739
740 static void *sock_map_seq_lookup_elem(struct sock_map_seq_info *info)
741 {
742 if (unlikely(info->index >= info->map->max_entries))
743 return NULL;
744
745 info->sk = __sock_map_lookup_elem(info->map, info->index);
746
747 /* can't return sk directly, since that might be NULL */
748 return info;
749 }
750
sock_map_seq_start(struct seq_file * seq,loff_t * pos)751 static void *sock_map_seq_start(struct seq_file *seq, loff_t *pos)
752 __acquires(rcu)
753 {
754 struct sock_map_seq_info *info = seq->private;
755
756 if (*pos == 0)
757 ++*pos;
758
759 /* pairs with sock_map_seq_stop */
760 rcu_read_lock();
761 return sock_map_seq_lookup_elem(info);
762 }
763
sock_map_seq_next(struct seq_file * seq,void * v,loff_t * pos)764 static void *sock_map_seq_next(struct seq_file *seq, void *v, loff_t *pos)
765 __must_hold(rcu)
766 {
767 struct sock_map_seq_info *info = seq->private;
768
769 ++*pos;
770 ++info->index;
771
772 return sock_map_seq_lookup_elem(info);
773 }
774
sock_map_seq_show(struct seq_file * seq,void * v)775 static int sock_map_seq_show(struct seq_file *seq, void *v)
776 __must_hold(rcu)
777 {
778 struct sock_map_seq_info *info = seq->private;
779 struct bpf_iter__sockmap ctx = {};
780 struct bpf_iter_meta meta;
781 struct bpf_prog *prog;
782
783 meta.seq = seq;
784 prog = bpf_iter_get_info(&meta, !v);
785 if (!prog)
786 return 0;
787
788 ctx.meta = &meta;
789 ctx.map = info->map;
790 if (v) {
791 ctx.key = &info->index;
792 ctx.sk = info->sk;
793 }
794
795 return bpf_iter_run_prog(prog, &ctx);
796 }
797
sock_map_seq_stop(struct seq_file * seq,void * v)798 static void sock_map_seq_stop(struct seq_file *seq, void *v)
799 __releases(rcu)
800 {
801 if (!v)
802 (void)sock_map_seq_show(seq, NULL);
803
804 /* pairs with sock_map_seq_start */
805 rcu_read_unlock();
806 }
807
808 static const struct seq_operations sock_map_seq_ops = {
809 .start = sock_map_seq_start,
810 .next = sock_map_seq_next,
811 .stop = sock_map_seq_stop,
812 .show = sock_map_seq_show,
813 };
814
sock_map_init_seq_private(void * priv_data,struct bpf_iter_aux_info * aux)815 static int sock_map_init_seq_private(void *priv_data,
816 struct bpf_iter_aux_info *aux)
817 {
818 struct sock_map_seq_info *info = priv_data;
819
820 bpf_map_inc_with_uref(aux->map);
821 info->map = aux->map;
822 return 0;
823 }
824
sock_map_fini_seq_private(void * priv_data)825 static void sock_map_fini_seq_private(void *priv_data)
826 {
827 struct sock_map_seq_info *info = priv_data;
828
829 bpf_map_put_with_uref(info->map);
830 }
831
832 static const struct bpf_iter_seq_info sock_map_iter_seq_info = {
833 .seq_ops = &sock_map_seq_ops,
834 .init_seq_private = sock_map_init_seq_private,
835 .fini_seq_private = sock_map_fini_seq_private,
836 .seq_priv_size = sizeof(struct sock_map_seq_info),
837 };
838
839 static int sock_map_btf_id;
840 const struct bpf_map_ops sock_map_ops = {
841 .map_meta_equal = bpf_map_meta_equal,
842 .map_alloc = sock_map_alloc,
843 .map_free = sock_map_free,
844 .map_get_next_key = sock_map_get_next_key,
845 .map_lookup_elem_sys_only = sock_map_lookup_sys,
846 .map_update_elem = sock_map_update_elem,
847 .map_delete_elem = sock_map_delete_elem,
848 .map_lookup_elem = sock_map_lookup,
849 .map_release_uref = sock_map_release_progs,
850 .map_check_btf = map_check_no_btf,
851 .map_btf_name = "bpf_stab",
852 .map_btf_id = &sock_map_btf_id,
853 .iter_seq_info = &sock_map_iter_seq_info,
854 };
855
856 struct bpf_shtab_elem {
857 struct rcu_head rcu;
858 u32 hash;
859 struct sock *sk;
860 struct hlist_node node;
861 u8 key[];
862 };
863
864 struct bpf_shtab_bucket {
865 struct hlist_head head;
866 raw_spinlock_t lock;
867 };
868
869 struct bpf_shtab {
870 struct bpf_map map;
871 struct bpf_shtab_bucket *buckets;
872 u32 buckets_num;
873 u32 elem_size;
874 struct sk_psock_progs progs;
875 atomic_t count;
876 };
877
sock_hash_bucket_hash(const void * key,u32 len)878 static inline u32 sock_hash_bucket_hash(const void *key, u32 len)
879 {
880 return jhash(key, len, 0);
881 }
882
sock_hash_select_bucket(struct bpf_shtab * htab,u32 hash)883 static struct bpf_shtab_bucket *sock_hash_select_bucket(struct bpf_shtab *htab,
884 u32 hash)
885 {
886 return &htab->buckets[hash & (htab->buckets_num - 1)];
887 }
888
889 static struct bpf_shtab_elem *
sock_hash_lookup_elem_raw(struct hlist_head * head,u32 hash,void * key,u32 key_size)890 sock_hash_lookup_elem_raw(struct hlist_head *head, u32 hash, void *key,
891 u32 key_size)
892 {
893 struct bpf_shtab_elem *elem;
894
895 hlist_for_each_entry_rcu(elem, head, node) {
896 if (elem->hash == hash &&
897 !memcmp(&elem->key, key, key_size))
898 return elem;
899 }
900
901 return NULL;
902 }
903
__sock_hash_lookup_elem(struct bpf_map * map,void * key)904 static struct sock *__sock_hash_lookup_elem(struct bpf_map *map, void *key)
905 {
906 struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
907 u32 key_size = map->key_size, hash;
908 struct bpf_shtab_bucket *bucket;
909 struct bpf_shtab_elem *elem;
910
911 WARN_ON_ONCE(!rcu_read_lock_held());
912
913 hash = sock_hash_bucket_hash(key, key_size);
914 bucket = sock_hash_select_bucket(htab, hash);
915 elem = sock_hash_lookup_elem_raw(&bucket->head, hash, key, key_size);
916
917 return elem ? elem->sk : NULL;
918 }
919
sock_hash_free_elem(struct bpf_shtab * htab,struct bpf_shtab_elem * elem)920 static void sock_hash_free_elem(struct bpf_shtab *htab,
921 struct bpf_shtab_elem *elem)
922 {
923 atomic_dec(&htab->count);
924 kfree_rcu(elem, rcu);
925 }
926
sock_hash_delete_from_link(struct bpf_map * map,struct sock * sk,void * link_raw)927 static void sock_hash_delete_from_link(struct bpf_map *map, struct sock *sk,
928 void *link_raw)
929 {
930 struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
931 struct bpf_shtab_elem *elem_probe, *elem = link_raw;
932 struct bpf_shtab_bucket *bucket;
933
934 WARN_ON_ONCE(!rcu_read_lock_held());
935 bucket = sock_hash_select_bucket(htab, elem->hash);
936
937 /* elem may be deleted in parallel from the map, but access here
938 * is okay since it's going away only after RCU grace period.
939 * However, we need to check whether it's still present.
940 */
941 raw_spin_lock_bh(&bucket->lock);
942 elem_probe = sock_hash_lookup_elem_raw(&bucket->head, elem->hash,
943 elem->key, map->key_size);
944 if (elem_probe && elem_probe == elem) {
945 hlist_del_rcu(&elem->node);
946 sock_map_unref(elem->sk, elem);
947 sock_hash_free_elem(htab, elem);
948 }
949 raw_spin_unlock_bh(&bucket->lock);
950 }
951
sock_hash_delete_elem(struct bpf_map * map,void * key)952 static int sock_hash_delete_elem(struct bpf_map *map, void *key)
953 {
954 struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
955 u32 hash, key_size = map->key_size;
956 struct bpf_shtab_bucket *bucket;
957 struct bpf_shtab_elem *elem;
958 int ret = -ENOENT;
959
960 hash = sock_hash_bucket_hash(key, key_size);
961 bucket = sock_hash_select_bucket(htab, hash);
962
963 raw_spin_lock_bh(&bucket->lock);
964 elem = sock_hash_lookup_elem_raw(&bucket->head, hash, key, key_size);
965 if (elem) {
966 hlist_del_rcu(&elem->node);
967 sock_map_unref(elem->sk, elem);
968 sock_hash_free_elem(htab, elem);
969 ret = 0;
970 }
971 raw_spin_unlock_bh(&bucket->lock);
972 return ret;
973 }
974
sock_hash_alloc_elem(struct bpf_shtab * htab,void * key,u32 key_size,u32 hash,struct sock * sk,struct bpf_shtab_elem * old)975 static struct bpf_shtab_elem *sock_hash_alloc_elem(struct bpf_shtab *htab,
976 void *key, u32 key_size,
977 u32 hash, struct sock *sk,
978 struct bpf_shtab_elem *old)
979 {
980 struct bpf_shtab_elem *new;
981
982 if (atomic_inc_return(&htab->count) > htab->map.max_entries) {
983 if (!old) {
984 atomic_dec(&htab->count);
985 return ERR_PTR(-E2BIG);
986 }
987 }
988
989 new = kmalloc_node(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN,
990 htab->map.numa_node);
991 if (!new) {
992 atomic_dec(&htab->count);
993 return ERR_PTR(-ENOMEM);
994 }
995 memcpy(new->key, key, key_size);
996 new->sk = sk;
997 new->hash = hash;
998 return new;
999 }
1000
sock_hash_update_common(struct bpf_map * map,void * key,struct sock * sk,u64 flags)1001 static int sock_hash_update_common(struct bpf_map *map, void *key,
1002 struct sock *sk, u64 flags)
1003 {
1004 struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
1005 u32 key_size = map->key_size, hash;
1006 struct bpf_shtab_elem *elem, *elem_new;
1007 struct bpf_shtab_bucket *bucket;
1008 struct sk_psock_link *link;
1009 struct sk_psock *psock;
1010 int ret;
1011
1012 WARN_ON_ONCE(!rcu_read_lock_held());
1013 if (unlikely(flags > BPF_EXIST))
1014 return -EINVAL;
1015
1016 link = sk_psock_init_link();
1017 if (!link)
1018 return -ENOMEM;
1019
1020 /* Only sockets we can redirect into/from in BPF need to hold
1021 * refs to parser/verdict progs and have their sk_data_ready
1022 * and sk_write_space callbacks overridden.
1023 */
1024 if (sock_map_redirect_allowed(sk))
1025 ret = sock_map_link(map, &htab->progs, sk);
1026 else
1027 ret = sock_map_link_no_progs(map, sk);
1028 if (ret < 0)
1029 goto out_free;
1030
1031 psock = sk_psock(sk);
1032 WARN_ON_ONCE(!psock);
1033
1034 hash = sock_hash_bucket_hash(key, key_size);
1035 bucket = sock_hash_select_bucket(htab, hash);
1036
1037 raw_spin_lock_bh(&bucket->lock);
1038 elem = sock_hash_lookup_elem_raw(&bucket->head, hash, key, key_size);
1039 if (elem && flags == BPF_NOEXIST) {
1040 ret = -EEXIST;
1041 goto out_unlock;
1042 } else if (!elem && flags == BPF_EXIST) {
1043 ret = -ENOENT;
1044 goto out_unlock;
1045 }
1046
1047 elem_new = sock_hash_alloc_elem(htab, key, key_size, hash, sk, elem);
1048 if (IS_ERR(elem_new)) {
1049 ret = PTR_ERR(elem_new);
1050 goto out_unlock;
1051 }
1052
1053 sock_map_add_link(psock, link, map, elem_new);
1054 /* Add new element to the head of the list, so that
1055 * concurrent search will find it before old elem.
1056 */
1057 hlist_add_head_rcu(&elem_new->node, &bucket->head);
1058 if (elem) {
1059 hlist_del_rcu(&elem->node);
1060 sock_map_unref(elem->sk, elem);
1061 sock_hash_free_elem(htab, elem);
1062 }
1063 raw_spin_unlock_bh(&bucket->lock);
1064 return 0;
1065 out_unlock:
1066 raw_spin_unlock_bh(&bucket->lock);
1067 sk_psock_put(sk, psock);
1068 out_free:
1069 sk_psock_free_link(link);
1070 return ret;
1071 }
1072
sock_hash_get_next_key(struct bpf_map * map,void * key,void * key_next)1073 static int sock_hash_get_next_key(struct bpf_map *map, void *key,
1074 void *key_next)
1075 {
1076 struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
1077 struct bpf_shtab_elem *elem, *elem_next;
1078 u32 hash, key_size = map->key_size;
1079 struct hlist_head *head;
1080 int i = 0;
1081
1082 if (!key)
1083 goto find_first_elem;
1084 hash = sock_hash_bucket_hash(key, key_size);
1085 head = &sock_hash_select_bucket(htab, hash)->head;
1086 elem = sock_hash_lookup_elem_raw(head, hash, key, key_size);
1087 if (!elem)
1088 goto find_first_elem;
1089
1090 elem_next = hlist_entry_safe(rcu_dereference(hlist_next_rcu(&elem->node)),
1091 struct bpf_shtab_elem, node);
1092 if (elem_next) {
1093 memcpy(key_next, elem_next->key, key_size);
1094 return 0;
1095 }
1096
1097 i = hash & (htab->buckets_num - 1);
1098 i++;
1099 find_first_elem:
1100 for (; i < htab->buckets_num; i++) {
1101 head = &sock_hash_select_bucket(htab, i)->head;
1102 elem_next = hlist_entry_safe(rcu_dereference(hlist_first_rcu(head)),
1103 struct bpf_shtab_elem, node);
1104 if (elem_next) {
1105 memcpy(key_next, elem_next->key, key_size);
1106 return 0;
1107 }
1108 }
1109
1110 return -ENOENT;
1111 }
1112
sock_hash_alloc(union bpf_attr * attr)1113 static struct bpf_map *sock_hash_alloc(union bpf_attr *attr)
1114 {
1115 struct bpf_shtab *htab;
1116 int i, err;
1117 u64 cost;
1118
1119 if (!capable(CAP_NET_ADMIN))
1120 return ERR_PTR(-EPERM);
1121 if (attr->max_entries == 0 ||
1122 attr->key_size == 0 ||
1123 (attr->value_size != sizeof(u32) &&
1124 attr->value_size != sizeof(u64)) ||
1125 attr->map_flags & ~SOCK_CREATE_FLAG_MASK)
1126 return ERR_PTR(-EINVAL);
1127 if (attr->key_size > MAX_BPF_STACK)
1128 return ERR_PTR(-E2BIG);
1129
1130 htab = kzalloc(sizeof(*htab), GFP_USER);
1131 if (!htab)
1132 return ERR_PTR(-ENOMEM);
1133
1134 bpf_map_init_from_attr(&htab->map, attr);
1135
1136 htab->buckets_num = roundup_pow_of_two(htab->map.max_entries);
1137 htab->elem_size = sizeof(struct bpf_shtab_elem) +
1138 round_up(htab->map.key_size, 8);
1139 if (htab->buckets_num == 0 ||
1140 htab->buckets_num > U32_MAX / sizeof(struct bpf_shtab_bucket)) {
1141 err = -EINVAL;
1142 goto free_htab;
1143 }
1144
1145 cost = (u64) htab->buckets_num * sizeof(struct bpf_shtab_bucket) +
1146 (u64) htab->elem_size * htab->map.max_entries;
1147 if (cost >= U32_MAX - PAGE_SIZE) {
1148 err = -EINVAL;
1149 goto free_htab;
1150 }
1151 err = bpf_map_charge_init(&htab->map.memory, cost);
1152 if (err)
1153 goto free_htab;
1154
1155 htab->buckets = bpf_map_area_alloc(htab->buckets_num *
1156 sizeof(struct bpf_shtab_bucket),
1157 htab->map.numa_node);
1158 if (!htab->buckets) {
1159 bpf_map_charge_finish(&htab->map.memory);
1160 err = -ENOMEM;
1161 goto free_htab;
1162 }
1163
1164 for (i = 0; i < htab->buckets_num; i++) {
1165 INIT_HLIST_HEAD(&htab->buckets[i].head);
1166 raw_spin_lock_init(&htab->buckets[i].lock);
1167 }
1168
1169 return &htab->map;
1170 free_htab:
1171 kfree(htab);
1172 return ERR_PTR(err);
1173 }
1174
sock_hash_free(struct bpf_map * map)1175 static void sock_hash_free(struct bpf_map *map)
1176 {
1177 struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);
1178 struct bpf_shtab_bucket *bucket;
1179 struct hlist_head unlink_list;
1180 struct bpf_shtab_elem *elem;
1181 struct hlist_node *node;
1182 int i;
1183
1184 /* After the sync no updates or deletes will be in-flight so it
1185 * is safe to walk map and remove entries without risking a race
1186 * in EEXIST update case.
1187 */
1188 synchronize_rcu();
1189 for (i = 0; i < htab->buckets_num; i++) {
1190 bucket = sock_hash_select_bucket(htab, i);
1191
1192 /* We are racing with sock_hash_delete_from_link to
1193 * enter the spin-lock critical section. Every socket on
1194 * the list is still linked to sockhash. Since link
1195 * exists, psock exists and holds a ref to socket. That
1196 * lets us to grab a socket ref too.
1197 */
1198 raw_spin_lock_bh(&bucket->lock);
1199 hlist_for_each_entry(elem, &bucket->head, node)
1200 sock_hold(elem->sk);
1201 hlist_move_list(&bucket->head, &unlink_list);
1202 raw_spin_unlock_bh(&bucket->lock);
1203
1204 /* Process removed entries out of atomic context to
1205 * block for socket lock before deleting the psock's
1206 * link to sockhash.
1207 */
1208 hlist_for_each_entry_safe(elem, node, &unlink_list, node) {
1209 hlist_del(&elem->node);
1210 lock_sock(elem->sk);
1211 rcu_read_lock();
1212 sock_map_unref(elem->sk, elem);
1213 rcu_read_unlock();
1214 release_sock(elem->sk);
1215 sock_put(elem->sk);
1216 sock_hash_free_elem(htab, elem);
1217 }
1218 }
1219
1220 /* wait for psock readers accessing its map link */
1221 synchronize_rcu();
1222
1223 bpf_map_area_free(htab->buckets);
1224 kfree(htab);
1225 }
1226
sock_hash_lookup_sys(struct bpf_map * map,void * key)1227 static void *sock_hash_lookup_sys(struct bpf_map *map, void *key)
1228 {
1229 struct sock *sk;
1230
1231 if (map->value_size != sizeof(u64))
1232 return ERR_PTR(-ENOSPC);
1233
1234 sk = __sock_hash_lookup_elem(map, key);
1235 if (!sk)
1236 return ERR_PTR(-ENOENT);
1237
1238 __sock_gen_cookie(sk);
1239 return &sk->sk_cookie;
1240 }
1241
sock_hash_lookup(struct bpf_map * map,void * key)1242 static void *sock_hash_lookup(struct bpf_map *map, void *key)
1243 {
1244 struct sock *sk;
1245
1246 sk = __sock_hash_lookup_elem(map, key);
1247 if (!sk)
1248 return NULL;
1249 if (sk_is_refcounted(sk) && !refcount_inc_not_zero(&sk->sk_refcnt))
1250 return NULL;
1251 return sk;
1252 }
1253
sock_hash_release_progs(struct bpf_map * map)1254 static void sock_hash_release_progs(struct bpf_map *map)
1255 {
1256 psock_progs_drop(&container_of(map, struct bpf_shtab, map)->progs);
1257 }
1258
BPF_CALL_4(bpf_sock_hash_update,struct bpf_sock_ops_kern *,sops,struct bpf_map *,map,void *,key,u64,flags)1259 BPF_CALL_4(bpf_sock_hash_update, struct bpf_sock_ops_kern *, sops,
1260 struct bpf_map *, map, void *, key, u64, flags)
1261 {
1262 WARN_ON_ONCE(!rcu_read_lock_held());
1263
1264 if (likely(sock_map_sk_is_suitable(sops->sk) &&
1265 sock_map_op_okay(sops)))
1266 return sock_hash_update_common(map, key, sops->sk, flags);
1267 return -EOPNOTSUPP;
1268 }
1269
1270 const struct bpf_func_proto bpf_sock_hash_update_proto = {
1271 .func = bpf_sock_hash_update,
1272 .gpl_only = false,
1273 .pkt_access = true,
1274 .ret_type = RET_INTEGER,
1275 .arg1_type = ARG_PTR_TO_CTX,
1276 .arg2_type = ARG_CONST_MAP_PTR,
1277 .arg3_type = ARG_PTR_TO_MAP_KEY,
1278 .arg4_type = ARG_ANYTHING,
1279 };
1280
BPF_CALL_4(bpf_sk_redirect_hash,struct sk_buff *,skb,struct bpf_map *,map,void *,key,u64,flags)1281 BPF_CALL_4(bpf_sk_redirect_hash, struct sk_buff *, skb,
1282 struct bpf_map *, map, void *, key, u64, flags)
1283 {
1284 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
1285 struct sock *sk;
1286
1287 if (unlikely(flags & ~(BPF_F_INGRESS)))
1288 return SK_DROP;
1289
1290 sk = __sock_hash_lookup_elem(map, key);
1291 if (unlikely(!sk || !sock_map_redirect_allowed(sk)))
1292 return SK_DROP;
1293
1294 tcb->bpf.flags = flags;
1295 tcb->bpf.sk_redir = sk;
1296 return SK_PASS;
1297 }
1298
1299 const struct bpf_func_proto bpf_sk_redirect_hash_proto = {
1300 .func = bpf_sk_redirect_hash,
1301 .gpl_only = false,
1302 .ret_type = RET_INTEGER,
1303 .arg1_type = ARG_PTR_TO_CTX,
1304 .arg2_type = ARG_CONST_MAP_PTR,
1305 .arg3_type = ARG_PTR_TO_MAP_KEY,
1306 .arg4_type = ARG_ANYTHING,
1307 };
1308
BPF_CALL_4(bpf_msg_redirect_hash,struct sk_msg *,msg,struct bpf_map *,map,void *,key,u64,flags)1309 BPF_CALL_4(bpf_msg_redirect_hash, struct sk_msg *, msg,
1310 struct bpf_map *, map, void *, key, u64, flags)
1311 {
1312 struct sock *sk;
1313
1314 if (unlikely(flags & ~(BPF_F_INGRESS)))
1315 return SK_DROP;
1316
1317 sk = __sock_hash_lookup_elem(map, key);
1318 if (unlikely(!sk || !sock_map_redirect_allowed(sk)))
1319 return SK_DROP;
1320
1321 msg->flags = flags;
1322 msg->sk_redir = sk;
1323 return SK_PASS;
1324 }
1325
1326 const struct bpf_func_proto bpf_msg_redirect_hash_proto = {
1327 .func = bpf_msg_redirect_hash,
1328 .gpl_only = false,
1329 .ret_type = RET_INTEGER,
1330 .arg1_type = ARG_PTR_TO_CTX,
1331 .arg2_type = ARG_CONST_MAP_PTR,
1332 .arg3_type = ARG_PTR_TO_MAP_KEY,
1333 .arg4_type = ARG_ANYTHING,
1334 };
1335
1336 struct sock_hash_seq_info {
1337 struct bpf_map *map;
1338 struct bpf_shtab *htab;
1339 u32 bucket_id;
1340 };
1341
sock_hash_seq_find_next(struct sock_hash_seq_info * info,struct bpf_shtab_elem * prev_elem)1342 static void *sock_hash_seq_find_next(struct sock_hash_seq_info *info,
1343 struct bpf_shtab_elem *prev_elem)
1344 {
1345 const struct bpf_shtab *htab = info->htab;
1346 struct bpf_shtab_bucket *bucket;
1347 struct bpf_shtab_elem *elem;
1348 struct hlist_node *node;
1349
1350 /* try to find next elem in the same bucket */
1351 if (prev_elem) {
1352 node = rcu_dereference(hlist_next_rcu(&prev_elem->node));
1353 elem = hlist_entry_safe(node, struct bpf_shtab_elem, node);
1354 if (elem)
1355 return elem;
1356
1357 /* no more elements, continue in the next bucket */
1358 info->bucket_id++;
1359 }
1360
1361 for (; info->bucket_id < htab->buckets_num; info->bucket_id++) {
1362 bucket = &htab->buckets[info->bucket_id];
1363 node = rcu_dereference(hlist_first_rcu(&bucket->head));
1364 elem = hlist_entry_safe(node, struct bpf_shtab_elem, node);
1365 if (elem)
1366 return elem;
1367 }
1368
1369 return NULL;
1370 }
1371
sock_hash_seq_start(struct seq_file * seq,loff_t * pos)1372 static void *sock_hash_seq_start(struct seq_file *seq, loff_t *pos)
1373 __acquires(rcu)
1374 {
1375 struct sock_hash_seq_info *info = seq->private;
1376
1377 if (*pos == 0)
1378 ++*pos;
1379
1380 /* pairs with sock_hash_seq_stop */
1381 rcu_read_lock();
1382 return sock_hash_seq_find_next(info, NULL);
1383 }
1384
sock_hash_seq_next(struct seq_file * seq,void * v,loff_t * pos)1385 static void *sock_hash_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1386 __must_hold(rcu)
1387 {
1388 struct sock_hash_seq_info *info = seq->private;
1389
1390 ++*pos;
1391 return sock_hash_seq_find_next(info, v);
1392 }
1393
sock_hash_seq_show(struct seq_file * seq,void * v)1394 static int sock_hash_seq_show(struct seq_file *seq, void *v)
1395 __must_hold(rcu)
1396 {
1397 struct sock_hash_seq_info *info = seq->private;
1398 struct bpf_iter__sockmap ctx = {};
1399 struct bpf_shtab_elem *elem = v;
1400 struct bpf_iter_meta meta;
1401 struct bpf_prog *prog;
1402
1403 meta.seq = seq;
1404 prog = bpf_iter_get_info(&meta, !elem);
1405 if (!prog)
1406 return 0;
1407
1408 ctx.meta = &meta;
1409 ctx.map = info->map;
1410 if (elem) {
1411 ctx.key = elem->key;
1412 ctx.sk = elem->sk;
1413 }
1414
1415 return bpf_iter_run_prog(prog, &ctx);
1416 }
1417
sock_hash_seq_stop(struct seq_file * seq,void * v)1418 static void sock_hash_seq_stop(struct seq_file *seq, void *v)
1419 __releases(rcu)
1420 {
1421 if (!v)
1422 (void)sock_hash_seq_show(seq, NULL);
1423
1424 /* pairs with sock_hash_seq_start */
1425 rcu_read_unlock();
1426 }
1427
1428 static const struct seq_operations sock_hash_seq_ops = {
1429 .start = sock_hash_seq_start,
1430 .next = sock_hash_seq_next,
1431 .stop = sock_hash_seq_stop,
1432 .show = sock_hash_seq_show,
1433 };
1434
sock_hash_init_seq_private(void * priv_data,struct bpf_iter_aux_info * aux)1435 static int sock_hash_init_seq_private(void *priv_data,
1436 struct bpf_iter_aux_info *aux)
1437 {
1438 struct sock_hash_seq_info *info = priv_data;
1439
1440 bpf_map_inc_with_uref(aux->map);
1441 info->map = aux->map;
1442 info->htab = container_of(aux->map, struct bpf_shtab, map);
1443 return 0;
1444 }
1445
sock_hash_fini_seq_private(void * priv_data)1446 static void sock_hash_fini_seq_private(void *priv_data)
1447 {
1448 struct sock_hash_seq_info *info = priv_data;
1449
1450 bpf_map_put_with_uref(info->map);
1451 }
1452
1453 static const struct bpf_iter_seq_info sock_hash_iter_seq_info = {
1454 .seq_ops = &sock_hash_seq_ops,
1455 .init_seq_private = sock_hash_init_seq_private,
1456 .fini_seq_private = sock_hash_fini_seq_private,
1457 .seq_priv_size = sizeof(struct sock_hash_seq_info),
1458 };
1459
1460 static int sock_hash_map_btf_id;
1461 const struct bpf_map_ops sock_hash_ops = {
1462 .map_meta_equal = bpf_map_meta_equal,
1463 .map_alloc = sock_hash_alloc,
1464 .map_free = sock_hash_free,
1465 .map_get_next_key = sock_hash_get_next_key,
1466 .map_update_elem = sock_map_update_elem,
1467 .map_delete_elem = sock_hash_delete_elem,
1468 .map_lookup_elem = sock_hash_lookup,
1469 .map_lookup_elem_sys_only = sock_hash_lookup_sys,
1470 .map_release_uref = sock_hash_release_progs,
1471 .map_check_btf = map_check_no_btf,
1472 .map_btf_name = "bpf_shtab",
1473 .map_btf_id = &sock_hash_map_btf_id,
1474 .iter_seq_info = &sock_hash_iter_seq_info,
1475 };
1476
sock_map_progs(struct bpf_map * map)1477 static struct sk_psock_progs *sock_map_progs(struct bpf_map *map)
1478 {
1479 switch (map->map_type) {
1480 case BPF_MAP_TYPE_SOCKMAP:
1481 return &container_of(map, struct bpf_stab, map)->progs;
1482 case BPF_MAP_TYPE_SOCKHASH:
1483 return &container_of(map, struct bpf_shtab, map)->progs;
1484 default:
1485 break;
1486 }
1487
1488 return NULL;
1489 }
1490
sock_map_prog_update(struct bpf_map * map,struct bpf_prog * prog,struct bpf_prog * old,u32 which)1491 int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog,
1492 struct bpf_prog *old, u32 which)
1493 {
1494 struct sk_psock_progs *progs = sock_map_progs(map);
1495 struct bpf_prog **pprog;
1496
1497 if (!progs)
1498 return -EOPNOTSUPP;
1499
1500 switch (which) {
1501 case BPF_SK_MSG_VERDICT:
1502 pprog = &progs->msg_parser;
1503 break;
1504 case BPF_SK_SKB_STREAM_PARSER:
1505 pprog = &progs->skb_parser;
1506 break;
1507 case BPF_SK_SKB_STREAM_VERDICT:
1508 pprog = &progs->skb_verdict;
1509 break;
1510 default:
1511 return -EOPNOTSUPP;
1512 }
1513
1514 if (old)
1515 return psock_replace_prog(pprog, prog, old);
1516
1517 psock_set_prog(pprog, prog);
1518 return 0;
1519 }
1520
sock_map_unlink(struct sock * sk,struct sk_psock_link * link)1521 static void sock_map_unlink(struct sock *sk, struct sk_psock_link *link)
1522 {
1523 switch (link->map->map_type) {
1524 case BPF_MAP_TYPE_SOCKMAP:
1525 return sock_map_delete_from_link(link->map, sk,
1526 link->link_raw);
1527 case BPF_MAP_TYPE_SOCKHASH:
1528 return sock_hash_delete_from_link(link->map, sk,
1529 link->link_raw);
1530 default:
1531 break;
1532 }
1533 }
1534
sock_map_remove_links(struct sock * sk,struct sk_psock * psock)1535 static void sock_map_remove_links(struct sock *sk, struct sk_psock *psock)
1536 {
1537 struct sk_psock_link *link;
1538
1539 while ((link = sk_psock_link_pop(psock))) {
1540 sock_map_unlink(sk, link);
1541 sk_psock_free_link(link);
1542 }
1543 }
1544
sock_map_unhash(struct sock * sk)1545 void sock_map_unhash(struct sock *sk)
1546 {
1547 void (*saved_unhash)(struct sock *sk);
1548 struct sk_psock *psock;
1549
1550 rcu_read_lock();
1551 psock = sk_psock(sk);
1552 if (unlikely(!psock)) {
1553 rcu_read_unlock();
1554 if (sk->sk_prot->unhash)
1555 sk->sk_prot->unhash(sk);
1556 return;
1557 }
1558
1559 saved_unhash = psock->saved_unhash;
1560 sock_map_remove_links(sk, psock);
1561 rcu_read_unlock();
1562 saved_unhash(sk);
1563 }
1564
sock_map_close(struct sock * sk,long timeout)1565 void sock_map_close(struct sock *sk, long timeout)
1566 {
1567 void (*saved_close)(struct sock *sk, long timeout);
1568 struct sk_psock *psock;
1569
1570 lock_sock(sk);
1571 rcu_read_lock();
1572 psock = sk_psock(sk);
1573 if (unlikely(!psock)) {
1574 rcu_read_unlock();
1575 release_sock(sk);
1576 return sk->sk_prot->close(sk, timeout);
1577 }
1578
1579 saved_close = psock->saved_close;
1580 sock_map_remove_links(sk, psock);
1581 rcu_read_unlock();
1582 release_sock(sk);
1583 saved_close(sk, timeout);
1584 }
1585
sock_map_iter_attach_target(struct bpf_prog * prog,union bpf_iter_link_info * linfo,struct bpf_iter_aux_info * aux)1586 static int sock_map_iter_attach_target(struct bpf_prog *prog,
1587 union bpf_iter_link_info *linfo,
1588 struct bpf_iter_aux_info *aux)
1589 {
1590 struct bpf_map *map;
1591 int err = -EINVAL;
1592
1593 if (!linfo->map.map_fd)
1594 return -EBADF;
1595
1596 map = bpf_map_get_with_uref(linfo->map.map_fd);
1597 if (IS_ERR(map))
1598 return PTR_ERR(map);
1599
1600 if (map->map_type != BPF_MAP_TYPE_SOCKMAP &&
1601 map->map_type != BPF_MAP_TYPE_SOCKHASH)
1602 goto put_map;
1603
1604 if (prog->aux->max_rdonly_access > map->key_size) {
1605 err = -EACCES;
1606 goto put_map;
1607 }
1608
1609 aux->map = map;
1610 return 0;
1611
1612 put_map:
1613 bpf_map_put_with_uref(map);
1614 return err;
1615 }
1616
sock_map_iter_detach_target(struct bpf_iter_aux_info * aux)1617 static void sock_map_iter_detach_target(struct bpf_iter_aux_info *aux)
1618 {
1619 bpf_map_put_with_uref(aux->map);
1620 }
1621
1622 static struct bpf_iter_reg sock_map_iter_reg = {
1623 .target = "sockmap",
1624 .attach_target = sock_map_iter_attach_target,
1625 .detach_target = sock_map_iter_detach_target,
1626 .show_fdinfo = bpf_iter_map_show_fdinfo,
1627 .fill_link_info = bpf_iter_map_fill_link_info,
1628 .ctx_arg_info_size = 2,
1629 .ctx_arg_info = {
1630 { offsetof(struct bpf_iter__sockmap, key),
1631 PTR_TO_BUF | PTR_MAYBE_NULL | MEM_RDONLY },
1632 { offsetof(struct bpf_iter__sockmap, sk),
1633 PTR_TO_BTF_ID_OR_NULL },
1634 },
1635 };
1636
bpf_sockmap_iter_init(void)1637 static int __init bpf_sockmap_iter_init(void)
1638 {
1639 sock_map_iter_reg.ctx_arg_info[1].btf_id =
1640 btf_sock_ids[BTF_SOCK_TYPE_SOCK];
1641 return bpf_iter_reg_target(&sock_map_iter_reg);
1642 }
1643 late_initcall(bpf_sockmap_iter_init);
1644