1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 Facebook */
3 #include <linux/rculist.h>
4 #include <linux/list.h>
5 #include <linux/hash.h>
6 #include <linux/types.h>
7 #include <linux/spinlock.h>
8 #include <linux/bpf.h>
9 #include <linux/btf_ids.h>
10 #include <linux/bpf_local_storage.h>
11 #include <net/bpf_sk_storage.h>
12 #include <net/sock.h>
13 #include <uapi/linux/sock_diag.h>
14 #include <uapi/linux/btf.h>
15
16 DEFINE_BPF_STORAGE_CACHE(sk_cache);
17
18 static struct bpf_local_storage_data *
sk_storage_lookup(struct sock * sk,struct bpf_map * map,bool cacheit_lockit)19 sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit)
20 {
21 struct bpf_local_storage *sk_storage;
22 struct bpf_local_storage_map *smap;
23
24 sk_storage = rcu_dereference(sk->sk_bpf_storage);
25 if (!sk_storage)
26 return NULL;
27
28 smap = (struct bpf_local_storage_map *)map;
29 return bpf_local_storage_lookup(sk_storage, smap, cacheit_lockit);
30 }
31
sk_storage_delete(struct sock * sk,struct bpf_map * map)32 static int sk_storage_delete(struct sock *sk, struct bpf_map *map)
33 {
34 struct bpf_local_storage_data *sdata;
35
36 sdata = sk_storage_lookup(sk, map, false);
37 if (!sdata)
38 return -ENOENT;
39
40 bpf_selem_unlink(SELEM(sdata));
41
42 return 0;
43 }
44
45 /* Called by __sk_destruct() & bpf_sk_storage_clone() */
bpf_sk_storage_free(struct sock * sk)46 void bpf_sk_storage_free(struct sock *sk)
47 {
48 struct bpf_local_storage_elem *selem;
49 struct bpf_local_storage *sk_storage;
50 bool free_sk_storage = false;
51 struct hlist_node *n;
52
53 rcu_read_lock();
54 sk_storage = rcu_dereference(sk->sk_bpf_storage);
55 if (!sk_storage) {
56 rcu_read_unlock();
57 return;
58 }
59
60 /* Netiher the bpf_prog nor the bpf-map's syscall
61 * could be modifying the sk_storage->list now.
62 * Thus, no elem can be added-to or deleted-from the
63 * sk_storage->list by the bpf_prog or by the bpf-map's syscall.
64 *
65 * It is racing with bpf_local_storage_map_free() alone
66 * when unlinking elem from the sk_storage->list and
67 * the map's bucket->list.
68 */
69 raw_spin_lock_bh(&sk_storage->lock);
70 hlist_for_each_entry_safe(selem, n, &sk_storage->list, snode) {
71 /* Always unlink from map before unlinking from
72 * sk_storage.
73 */
74 bpf_selem_unlink_map(selem);
75 free_sk_storage = bpf_selem_unlink_storage_nolock(sk_storage,
76 selem, true);
77 }
78 raw_spin_unlock_bh(&sk_storage->lock);
79 rcu_read_unlock();
80
81 if (free_sk_storage)
82 kfree_rcu(sk_storage, rcu);
83 }
84
sk_storage_map_free(struct bpf_map * map)85 static void sk_storage_map_free(struct bpf_map *map)
86 {
87 struct bpf_local_storage_map *smap;
88
89 smap = (struct bpf_local_storage_map *)map;
90 bpf_local_storage_cache_idx_free(&sk_cache, smap->cache_idx);
91 bpf_local_storage_map_free(smap);
92 }
93
sk_storage_map_alloc(union bpf_attr * attr)94 static struct bpf_map *sk_storage_map_alloc(union bpf_attr *attr)
95 {
96 struct bpf_local_storage_map *smap;
97
98 smap = bpf_local_storage_map_alloc(attr);
99 if (IS_ERR(smap))
100 return ERR_CAST(smap);
101
102 smap->cache_idx = bpf_local_storage_cache_idx_get(&sk_cache);
103 return &smap->map;
104 }
105
notsupp_get_next_key(struct bpf_map * map,void * key,void * next_key)106 static int notsupp_get_next_key(struct bpf_map *map, void *key,
107 void *next_key)
108 {
109 return -ENOTSUPP;
110 }
111
bpf_fd_sk_storage_lookup_elem(struct bpf_map * map,void * key)112 static void *bpf_fd_sk_storage_lookup_elem(struct bpf_map *map, void *key)
113 {
114 struct bpf_local_storage_data *sdata;
115 struct socket *sock;
116 int fd, err;
117
118 fd = *(int *)key;
119 sock = sockfd_lookup(fd, &err);
120 if (sock) {
121 sdata = sk_storage_lookup(sock->sk, map, true);
122 sockfd_put(sock);
123 return sdata ? sdata->data : NULL;
124 }
125
126 return ERR_PTR(err);
127 }
128
bpf_fd_sk_storage_update_elem(struct bpf_map * map,void * key,void * value,u64 map_flags)129 static int bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key,
130 void *value, u64 map_flags)
131 {
132 struct bpf_local_storage_data *sdata;
133 struct socket *sock;
134 int fd, err;
135
136 fd = *(int *)key;
137 sock = sockfd_lookup(fd, &err);
138 if (sock) {
139 sdata = bpf_local_storage_update(
140 sock->sk, (struct bpf_local_storage_map *)map, value,
141 map_flags);
142 sockfd_put(sock);
143 return PTR_ERR_OR_ZERO(sdata);
144 }
145
146 return err;
147 }
148
bpf_fd_sk_storage_delete_elem(struct bpf_map * map,void * key)149 static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key)
150 {
151 struct socket *sock;
152 int fd, err;
153
154 fd = *(int *)key;
155 sock = sockfd_lookup(fd, &err);
156 if (sock) {
157 err = sk_storage_delete(sock->sk, map);
158 sockfd_put(sock);
159 return err;
160 }
161
162 return err;
163 }
164
165 static struct bpf_local_storage_elem *
bpf_sk_storage_clone_elem(struct sock * newsk,struct bpf_local_storage_map * smap,struct bpf_local_storage_elem * selem)166 bpf_sk_storage_clone_elem(struct sock *newsk,
167 struct bpf_local_storage_map *smap,
168 struct bpf_local_storage_elem *selem)
169 {
170 struct bpf_local_storage_elem *copy_selem;
171
172 copy_selem = bpf_selem_alloc(smap, newsk, NULL, true);
173 if (!copy_selem)
174 return NULL;
175
176 if (map_value_has_spin_lock(&smap->map))
177 copy_map_value_locked(&smap->map, SDATA(copy_selem)->data,
178 SDATA(selem)->data, true);
179 else
180 copy_map_value(&smap->map, SDATA(copy_selem)->data,
181 SDATA(selem)->data);
182
183 return copy_selem;
184 }
185
bpf_sk_storage_clone(const struct sock * sk,struct sock * newsk)186 int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
187 {
188 struct bpf_local_storage *new_sk_storage = NULL;
189 struct bpf_local_storage *sk_storage;
190 struct bpf_local_storage_elem *selem;
191 int ret = 0;
192
193 RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
194
195 rcu_read_lock();
196 sk_storage = rcu_dereference(sk->sk_bpf_storage);
197
198 if (!sk_storage || hlist_empty(&sk_storage->list))
199 goto out;
200
201 hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
202 struct bpf_local_storage_elem *copy_selem;
203 struct bpf_local_storage_map *smap;
204 struct bpf_map *map;
205
206 smap = rcu_dereference(SDATA(selem)->smap);
207 if (!(smap->map.map_flags & BPF_F_CLONE))
208 continue;
209
210 /* Note that for lockless listeners adding new element
211 * here can race with cleanup in bpf_local_storage_map_free.
212 * Try to grab map refcnt to make sure that it's still
213 * alive and prevent concurrent removal.
214 */
215 map = bpf_map_inc_not_zero(&smap->map);
216 if (IS_ERR(map))
217 continue;
218
219 copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem);
220 if (!copy_selem) {
221 ret = -ENOMEM;
222 bpf_map_put(map);
223 goto out;
224 }
225
226 if (new_sk_storage) {
227 bpf_selem_link_map(smap, copy_selem);
228 bpf_selem_link_storage_nolock(new_sk_storage, copy_selem);
229 } else {
230 ret = bpf_local_storage_alloc(newsk, smap, copy_selem);
231 if (ret) {
232 kfree(copy_selem);
233 atomic_sub(smap->elem_size,
234 &newsk->sk_omem_alloc);
235 bpf_map_put(map);
236 goto out;
237 }
238
239 new_sk_storage =
240 rcu_dereference(copy_selem->local_storage);
241 }
242 bpf_map_put(map);
243 }
244
245 out:
246 rcu_read_unlock();
247
248 /* In case of an error, don't free anything explicitly here, the
249 * caller is responsible to call bpf_sk_storage_free.
250 */
251
252 return ret;
253 }
254
BPF_CALL_4(bpf_sk_storage_get,struct bpf_map *,map,struct sock *,sk,void *,value,u64,flags)255 BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
256 void *, value, u64, flags)
257 {
258 struct bpf_local_storage_data *sdata;
259
260 if (!sk || !sk_fullsock(sk) || flags > BPF_SK_STORAGE_GET_F_CREATE)
261 return (unsigned long)NULL;
262
263 sdata = sk_storage_lookup(sk, map, true);
264 if (sdata)
265 return (unsigned long)sdata->data;
266
267 if (flags == BPF_SK_STORAGE_GET_F_CREATE &&
268 /* Cannot add new elem to a going away sk.
269 * Otherwise, the new elem may become a leak
270 * (and also other memory issues during map
271 * destruction).
272 */
273 refcount_inc_not_zero(&sk->sk_refcnt)) {
274 sdata = bpf_local_storage_update(
275 sk, (struct bpf_local_storage_map *)map, value,
276 BPF_NOEXIST);
277 /* sk must be a fullsock (guaranteed by verifier),
278 * so sock_gen_put() is unnecessary.
279 */
280 sock_put(sk);
281 return IS_ERR(sdata) ?
282 (unsigned long)NULL : (unsigned long)sdata->data;
283 }
284
285 return (unsigned long)NULL;
286 }
287
BPF_CALL_2(bpf_sk_storage_delete,struct bpf_map *,map,struct sock *,sk)288 BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk)
289 {
290 if (!sk || !sk_fullsock(sk))
291 return -EINVAL;
292
293 if (refcount_inc_not_zero(&sk->sk_refcnt)) {
294 int err;
295
296 err = sk_storage_delete(sk, map);
297 sock_put(sk);
298 return err;
299 }
300
301 return -ENOENT;
302 }
303
sk_storage_charge(struct bpf_local_storage_map * smap,void * owner,u32 size)304 static int sk_storage_charge(struct bpf_local_storage_map *smap,
305 void *owner, u32 size)
306 {
307 int optmem_max = READ_ONCE(sysctl_optmem_max);
308 struct sock *sk = (struct sock *)owner;
309
310 /* same check as in sock_kmalloc() */
311 if (size <= optmem_max &&
312 atomic_read(&sk->sk_omem_alloc) + size < optmem_max) {
313 atomic_add(size, &sk->sk_omem_alloc);
314 return 0;
315 }
316
317 return -ENOMEM;
318 }
319
sk_storage_uncharge(struct bpf_local_storage_map * smap,void * owner,u32 size)320 static void sk_storage_uncharge(struct bpf_local_storage_map *smap,
321 void *owner, u32 size)
322 {
323 struct sock *sk = owner;
324
325 atomic_sub(size, &sk->sk_omem_alloc);
326 }
327
328 static struct bpf_local_storage __rcu **
sk_storage_ptr(void * owner)329 sk_storage_ptr(void *owner)
330 {
331 struct sock *sk = owner;
332
333 return &sk->sk_bpf_storage;
334 }
335
336 static int sk_storage_map_btf_id;
337 const struct bpf_map_ops sk_storage_map_ops = {
338 .map_meta_equal = bpf_map_meta_equal,
339 .map_alloc_check = bpf_local_storage_map_alloc_check,
340 .map_alloc = sk_storage_map_alloc,
341 .map_free = sk_storage_map_free,
342 .map_get_next_key = notsupp_get_next_key,
343 .map_lookup_elem = bpf_fd_sk_storage_lookup_elem,
344 .map_update_elem = bpf_fd_sk_storage_update_elem,
345 .map_delete_elem = bpf_fd_sk_storage_delete_elem,
346 .map_check_btf = bpf_local_storage_map_check_btf,
347 .map_btf_name = "bpf_local_storage_map",
348 .map_btf_id = &sk_storage_map_btf_id,
349 .map_local_storage_charge = sk_storage_charge,
350 .map_local_storage_uncharge = sk_storage_uncharge,
351 .map_owner_storage_ptr = sk_storage_ptr,
352 };
353
354 const struct bpf_func_proto bpf_sk_storage_get_proto = {
355 .func = bpf_sk_storage_get,
356 .gpl_only = false,
357 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
358 .arg1_type = ARG_CONST_MAP_PTR,
359 .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
360 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
361 .arg4_type = ARG_ANYTHING,
362 };
363
364 const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto = {
365 .func = bpf_sk_storage_get,
366 .gpl_only = false,
367 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
368 .arg1_type = ARG_CONST_MAP_PTR,
369 .arg2_type = ARG_PTR_TO_CTX, /* context is 'struct sock' */
370 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
371 .arg4_type = ARG_ANYTHING,
372 };
373
374 const struct bpf_func_proto bpf_sk_storage_delete_proto = {
375 .func = bpf_sk_storage_delete,
376 .gpl_only = false,
377 .ret_type = RET_INTEGER,
378 .arg1_type = ARG_CONST_MAP_PTR,
379 .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
380 };
381
382 struct bpf_sk_storage_diag {
383 u32 nr_maps;
384 struct bpf_map *maps[];
385 };
386
387 /* The reply will be like:
388 * INET_DIAG_BPF_SK_STORAGES (nla_nest)
389 * SK_DIAG_BPF_STORAGE (nla_nest)
390 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
391 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
392 * SK_DIAG_BPF_STORAGE (nla_nest)
393 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
394 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
395 * ....
396 */
nla_value_size(u32 value_size)397 static int nla_value_size(u32 value_size)
398 {
399 /* SK_DIAG_BPF_STORAGE (nla_nest)
400 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
401 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
402 */
403 return nla_total_size(0) + nla_total_size(sizeof(u32)) +
404 nla_total_size_64bit(value_size);
405 }
406
bpf_sk_storage_diag_free(struct bpf_sk_storage_diag * diag)407 void bpf_sk_storage_diag_free(struct bpf_sk_storage_diag *diag)
408 {
409 u32 i;
410
411 if (!diag)
412 return;
413
414 for (i = 0; i < diag->nr_maps; i++)
415 bpf_map_put(diag->maps[i]);
416
417 kfree(diag);
418 }
419 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_free);
420
diag_check_dup(const struct bpf_sk_storage_diag * diag,const struct bpf_map * map)421 static bool diag_check_dup(const struct bpf_sk_storage_diag *diag,
422 const struct bpf_map *map)
423 {
424 u32 i;
425
426 for (i = 0; i < diag->nr_maps; i++) {
427 if (diag->maps[i] == map)
428 return true;
429 }
430
431 return false;
432 }
433
434 struct bpf_sk_storage_diag *
bpf_sk_storage_diag_alloc(const struct nlattr * nla_stgs)435 bpf_sk_storage_diag_alloc(const struct nlattr *nla_stgs)
436 {
437 struct bpf_sk_storage_diag *diag;
438 struct nlattr *nla;
439 u32 nr_maps = 0;
440 int rem, err;
441
442 /* bpf_local_storage_map is currently limited to CAP_SYS_ADMIN as
443 * the map_alloc_check() side also does.
444 */
445 if (!bpf_capable())
446 return ERR_PTR(-EPERM);
447
448 nla_for_each_nested(nla, nla_stgs, rem) {
449 if (nla_type(nla) == SK_DIAG_BPF_STORAGE_REQ_MAP_FD) {
450 if (nla_len(nla) != sizeof(u32))
451 return ERR_PTR(-EINVAL);
452 nr_maps++;
453 }
454 }
455
456 diag = kzalloc(sizeof(*diag) + sizeof(diag->maps[0]) * nr_maps,
457 GFP_KERNEL);
458 if (!diag)
459 return ERR_PTR(-ENOMEM);
460
461 nla_for_each_nested(nla, nla_stgs, rem) {
462 struct bpf_map *map;
463 int map_fd;
464
465 if (nla_type(nla) != SK_DIAG_BPF_STORAGE_REQ_MAP_FD)
466 continue;
467
468 map_fd = nla_get_u32(nla);
469 map = bpf_map_get(map_fd);
470 if (IS_ERR(map)) {
471 err = PTR_ERR(map);
472 goto err_free;
473 }
474 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) {
475 bpf_map_put(map);
476 err = -EINVAL;
477 goto err_free;
478 }
479 if (diag_check_dup(diag, map)) {
480 bpf_map_put(map);
481 err = -EEXIST;
482 goto err_free;
483 }
484 diag->maps[diag->nr_maps++] = map;
485 }
486
487 return diag;
488
489 err_free:
490 bpf_sk_storage_diag_free(diag);
491 return ERR_PTR(err);
492 }
493 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_alloc);
494
diag_get(struct bpf_local_storage_data * sdata,struct sk_buff * skb)495 static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb)
496 {
497 struct nlattr *nla_stg, *nla_value;
498 struct bpf_local_storage_map *smap;
499
500 /* It cannot exceed max nlattr's payload */
501 BUILD_BUG_ON(U16_MAX - NLA_HDRLEN < BPF_LOCAL_STORAGE_MAX_VALUE_SIZE);
502
503 nla_stg = nla_nest_start(skb, SK_DIAG_BPF_STORAGE);
504 if (!nla_stg)
505 return -EMSGSIZE;
506
507 smap = rcu_dereference(sdata->smap);
508 if (nla_put_u32(skb, SK_DIAG_BPF_STORAGE_MAP_ID, smap->map.id))
509 goto errout;
510
511 nla_value = nla_reserve_64bit(skb, SK_DIAG_BPF_STORAGE_MAP_VALUE,
512 smap->map.value_size,
513 SK_DIAG_BPF_STORAGE_PAD);
514 if (!nla_value)
515 goto errout;
516
517 if (map_value_has_spin_lock(&smap->map))
518 copy_map_value_locked(&smap->map, nla_data(nla_value),
519 sdata->data, true);
520 else
521 copy_map_value(&smap->map, nla_data(nla_value), sdata->data);
522
523 nla_nest_end(skb, nla_stg);
524 return 0;
525
526 errout:
527 nla_nest_cancel(skb, nla_stg);
528 return -EMSGSIZE;
529 }
530
bpf_sk_storage_diag_put_all(struct sock * sk,struct sk_buff * skb,int stg_array_type,unsigned int * res_diag_size)531 static int bpf_sk_storage_diag_put_all(struct sock *sk, struct sk_buff *skb,
532 int stg_array_type,
533 unsigned int *res_diag_size)
534 {
535 /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */
536 unsigned int diag_size = nla_total_size(0);
537 struct bpf_local_storage *sk_storage;
538 struct bpf_local_storage_elem *selem;
539 struct bpf_local_storage_map *smap;
540 struct nlattr *nla_stgs;
541 unsigned int saved_len;
542 int err = 0;
543
544 rcu_read_lock();
545
546 sk_storage = rcu_dereference(sk->sk_bpf_storage);
547 if (!sk_storage || hlist_empty(&sk_storage->list)) {
548 rcu_read_unlock();
549 return 0;
550 }
551
552 nla_stgs = nla_nest_start(skb, stg_array_type);
553 if (!nla_stgs)
554 /* Continue to learn diag_size */
555 err = -EMSGSIZE;
556
557 saved_len = skb->len;
558 hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
559 smap = rcu_dereference(SDATA(selem)->smap);
560 diag_size += nla_value_size(smap->map.value_size);
561
562 if (nla_stgs && diag_get(SDATA(selem), skb))
563 /* Continue to learn diag_size */
564 err = -EMSGSIZE;
565 }
566
567 rcu_read_unlock();
568
569 if (nla_stgs) {
570 if (saved_len == skb->len)
571 nla_nest_cancel(skb, nla_stgs);
572 else
573 nla_nest_end(skb, nla_stgs);
574 }
575
576 if (diag_size == nla_total_size(0)) {
577 *res_diag_size = 0;
578 return 0;
579 }
580
581 *res_diag_size = diag_size;
582 return err;
583 }
584
bpf_sk_storage_diag_put(struct bpf_sk_storage_diag * diag,struct sock * sk,struct sk_buff * skb,int stg_array_type,unsigned int * res_diag_size)585 int bpf_sk_storage_diag_put(struct bpf_sk_storage_diag *diag,
586 struct sock *sk, struct sk_buff *skb,
587 int stg_array_type,
588 unsigned int *res_diag_size)
589 {
590 /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */
591 unsigned int diag_size = nla_total_size(0);
592 struct bpf_local_storage *sk_storage;
593 struct bpf_local_storage_data *sdata;
594 struct nlattr *nla_stgs;
595 unsigned int saved_len;
596 int err = 0;
597 u32 i;
598
599 *res_diag_size = 0;
600
601 /* No map has been specified. Dump all. */
602 if (!diag->nr_maps)
603 return bpf_sk_storage_diag_put_all(sk, skb, stg_array_type,
604 res_diag_size);
605
606 rcu_read_lock();
607 sk_storage = rcu_dereference(sk->sk_bpf_storage);
608 if (!sk_storage || hlist_empty(&sk_storage->list)) {
609 rcu_read_unlock();
610 return 0;
611 }
612
613 nla_stgs = nla_nest_start(skb, stg_array_type);
614 if (!nla_stgs)
615 /* Continue to learn diag_size */
616 err = -EMSGSIZE;
617
618 saved_len = skb->len;
619 for (i = 0; i < diag->nr_maps; i++) {
620 sdata = bpf_local_storage_lookup(sk_storage,
621 (struct bpf_local_storage_map *)diag->maps[i],
622 false);
623
624 if (!sdata)
625 continue;
626
627 diag_size += nla_value_size(diag->maps[i]->value_size);
628
629 if (nla_stgs && diag_get(sdata, skb))
630 /* Continue to learn diag_size */
631 err = -EMSGSIZE;
632 }
633 rcu_read_unlock();
634
635 if (nla_stgs) {
636 if (saved_len == skb->len)
637 nla_nest_cancel(skb, nla_stgs);
638 else
639 nla_nest_end(skb, nla_stgs);
640 }
641
642 if (diag_size == nla_total_size(0)) {
643 *res_diag_size = 0;
644 return 0;
645 }
646
647 *res_diag_size = diag_size;
648 return err;
649 }
650 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_put);
651
652 struct bpf_iter_seq_sk_storage_map_info {
653 struct bpf_map *map;
654 unsigned int bucket_id;
655 unsigned skip_elems;
656 };
657
658 static struct bpf_local_storage_elem *
bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info * info,struct bpf_local_storage_elem * prev_selem)659 bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info,
660 struct bpf_local_storage_elem *prev_selem)
661 __acquires(RCU) __releases(RCU)
662 {
663 struct bpf_local_storage *sk_storage;
664 struct bpf_local_storage_elem *selem;
665 u32 skip_elems = info->skip_elems;
666 struct bpf_local_storage_map *smap;
667 u32 bucket_id = info->bucket_id;
668 u32 i, count, n_buckets;
669 struct bpf_local_storage_map_bucket *b;
670
671 smap = (struct bpf_local_storage_map *)info->map;
672 n_buckets = 1U << smap->bucket_log;
673 if (bucket_id >= n_buckets)
674 return NULL;
675
676 /* try to find next selem in the same bucket */
677 selem = prev_selem;
678 count = 0;
679 while (selem) {
680 selem = hlist_entry_safe(rcu_dereference(hlist_next_rcu(&selem->map_node)),
681 struct bpf_local_storage_elem, map_node);
682 if (!selem) {
683 /* not found, unlock and go to the next bucket */
684 b = &smap->buckets[bucket_id++];
685 rcu_read_unlock();
686 skip_elems = 0;
687 break;
688 }
689 sk_storage = rcu_dereference(selem->local_storage);
690 if (sk_storage) {
691 info->skip_elems = skip_elems + count;
692 return selem;
693 }
694 count++;
695 }
696
697 for (i = bucket_id; i < (1U << smap->bucket_log); i++) {
698 b = &smap->buckets[i];
699 rcu_read_lock();
700 count = 0;
701 hlist_for_each_entry_rcu(selem, &b->list, map_node) {
702 sk_storage = rcu_dereference(selem->local_storage);
703 if (sk_storage && count >= skip_elems) {
704 info->bucket_id = i;
705 info->skip_elems = count;
706 return selem;
707 }
708 count++;
709 }
710 rcu_read_unlock();
711 skip_elems = 0;
712 }
713
714 info->bucket_id = i;
715 info->skip_elems = 0;
716 return NULL;
717 }
718
bpf_sk_storage_map_seq_start(struct seq_file * seq,loff_t * pos)719 static void *bpf_sk_storage_map_seq_start(struct seq_file *seq, loff_t *pos)
720 {
721 struct bpf_local_storage_elem *selem;
722
723 selem = bpf_sk_storage_map_seq_find_next(seq->private, NULL);
724 if (!selem)
725 return NULL;
726
727 if (*pos == 0)
728 ++*pos;
729 return selem;
730 }
731
bpf_sk_storage_map_seq_next(struct seq_file * seq,void * v,loff_t * pos)732 static void *bpf_sk_storage_map_seq_next(struct seq_file *seq, void *v,
733 loff_t *pos)
734 {
735 struct bpf_iter_seq_sk_storage_map_info *info = seq->private;
736
737 ++*pos;
738 ++info->skip_elems;
739 return bpf_sk_storage_map_seq_find_next(seq->private, v);
740 }
741
742 struct bpf_iter__bpf_sk_storage_map {
743 __bpf_md_ptr(struct bpf_iter_meta *, meta);
744 __bpf_md_ptr(struct bpf_map *, map);
745 __bpf_md_ptr(struct sock *, sk);
746 __bpf_md_ptr(void *, value);
747 };
748
DEFINE_BPF_ITER_FUNC(bpf_sk_storage_map,struct bpf_iter_meta * meta,struct bpf_map * map,struct sock * sk,void * value)749 DEFINE_BPF_ITER_FUNC(bpf_sk_storage_map, struct bpf_iter_meta *meta,
750 struct bpf_map *map, struct sock *sk,
751 void *value)
752
753 static int __bpf_sk_storage_map_seq_show(struct seq_file *seq,
754 struct bpf_local_storage_elem *selem)
755 {
756 struct bpf_iter_seq_sk_storage_map_info *info = seq->private;
757 struct bpf_iter__bpf_sk_storage_map ctx = {};
758 struct bpf_local_storage *sk_storage;
759 struct bpf_iter_meta meta;
760 struct bpf_prog *prog;
761 int ret = 0;
762
763 meta.seq = seq;
764 prog = bpf_iter_get_info(&meta, selem == NULL);
765 if (prog) {
766 ctx.meta = &meta;
767 ctx.map = info->map;
768 if (selem) {
769 sk_storage = rcu_dereference(selem->local_storage);
770 ctx.sk = sk_storage->owner;
771 ctx.value = SDATA(selem)->data;
772 }
773 ret = bpf_iter_run_prog(prog, &ctx);
774 }
775
776 return ret;
777 }
778
bpf_sk_storage_map_seq_show(struct seq_file * seq,void * v)779 static int bpf_sk_storage_map_seq_show(struct seq_file *seq, void *v)
780 {
781 return __bpf_sk_storage_map_seq_show(seq, v);
782 }
783
bpf_sk_storage_map_seq_stop(struct seq_file * seq,void * v)784 static void bpf_sk_storage_map_seq_stop(struct seq_file *seq, void *v)
785 __releases(RCU)
786 {
787 if (!v)
788 (void)__bpf_sk_storage_map_seq_show(seq, v);
789 else
790 rcu_read_unlock();
791 }
792
bpf_iter_init_sk_storage_map(void * priv_data,struct bpf_iter_aux_info * aux)793 static int bpf_iter_init_sk_storage_map(void *priv_data,
794 struct bpf_iter_aux_info *aux)
795 {
796 struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data;
797
798 bpf_map_inc_with_uref(aux->map);
799 seq_info->map = aux->map;
800 return 0;
801 }
802
bpf_iter_fini_sk_storage_map(void * priv_data)803 static void bpf_iter_fini_sk_storage_map(void *priv_data)
804 {
805 struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data;
806
807 bpf_map_put_with_uref(seq_info->map);
808 }
809
bpf_iter_attach_map(struct bpf_prog * prog,union bpf_iter_link_info * linfo,struct bpf_iter_aux_info * aux)810 static int bpf_iter_attach_map(struct bpf_prog *prog,
811 union bpf_iter_link_info *linfo,
812 struct bpf_iter_aux_info *aux)
813 {
814 struct bpf_map *map;
815 int err = -EINVAL;
816
817 if (!linfo->map.map_fd)
818 return -EBADF;
819
820 map = bpf_map_get_with_uref(linfo->map.map_fd);
821 if (IS_ERR(map))
822 return PTR_ERR(map);
823
824 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
825 goto put_map;
826
827 if (prog->aux->max_rdwr_access > map->value_size) {
828 err = -EACCES;
829 goto put_map;
830 }
831
832 aux->map = map;
833 return 0;
834
835 put_map:
836 bpf_map_put_with_uref(map);
837 return err;
838 }
839
bpf_iter_detach_map(struct bpf_iter_aux_info * aux)840 static void bpf_iter_detach_map(struct bpf_iter_aux_info *aux)
841 {
842 bpf_map_put_with_uref(aux->map);
843 }
844
845 static const struct seq_operations bpf_sk_storage_map_seq_ops = {
846 .start = bpf_sk_storage_map_seq_start,
847 .next = bpf_sk_storage_map_seq_next,
848 .stop = bpf_sk_storage_map_seq_stop,
849 .show = bpf_sk_storage_map_seq_show,
850 };
851
852 static const struct bpf_iter_seq_info iter_seq_info = {
853 .seq_ops = &bpf_sk_storage_map_seq_ops,
854 .init_seq_private = bpf_iter_init_sk_storage_map,
855 .fini_seq_private = bpf_iter_fini_sk_storage_map,
856 .seq_priv_size = sizeof(struct bpf_iter_seq_sk_storage_map_info),
857 };
858
859 static struct bpf_iter_reg bpf_sk_storage_map_reg_info = {
860 .target = "bpf_sk_storage_map",
861 .attach_target = bpf_iter_attach_map,
862 .detach_target = bpf_iter_detach_map,
863 .show_fdinfo = bpf_iter_map_show_fdinfo,
864 .fill_link_info = bpf_iter_map_fill_link_info,
865 .ctx_arg_info_size = 2,
866 .ctx_arg_info = {
867 { offsetof(struct bpf_iter__bpf_sk_storage_map, sk),
868 PTR_TO_BTF_ID_OR_NULL },
869 { offsetof(struct bpf_iter__bpf_sk_storage_map, value),
870 PTR_TO_RDWR_BUF_OR_NULL },
871 },
872 .seq_info = &iter_seq_info,
873 };
874
bpf_sk_storage_map_iter_init(void)875 static int __init bpf_sk_storage_map_iter_init(void)
876 {
877 bpf_sk_storage_map_reg_info.ctx_arg_info[0].btf_id =
878 btf_sock_ids[BTF_SOCK_TYPE_SOCK];
879 return bpf_iter_reg_target(&bpf_sk_storage_map_reg_info);
880 }
881 late_initcall(bpf_sk_storage_map_iter_init);
882