1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
3
4 #include <linux/skmsg.h>
5 #include <linux/filter.h>
6 #include <linux/bpf.h>
7 #include <linux/init.h>
8 #include <linux/wait.h>
9 #include <linux/util_macros.h>
10
11 #include <net/inet_common.h>
12 #include <net/tls.h>
13
__tcp_bpf_recvmsg(struct sock * sk,struct sk_psock * psock,struct msghdr * msg,int len,int flags)14 int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,
15 struct msghdr *msg, int len, int flags)
16 {
17 struct iov_iter *iter = &msg->msg_iter;
18 int peek = flags & MSG_PEEK;
19 struct sk_msg *msg_rx;
20 int i, copied = 0;
21
22 msg_rx = list_first_entry_or_null(&psock->ingress_msg,
23 struct sk_msg, list);
24
25 while (copied != len) {
26 struct scatterlist *sge;
27
28 if (unlikely(!msg_rx))
29 break;
30
31 i = msg_rx->sg.start;
32 do {
33 struct page *page;
34 int copy;
35
36 sge = sk_msg_elem(msg_rx, i);
37 copy = sge->length;
38 page = sg_page(sge);
39 if (copied + copy > len)
40 copy = len - copied;
41 copy = copy_page_to_iter(page, sge->offset, copy, iter);
42 if (!copy)
43 return copied ? copied : -EFAULT;
44
45 copied += copy;
46 if (likely(!peek)) {
47 sge->offset += copy;
48 sge->length -= copy;
49 if (!msg_rx->skb)
50 sk_mem_uncharge(sk, copy);
51 msg_rx->sg.size -= copy;
52
53 if (!sge->length) {
54 sk_msg_iter_var_next(i);
55 if (!msg_rx->skb)
56 put_page(page);
57 }
58 } else {
59 /* Lets not optimize peek case if copy_page_to_iter
60 * didn't copy the entire length lets just break.
61 */
62 if (copy != sge->length)
63 return copied;
64 sk_msg_iter_var_next(i);
65 }
66
67 if (copied == len)
68 break;
69 } while (i != msg_rx->sg.end);
70
71 if (unlikely(peek)) {
72 if (msg_rx == list_last_entry(&psock->ingress_msg,
73 struct sk_msg, list))
74 break;
75 msg_rx = list_next_entry(msg_rx, list);
76 continue;
77 }
78
79 msg_rx->sg.start = i;
80 if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) {
81 list_del(&msg_rx->list);
82 if (msg_rx->skb)
83 consume_skb(msg_rx->skb);
84 kfree(msg_rx);
85 }
86 msg_rx = list_first_entry_or_null(&psock->ingress_msg,
87 struct sk_msg, list);
88 }
89
90 return copied;
91 }
92 EXPORT_SYMBOL_GPL(__tcp_bpf_recvmsg);
93
bpf_tcp_ingress(struct sock * sk,struct sk_psock * psock,struct sk_msg * msg,u32 apply_bytes,int flags)94 static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock,
95 struct sk_msg *msg, u32 apply_bytes, int flags)
96 {
97 bool apply = apply_bytes;
98 struct scatterlist *sge;
99 u32 size, copied = 0;
100 struct sk_msg *tmp;
101 int i, ret = 0;
102
103 tmp = kzalloc(sizeof(*tmp), __GFP_NOWARN | GFP_KERNEL);
104 if (unlikely(!tmp))
105 return -ENOMEM;
106
107 lock_sock(sk);
108 tmp->sg.start = msg->sg.start;
109 i = msg->sg.start;
110 do {
111 sge = sk_msg_elem(msg, i);
112 size = (apply && apply_bytes < sge->length) ?
113 apply_bytes : sge->length;
114 if (!sk_wmem_schedule(sk, size)) {
115 if (!copied)
116 ret = -ENOMEM;
117 break;
118 }
119
120 sk_mem_charge(sk, size);
121 sk_msg_xfer(tmp, msg, i, size);
122 copied += size;
123 if (sge->length)
124 get_page(sk_msg_page(tmp, i));
125 sk_msg_iter_var_next(i);
126 tmp->sg.end = i;
127 if (apply) {
128 apply_bytes -= size;
129 if (!apply_bytes) {
130 if (sge->length)
131 sk_msg_iter_var_prev(i);
132 break;
133 }
134 }
135 } while (i != msg->sg.end);
136
137 if (!ret) {
138 msg->sg.start = i;
139 sk_psock_queue_msg(psock, tmp);
140 sk_psock_data_ready(sk, psock);
141 } else {
142 sk_msg_free(sk, tmp);
143 kfree(tmp);
144 }
145
146 release_sock(sk);
147 return ret;
148 }
149
tcp_bpf_push(struct sock * sk,struct sk_msg * msg,u32 apply_bytes,int flags,bool uncharge)150 static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes,
151 int flags, bool uncharge)
152 {
153 bool apply = apply_bytes;
154 struct scatterlist *sge;
155 struct page *page;
156 int size, ret = 0;
157 u32 off;
158
159 while (1) {
160 bool has_tx_ulp;
161
162 sge = sk_msg_elem(msg, msg->sg.start);
163 size = (apply && apply_bytes < sge->length) ?
164 apply_bytes : sge->length;
165 off = sge->offset;
166 page = sg_page(sge);
167
168 tcp_rate_check_app_limited(sk);
169 retry:
170 has_tx_ulp = tls_sw_has_ctx_tx(sk);
171 if (has_tx_ulp) {
172 flags |= MSG_SENDPAGE_NOPOLICY;
173 ret = kernel_sendpage_locked(sk,
174 page, off, size, flags);
175 } else {
176 ret = do_tcp_sendpages(sk, page, off, size, flags);
177 }
178
179 if (ret <= 0)
180 return ret;
181 if (apply)
182 apply_bytes -= ret;
183 msg->sg.size -= ret;
184 sge->offset += ret;
185 sge->length -= ret;
186 if (uncharge)
187 sk_mem_uncharge(sk, ret);
188 if (ret != size) {
189 size -= ret;
190 off += ret;
191 goto retry;
192 }
193 if (!sge->length) {
194 put_page(page);
195 sk_msg_iter_next(msg, start);
196 sg_init_table(sge, 1);
197 if (msg->sg.start == msg->sg.end)
198 break;
199 }
200 if (apply && !apply_bytes)
201 break;
202 }
203
204 return 0;
205 }
206
tcp_bpf_push_locked(struct sock * sk,struct sk_msg * msg,u32 apply_bytes,int flags,bool uncharge)207 static int tcp_bpf_push_locked(struct sock *sk, struct sk_msg *msg,
208 u32 apply_bytes, int flags, bool uncharge)
209 {
210 int ret;
211
212 lock_sock(sk);
213 ret = tcp_bpf_push(sk, msg, apply_bytes, flags, uncharge);
214 release_sock(sk);
215 return ret;
216 }
217
tcp_bpf_sendmsg_redir(struct sock * sk,struct sk_msg * msg,u32 bytes,int flags)218 int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg,
219 u32 bytes, int flags)
220 {
221 bool ingress = sk_msg_to_ingress(msg);
222 struct sk_psock *psock = sk_psock_get(sk);
223 int ret;
224
225 if (unlikely(!psock))
226 return -EPIPE;
227
228 ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) :
229 tcp_bpf_push_locked(sk, msg, bytes, flags, false);
230 sk_psock_put(sk, psock);
231 return ret;
232 }
233 EXPORT_SYMBOL_GPL(tcp_bpf_sendmsg_redir);
234
235 #ifdef CONFIG_BPF_STREAM_PARSER
tcp_bpf_stream_read(const struct sock * sk)236 static bool tcp_bpf_stream_read(const struct sock *sk)
237 {
238 struct sk_psock *psock;
239 bool empty = true;
240
241 rcu_read_lock();
242 psock = sk_psock(sk);
243 if (likely(psock))
244 empty = list_empty(&psock->ingress_msg);
245 rcu_read_unlock();
246 return !empty;
247 }
248
tcp_bpf_wait_data(struct sock * sk,struct sk_psock * psock,int flags,long timeo,int * err)249 static int tcp_bpf_wait_data(struct sock *sk, struct sk_psock *psock,
250 int flags, long timeo, int *err)
251 {
252 DEFINE_WAIT_FUNC(wait, woken_wake_function);
253 int ret = 0;
254
255 if (sk->sk_shutdown & RCV_SHUTDOWN)
256 return 1;
257
258 if (!timeo)
259 return ret;
260
261 add_wait_queue(sk_sleep(sk), &wait);
262 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
263 ret = sk_wait_event(sk, &timeo,
264 !list_empty(&psock->ingress_msg) ||
265 !skb_queue_empty_lockless(&sk->sk_receive_queue), &wait);
266 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
267 remove_wait_queue(sk_sleep(sk), &wait);
268 return ret;
269 }
270
tcp_bpf_recvmsg(struct sock * sk,struct msghdr * msg,size_t len,int nonblock,int flags,int * addr_len)271 static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
272 int nonblock, int flags, int *addr_len)
273 {
274 struct sk_psock *psock;
275 int copied, ret;
276
277 if (unlikely(flags & MSG_ERRQUEUE))
278 return inet_recv_error(sk, msg, len, addr_len);
279
280 psock = sk_psock_get(sk);
281 if (unlikely(!psock))
282 return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
283 if (!skb_queue_empty(&sk->sk_receive_queue) &&
284 sk_psock_queue_empty(psock)) {
285 sk_psock_put(sk, psock);
286 return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
287 }
288 lock_sock(sk);
289 msg_bytes_ready:
290 copied = __tcp_bpf_recvmsg(sk, psock, msg, len, flags);
291 if (!copied) {
292 int data, err = 0;
293 long timeo;
294
295 timeo = sock_rcvtimeo(sk, nonblock);
296 data = tcp_bpf_wait_data(sk, psock, flags, timeo, &err);
297 if (data) {
298 if (!sk_psock_queue_empty(psock))
299 goto msg_bytes_ready;
300 release_sock(sk);
301 sk_psock_put(sk, psock);
302 return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
303 }
304 if (err) {
305 ret = err;
306 goto out;
307 }
308 copied = -EAGAIN;
309 }
310 ret = copied;
311 out:
312 release_sock(sk);
313 sk_psock_put(sk, psock);
314 return ret;
315 }
316
tcp_bpf_send_verdict(struct sock * sk,struct sk_psock * psock,struct sk_msg * msg,int * copied,int flags)317 static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock,
318 struct sk_msg *msg, int *copied, int flags)
319 {
320 bool cork = false, enospc = sk_msg_full(msg);
321 struct sock *sk_redir;
322 u32 tosend, origsize, sent, delta = 0;
323 u32 eval;
324 int ret;
325
326 more_data:
327 if (psock->eval == __SK_NONE) {
328 /* Track delta in msg size to add/subtract it on SK_DROP from
329 * returned to user copied size. This ensures user doesn't
330 * get a positive return code with msg_cut_data and SK_DROP
331 * verdict.
332 */
333 delta = msg->sg.size;
334 psock->eval = sk_psock_msg_verdict(sk, psock, msg);
335 delta -= msg->sg.size;
336 }
337
338 if (msg->cork_bytes &&
339 msg->cork_bytes > msg->sg.size && !enospc) {
340 psock->cork_bytes = msg->cork_bytes - msg->sg.size;
341 if (!psock->cork) {
342 psock->cork = kzalloc(sizeof(*psock->cork),
343 GFP_ATOMIC | __GFP_NOWARN);
344 if (!psock->cork)
345 return -ENOMEM;
346 }
347 memcpy(psock->cork, msg, sizeof(*msg));
348 return 0;
349 }
350
351 tosend = msg->sg.size;
352 if (psock->apply_bytes && psock->apply_bytes < tosend)
353 tosend = psock->apply_bytes;
354 eval = __SK_NONE;
355
356 switch (psock->eval) {
357 case __SK_PASS:
358 ret = tcp_bpf_push(sk, msg, tosend, flags, true);
359 if (unlikely(ret)) {
360 *copied -= sk_msg_free(sk, msg);
361 break;
362 }
363 sk_msg_apply_bytes(psock, tosend);
364 break;
365 case __SK_REDIRECT:
366 sk_redir = psock->sk_redir;
367 sk_msg_apply_bytes(psock, tosend);
368 if (!psock->apply_bytes) {
369 /* Clean up before releasing the sock lock. */
370 eval = psock->eval;
371 psock->eval = __SK_NONE;
372 psock->sk_redir = NULL;
373 }
374 if (psock->cork) {
375 cork = true;
376 psock->cork = NULL;
377 }
378 sk_msg_return(sk, msg, tosend);
379 release_sock(sk);
380
381 origsize = msg->sg.size;
382 ret = tcp_bpf_sendmsg_redir(sk_redir, msg, tosend, flags);
383 sent = origsize - msg->sg.size;
384
385 if (eval == __SK_REDIRECT)
386 sock_put(sk_redir);
387
388 lock_sock(sk);
389 if (unlikely(ret < 0)) {
390 int free = sk_msg_free_nocharge(sk, msg);
391
392 if (!cork)
393 *copied -= free;
394 }
395 if (cork) {
396 sk_msg_free(sk, msg);
397 kfree(msg);
398 msg = NULL;
399 ret = 0;
400 }
401 break;
402 case __SK_DROP:
403 default:
404 sk_msg_free_partial(sk, msg, tosend);
405 sk_msg_apply_bytes(psock, tosend);
406 *copied -= (tosend + delta);
407 return -EACCES;
408 }
409
410 if (likely(!ret)) {
411 if (!psock->apply_bytes) {
412 psock->eval = __SK_NONE;
413 if (psock->sk_redir) {
414 sock_put(psock->sk_redir);
415 psock->sk_redir = NULL;
416 }
417 }
418 if (msg &&
419 msg->sg.data[msg->sg.start].page_link &&
420 msg->sg.data[msg->sg.start].length) {
421 if (eval == __SK_REDIRECT)
422 sk_mem_charge(sk, tosend - sent);
423 goto more_data;
424 }
425 }
426 return ret;
427 }
428
tcp_bpf_sendmsg(struct sock * sk,struct msghdr * msg,size_t size)429 static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
430 {
431 struct sk_msg tmp, *msg_tx = NULL;
432 int copied = 0, err = 0;
433 struct sk_psock *psock;
434 long timeo;
435 int flags;
436
437 /* Don't let internal do_tcp_sendpages() flags through */
438 flags = (msg->msg_flags & ~MSG_SENDPAGE_DECRYPTED);
439 flags |= MSG_NO_SHARED_FRAGS;
440
441 psock = sk_psock_get(sk);
442 if (unlikely(!psock))
443 return tcp_sendmsg(sk, msg, size);
444
445 lock_sock(sk);
446 timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
447 while (msg_data_left(msg)) {
448 bool enospc = false;
449 u32 copy, osize;
450
451 if (sk->sk_err) {
452 err = -sk->sk_err;
453 goto out_err;
454 }
455
456 copy = msg_data_left(msg);
457 if (!sk_stream_memory_free(sk))
458 goto wait_for_sndbuf;
459 if (psock->cork) {
460 msg_tx = psock->cork;
461 } else {
462 msg_tx = &tmp;
463 sk_msg_init(msg_tx);
464 }
465
466 osize = msg_tx->sg.size;
467 err = sk_msg_alloc(sk, msg_tx, msg_tx->sg.size + copy, msg_tx->sg.end - 1);
468 if (err) {
469 if (err != -ENOSPC)
470 goto wait_for_memory;
471 enospc = true;
472 copy = msg_tx->sg.size - osize;
473 }
474
475 err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, msg_tx,
476 copy);
477 if (err < 0) {
478 sk_msg_trim(sk, msg_tx, osize);
479 goto out_err;
480 }
481
482 copied += copy;
483 if (psock->cork_bytes) {
484 if (size > psock->cork_bytes)
485 psock->cork_bytes = 0;
486 else
487 psock->cork_bytes -= size;
488 if (psock->cork_bytes && !enospc)
489 goto out_err;
490 /* All cork bytes are accounted, rerun the prog. */
491 psock->eval = __SK_NONE;
492 psock->cork_bytes = 0;
493 }
494
495 err = tcp_bpf_send_verdict(sk, psock, msg_tx, &copied, flags);
496 if (unlikely(err < 0))
497 goto out_err;
498 continue;
499 wait_for_sndbuf:
500 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
501 wait_for_memory:
502 err = sk_stream_wait_memory(sk, &timeo);
503 if (err) {
504 if (msg_tx && msg_tx != psock->cork)
505 sk_msg_free(sk, msg_tx);
506 goto out_err;
507 }
508 }
509 out_err:
510 if (err < 0)
511 err = sk_stream_error(sk, msg->msg_flags, err);
512 release_sock(sk);
513 sk_psock_put(sk, psock);
514 return copied ? copied : err;
515 }
516
tcp_bpf_sendpage(struct sock * sk,struct page * page,int offset,size_t size,int flags)517 static int tcp_bpf_sendpage(struct sock *sk, struct page *page, int offset,
518 size_t size, int flags)
519 {
520 struct sk_msg tmp, *msg = NULL;
521 int err = 0, copied = 0;
522 struct sk_psock *psock;
523 bool enospc = false;
524
525 psock = sk_psock_get(sk);
526 if (unlikely(!psock))
527 return tcp_sendpage(sk, page, offset, size, flags);
528
529 lock_sock(sk);
530 if (psock->cork) {
531 msg = psock->cork;
532 } else {
533 msg = &tmp;
534 sk_msg_init(msg);
535 }
536
537 /* Catch case where ring is full and sendpage is stalled. */
538 if (unlikely(sk_msg_full(msg)))
539 goto out_err;
540
541 sk_msg_page_add(msg, page, size, offset);
542 sk_mem_charge(sk, size);
543 copied = size;
544 if (sk_msg_full(msg))
545 enospc = true;
546 if (psock->cork_bytes) {
547 if (size > psock->cork_bytes)
548 psock->cork_bytes = 0;
549 else
550 psock->cork_bytes -= size;
551 if (psock->cork_bytes && !enospc)
552 goto out_err;
553 /* All cork bytes are accounted, rerun the prog. */
554 psock->eval = __SK_NONE;
555 psock->cork_bytes = 0;
556 }
557
558 err = tcp_bpf_send_verdict(sk, psock, msg, &copied, flags);
559 out_err:
560 release_sock(sk);
561 sk_psock_put(sk, psock);
562 return copied ? copied : err;
563 }
564
565 enum {
566 TCP_BPF_IPV4,
567 TCP_BPF_IPV6,
568 TCP_BPF_NUM_PROTS,
569 };
570
571 enum {
572 TCP_BPF_BASE,
573 TCP_BPF_TX,
574 TCP_BPF_NUM_CFGS,
575 };
576
577 static struct proto *tcpv6_prot_saved __read_mostly;
578 static DEFINE_SPINLOCK(tcpv6_prot_lock);
579 static struct proto tcp_bpf_prots[TCP_BPF_NUM_PROTS][TCP_BPF_NUM_CFGS];
580
tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS],struct proto * base)581 static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS],
582 struct proto *base)
583 {
584 prot[TCP_BPF_BASE] = *base;
585 prot[TCP_BPF_BASE].close = sock_map_close;
586 prot[TCP_BPF_BASE].recvmsg = tcp_bpf_recvmsg;
587 prot[TCP_BPF_BASE].stream_memory_read = tcp_bpf_stream_read;
588
589 prot[TCP_BPF_TX] = prot[TCP_BPF_BASE];
590 prot[TCP_BPF_TX].sendmsg = tcp_bpf_sendmsg;
591 prot[TCP_BPF_TX].sendpage = tcp_bpf_sendpage;
592 }
593
tcp_bpf_check_v6_needs_rebuild(struct proto * ops)594 static void tcp_bpf_check_v6_needs_rebuild(struct proto *ops)
595 {
596 if (unlikely(ops != smp_load_acquire(&tcpv6_prot_saved))) {
597 spin_lock_bh(&tcpv6_prot_lock);
598 if (likely(ops != tcpv6_prot_saved)) {
599 tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV6], ops);
600 smp_store_release(&tcpv6_prot_saved, ops);
601 }
602 spin_unlock_bh(&tcpv6_prot_lock);
603 }
604 }
605
tcp_bpf_v4_build_proto(void)606 static int __init tcp_bpf_v4_build_proto(void)
607 {
608 tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV4], &tcp_prot);
609 return 0;
610 }
611 late_initcall(tcp_bpf_v4_build_proto);
612
tcp_bpf_assert_proto_ops(struct proto * ops)613 static int tcp_bpf_assert_proto_ops(struct proto *ops)
614 {
615 /* In order to avoid retpoline, we make assumptions when we call
616 * into ops if e.g. a psock is not present. Make sure they are
617 * indeed valid assumptions.
618 */
619 return ops->recvmsg == tcp_recvmsg &&
620 ops->sendmsg == tcp_sendmsg &&
621 ops->sendpage == tcp_sendpage ? 0 : -ENOTSUPP;
622 }
623
tcp_bpf_get_proto(struct sock * sk,struct sk_psock * psock)624 struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock)
625 {
626 int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4;
627 int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE;
628
629 if (sk->sk_family == AF_INET6) {
630 if (tcp_bpf_assert_proto_ops(psock->sk_proto))
631 return ERR_PTR(-EINVAL);
632
633 tcp_bpf_check_v6_needs_rebuild(psock->sk_proto);
634 }
635
636 return &tcp_bpf_prots[family][config];
637 }
638
639 /* If a child got cloned from a listening socket that had tcp_bpf
640 * protocol callbacks installed, we need to restore the callbacks to
641 * the default ones because the child does not inherit the psock state
642 * that tcp_bpf callbacks expect.
643 */
tcp_bpf_clone(const struct sock * sk,struct sock * newsk)644 void tcp_bpf_clone(const struct sock *sk, struct sock *newsk)
645 {
646 struct proto *prot = newsk->sk_prot;
647
648 if (is_insidevar(prot, tcp_bpf_prots))
649 newsk->sk_prot = sk->sk_prot_creator;
650 }
651 #endif /* CONFIG_BPF_STREAM_PARSER */
652