1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
3
4 #include <linux/skmsg.h>
5 #include <linux/filter.h>
6 #include <linux/bpf.h>
7 #include <linux/init.h>
8 #include <linux/wait.h>
9
10 #include <net/inet_common.h>
11 #include <net/tls.h>
12
__tcp_bpf_recvmsg(struct sock * sk,struct sk_psock * psock,struct msghdr * msg,int len,int flags)13 int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,
14 struct msghdr *msg, int len, int flags)
15 {
16 struct iov_iter *iter = &msg->msg_iter;
17 int peek = flags & MSG_PEEK;
18 struct sk_msg *msg_rx;
19 int i, copied = 0;
20
21 msg_rx = list_first_entry_or_null(&psock->ingress_msg,
22 struct sk_msg, list);
23
24 while (copied != len) {
25 struct scatterlist *sge;
26
27 if (unlikely(!msg_rx))
28 break;
29
30 i = msg_rx->sg.start;
31 do {
32 struct page *page;
33 int copy;
34
35 sge = sk_msg_elem(msg_rx, i);
36 copy = sge->length;
37 page = sg_page(sge);
38 if (copied + copy > len)
39 copy = len - copied;
40 copy = copy_page_to_iter(page, sge->offset, copy, iter);
41 if (!copy)
42 return copied ? copied : -EFAULT;
43
44 copied += copy;
45 if (likely(!peek)) {
46 sge->offset += copy;
47 sge->length -= copy;
48 if (!msg_rx->skb)
49 sk_mem_uncharge(sk, copy);
50 msg_rx->sg.size -= copy;
51
52 if (!sge->length) {
53 sk_msg_iter_var_next(i);
54 if (!msg_rx->skb)
55 put_page(page);
56 }
57 } else {
58 /* Lets not optimize peek case if copy_page_to_iter
59 * didn't copy the entire length lets just break.
60 */
61 if (copy != sge->length)
62 return copied;
63 sk_msg_iter_var_next(i);
64 }
65
66 if (copied == len)
67 break;
68 } while (i != msg_rx->sg.end);
69
70 if (unlikely(peek)) {
71 if (msg_rx == list_last_entry(&psock->ingress_msg,
72 struct sk_msg, list))
73 break;
74 msg_rx = list_next_entry(msg_rx, list);
75 continue;
76 }
77
78 msg_rx->sg.start = i;
79 if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) {
80 list_del(&msg_rx->list);
81 if (msg_rx->skb)
82 consume_skb(msg_rx->skb);
83 kfree(msg_rx);
84 }
85 msg_rx = list_first_entry_or_null(&psock->ingress_msg,
86 struct sk_msg, list);
87 }
88
89 return copied;
90 }
91 EXPORT_SYMBOL_GPL(__tcp_bpf_recvmsg);
92
bpf_tcp_ingress(struct sock * sk,struct sk_psock * psock,struct sk_msg * msg,u32 apply_bytes,int flags)93 static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock,
94 struct sk_msg *msg, u32 apply_bytes, int flags)
95 {
96 bool apply = apply_bytes;
97 struct scatterlist *sge;
98 u32 size, copied = 0;
99 struct sk_msg *tmp;
100 int i, ret = 0;
101
102 tmp = kzalloc(sizeof(*tmp), __GFP_NOWARN | GFP_KERNEL);
103 if (unlikely(!tmp))
104 return -ENOMEM;
105
106 lock_sock(sk);
107 tmp->sg.start = msg->sg.start;
108 i = msg->sg.start;
109 do {
110 sge = sk_msg_elem(msg, i);
111 size = (apply && apply_bytes < sge->length) ?
112 apply_bytes : sge->length;
113 if (!sk_wmem_schedule(sk, size)) {
114 if (!copied)
115 ret = -ENOMEM;
116 break;
117 }
118
119 sk_mem_charge(sk, size);
120 sk_msg_xfer(tmp, msg, i, size);
121 copied += size;
122 if (sge->length)
123 get_page(sk_msg_page(tmp, i));
124 sk_msg_iter_var_next(i);
125 tmp->sg.end = i;
126 if (apply) {
127 apply_bytes -= size;
128 if (!apply_bytes)
129 break;
130 }
131 } while (i != msg->sg.end);
132
133 if (!ret) {
134 msg->sg.start = i;
135 sk_psock_queue_msg(psock, tmp);
136 sk_psock_data_ready(sk, psock);
137 } else {
138 sk_msg_free(sk, tmp);
139 kfree(tmp);
140 }
141
142 release_sock(sk);
143 return ret;
144 }
145
tcp_bpf_push(struct sock * sk,struct sk_msg * msg,u32 apply_bytes,int flags,bool uncharge)146 static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes,
147 int flags, bool uncharge)
148 {
149 bool apply = apply_bytes;
150 struct scatterlist *sge;
151 struct page *page;
152 int size, ret = 0;
153 u32 off;
154
155 while (1) {
156 bool has_tx_ulp;
157
158 sge = sk_msg_elem(msg, msg->sg.start);
159 size = (apply && apply_bytes < sge->length) ?
160 apply_bytes : sge->length;
161 off = sge->offset;
162 page = sg_page(sge);
163
164 tcp_rate_check_app_limited(sk);
165 retry:
166 has_tx_ulp = tls_sw_has_ctx_tx(sk);
167 if (has_tx_ulp) {
168 flags |= MSG_SENDPAGE_NOPOLICY;
169 ret = kernel_sendpage_locked(sk,
170 page, off, size, flags);
171 } else {
172 ret = do_tcp_sendpages(sk, page, off, size, flags);
173 }
174
175 if (ret <= 0)
176 return ret;
177 if (apply)
178 apply_bytes -= ret;
179 msg->sg.size -= ret;
180 sge->offset += ret;
181 sge->length -= ret;
182 if (uncharge)
183 sk_mem_uncharge(sk, ret);
184 if (ret != size) {
185 size -= ret;
186 off += ret;
187 goto retry;
188 }
189 if (!sge->length) {
190 put_page(page);
191 sk_msg_iter_next(msg, start);
192 sg_init_table(sge, 1);
193 if (msg->sg.start == msg->sg.end)
194 break;
195 }
196 if (apply && !apply_bytes)
197 break;
198 }
199
200 return 0;
201 }
202
tcp_bpf_push_locked(struct sock * sk,struct sk_msg * msg,u32 apply_bytes,int flags,bool uncharge)203 static int tcp_bpf_push_locked(struct sock *sk, struct sk_msg *msg,
204 u32 apply_bytes, int flags, bool uncharge)
205 {
206 int ret;
207
208 lock_sock(sk);
209 ret = tcp_bpf_push(sk, msg, apply_bytes, flags, uncharge);
210 release_sock(sk);
211 return ret;
212 }
213
tcp_bpf_sendmsg_redir(struct sock * sk,struct sk_msg * msg,u32 bytes,int flags)214 int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg,
215 u32 bytes, int flags)
216 {
217 bool ingress = sk_msg_to_ingress(msg);
218 struct sk_psock *psock = sk_psock_get(sk);
219 int ret;
220
221 if (unlikely(!psock)) {
222 sk_msg_free(sk, msg);
223 return 0;
224 }
225 ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) :
226 tcp_bpf_push_locked(sk, msg, bytes, flags, false);
227 sk_psock_put(sk, psock);
228 return ret;
229 }
230 EXPORT_SYMBOL_GPL(tcp_bpf_sendmsg_redir);
231
232 #ifdef CONFIG_BPF_STREAM_PARSER
tcp_bpf_stream_read(const struct sock * sk)233 static bool tcp_bpf_stream_read(const struct sock *sk)
234 {
235 struct sk_psock *psock;
236 bool empty = true;
237
238 rcu_read_lock();
239 psock = sk_psock(sk);
240 if (likely(psock))
241 empty = list_empty(&psock->ingress_msg);
242 rcu_read_unlock();
243 return !empty;
244 }
245
tcp_bpf_wait_data(struct sock * sk,struct sk_psock * psock,int flags,long timeo,int * err)246 static int tcp_bpf_wait_data(struct sock *sk, struct sk_psock *psock,
247 int flags, long timeo, int *err)
248 {
249 DEFINE_WAIT_FUNC(wait, woken_wake_function);
250 int ret = 0;
251
252 if (sk->sk_shutdown & RCV_SHUTDOWN)
253 return 1;
254
255 if (!timeo)
256 return ret;
257
258 add_wait_queue(sk_sleep(sk), &wait);
259 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
260 ret = sk_wait_event(sk, &timeo,
261 !list_empty(&psock->ingress_msg) ||
262 !skb_queue_empty(&sk->sk_receive_queue), &wait);
263 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
264 remove_wait_queue(sk_sleep(sk), &wait);
265 return ret;
266 }
267
tcp_bpf_recvmsg(struct sock * sk,struct msghdr * msg,size_t len,int nonblock,int flags,int * addr_len)268 static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
269 int nonblock, int flags, int *addr_len)
270 {
271 struct sk_psock *psock;
272 int copied, ret;
273
274 if (unlikely(flags & MSG_ERRQUEUE))
275 return inet_recv_error(sk, msg, len, addr_len);
276
277 psock = sk_psock_get(sk);
278 if (unlikely(!psock))
279 return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
280 if (!skb_queue_empty(&sk->sk_receive_queue) &&
281 sk_psock_queue_empty(psock)) {
282 sk_psock_put(sk, psock);
283 return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
284 }
285 lock_sock(sk);
286 msg_bytes_ready:
287 copied = __tcp_bpf_recvmsg(sk, psock, msg, len, flags);
288 if (!copied) {
289 int data, err = 0;
290 long timeo;
291
292 timeo = sock_rcvtimeo(sk, nonblock);
293 data = tcp_bpf_wait_data(sk, psock, flags, timeo, &err);
294 if (data) {
295 if (!sk_psock_queue_empty(psock))
296 goto msg_bytes_ready;
297 release_sock(sk);
298 sk_psock_put(sk, psock);
299 return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
300 }
301 if (err) {
302 ret = err;
303 goto out;
304 }
305 copied = -EAGAIN;
306 }
307 ret = copied;
308 out:
309 release_sock(sk);
310 sk_psock_put(sk, psock);
311 return ret;
312 }
313
tcp_bpf_send_verdict(struct sock * sk,struct sk_psock * psock,struct sk_msg * msg,int * copied,int flags)314 static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock,
315 struct sk_msg *msg, int *copied, int flags)
316 {
317 bool cork = false, enospc = sk_msg_full(msg);
318 struct sock *sk_redir;
319 u32 tosend, delta = 0;
320 u32 eval = __SK_NONE;
321 int ret;
322
323 more_data:
324 if (psock->eval == __SK_NONE) {
325 /* Track delta in msg size to add/subtract it on SK_DROP from
326 * returned to user copied size. This ensures user doesn't
327 * get a positive return code with msg_cut_data and SK_DROP
328 * verdict.
329 */
330 delta = msg->sg.size;
331 psock->eval = sk_psock_msg_verdict(sk, psock, msg);
332 delta -= msg->sg.size;
333 }
334
335 if (msg->cork_bytes &&
336 msg->cork_bytes > msg->sg.size && !enospc) {
337 psock->cork_bytes = msg->cork_bytes - msg->sg.size;
338 if (!psock->cork) {
339 psock->cork = kzalloc(sizeof(*psock->cork),
340 GFP_ATOMIC | __GFP_NOWARN);
341 if (!psock->cork)
342 return -ENOMEM;
343 }
344 memcpy(psock->cork, msg, sizeof(*msg));
345 return 0;
346 }
347
348 tosend = msg->sg.size;
349 if (psock->apply_bytes && psock->apply_bytes < tosend)
350 tosend = psock->apply_bytes;
351
352 switch (psock->eval) {
353 case __SK_PASS:
354 ret = tcp_bpf_push(sk, msg, tosend, flags, true);
355 if (unlikely(ret)) {
356 *copied -= sk_msg_free(sk, msg);
357 break;
358 }
359 sk_msg_apply_bytes(psock, tosend);
360 break;
361 case __SK_REDIRECT:
362 sk_redir = psock->sk_redir;
363 sk_msg_apply_bytes(psock, tosend);
364 if (!psock->apply_bytes) {
365 /* Clean up before releasing the sock lock. */
366 eval = psock->eval;
367 psock->eval = __SK_NONE;
368 psock->sk_redir = NULL;
369 }
370 if (psock->cork) {
371 cork = true;
372 psock->cork = NULL;
373 }
374 sk_msg_return(sk, msg, tosend);
375 release_sock(sk);
376
377 ret = tcp_bpf_sendmsg_redir(sk_redir, msg, tosend, flags);
378
379 if (eval == __SK_REDIRECT)
380 sock_put(sk_redir);
381
382 lock_sock(sk);
383 if (unlikely(ret < 0)) {
384 int free = sk_msg_free_nocharge(sk, msg);
385
386 if (!cork)
387 *copied -= free;
388 }
389 if (cork) {
390 sk_msg_free(sk, msg);
391 kfree(msg);
392 msg = NULL;
393 ret = 0;
394 }
395 break;
396 case __SK_DROP:
397 default:
398 sk_msg_free_partial(sk, msg, tosend);
399 sk_msg_apply_bytes(psock, tosend);
400 *copied -= (tosend + delta);
401 return -EACCES;
402 }
403
404 if (likely(!ret)) {
405 if (!psock->apply_bytes) {
406 psock->eval = __SK_NONE;
407 if (psock->sk_redir) {
408 sock_put(psock->sk_redir);
409 psock->sk_redir = NULL;
410 }
411 }
412 if (msg &&
413 msg->sg.data[msg->sg.start].page_link &&
414 msg->sg.data[msg->sg.start].length)
415 goto more_data;
416 }
417 return ret;
418 }
419
tcp_bpf_sendmsg(struct sock * sk,struct msghdr * msg,size_t size)420 static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
421 {
422 struct sk_msg tmp, *msg_tx = NULL;
423 int copied = 0, err = 0;
424 struct sk_psock *psock;
425 long timeo;
426 int flags;
427
428 /* Don't let internal do_tcp_sendpages() flags through */
429 flags = (msg->msg_flags & ~MSG_SENDPAGE_DECRYPTED);
430 flags |= MSG_NO_SHARED_FRAGS;
431
432 psock = sk_psock_get(sk);
433 if (unlikely(!psock))
434 return tcp_sendmsg(sk, msg, size);
435
436 lock_sock(sk);
437 timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
438 while (msg_data_left(msg)) {
439 bool enospc = false;
440 u32 copy, osize;
441
442 if (sk->sk_err) {
443 err = -sk->sk_err;
444 goto out_err;
445 }
446
447 copy = msg_data_left(msg);
448 if (!sk_stream_memory_free(sk))
449 goto wait_for_sndbuf;
450 if (psock->cork) {
451 msg_tx = psock->cork;
452 } else {
453 msg_tx = &tmp;
454 sk_msg_init(msg_tx);
455 }
456
457 osize = msg_tx->sg.size;
458 err = sk_msg_alloc(sk, msg_tx, msg_tx->sg.size + copy, msg_tx->sg.end - 1);
459 if (err) {
460 if (err != -ENOSPC)
461 goto wait_for_memory;
462 enospc = true;
463 copy = msg_tx->sg.size - osize;
464 }
465
466 err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, msg_tx,
467 copy);
468 if (err < 0) {
469 sk_msg_trim(sk, msg_tx, osize);
470 goto out_err;
471 }
472
473 copied += copy;
474 if (psock->cork_bytes) {
475 if (size > psock->cork_bytes)
476 psock->cork_bytes = 0;
477 else
478 psock->cork_bytes -= size;
479 if (psock->cork_bytes && !enospc)
480 goto out_err;
481 /* All cork bytes are accounted, rerun the prog. */
482 psock->eval = __SK_NONE;
483 psock->cork_bytes = 0;
484 }
485
486 err = tcp_bpf_send_verdict(sk, psock, msg_tx, &copied, flags);
487 if (unlikely(err < 0))
488 goto out_err;
489 continue;
490 wait_for_sndbuf:
491 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
492 wait_for_memory:
493 err = sk_stream_wait_memory(sk, &timeo);
494 if (err) {
495 if (msg_tx && msg_tx != psock->cork)
496 sk_msg_free(sk, msg_tx);
497 goto out_err;
498 }
499 }
500 out_err:
501 if (err < 0)
502 err = sk_stream_error(sk, msg->msg_flags, err);
503 release_sock(sk);
504 sk_psock_put(sk, psock);
505 return copied ? copied : err;
506 }
507
tcp_bpf_sendpage(struct sock * sk,struct page * page,int offset,size_t size,int flags)508 static int tcp_bpf_sendpage(struct sock *sk, struct page *page, int offset,
509 size_t size, int flags)
510 {
511 struct sk_msg tmp, *msg = NULL;
512 int err = 0, copied = 0;
513 struct sk_psock *psock;
514 bool enospc = false;
515
516 psock = sk_psock_get(sk);
517 if (unlikely(!psock))
518 return tcp_sendpage(sk, page, offset, size, flags);
519
520 lock_sock(sk);
521 if (psock->cork) {
522 msg = psock->cork;
523 } else {
524 msg = &tmp;
525 sk_msg_init(msg);
526 }
527
528 /* Catch case where ring is full and sendpage is stalled. */
529 if (unlikely(sk_msg_full(msg)))
530 goto out_err;
531
532 sk_msg_page_add(msg, page, size, offset);
533 sk_mem_charge(sk, size);
534 copied = size;
535 if (sk_msg_full(msg))
536 enospc = true;
537 if (psock->cork_bytes) {
538 if (size > psock->cork_bytes)
539 psock->cork_bytes = 0;
540 else
541 psock->cork_bytes -= size;
542 if (psock->cork_bytes && !enospc)
543 goto out_err;
544 /* All cork bytes are accounted, rerun the prog. */
545 psock->eval = __SK_NONE;
546 psock->cork_bytes = 0;
547 }
548
549 err = tcp_bpf_send_verdict(sk, psock, msg, &copied, flags);
550 out_err:
551 release_sock(sk);
552 sk_psock_put(sk, psock);
553 return copied ? copied : err;
554 }
555
556 enum {
557 TCP_BPF_IPV4,
558 TCP_BPF_IPV6,
559 TCP_BPF_NUM_PROTS,
560 };
561
562 enum {
563 TCP_BPF_BASE,
564 TCP_BPF_TX,
565 TCP_BPF_NUM_CFGS,
566 };
567
568 static struct proto *tcpv6_prot_saved __read_mostly;
569 static DEFINE_SPINLOCK(tcpv6_prot_lock);
570 static struct proto tcp_bpf_prots[TCP_BPF_NUM_PROTS][TCP_BPF_NUM_CFGS];
571
tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS],struct proto * base)572 static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS],
573 struct proto *base)
574 {
575 prot[TCP_BPF_BASE] = *base;
576 prot[TCP_BPF_BASE].close = sock_map_close;
577 prot[TCP_BPF_BASE].recvmsg = tcp_bpf_recvmsg;
578 prot[TCP_BPF_BASE].stream_memory_read = tcp_bpf_stream_read;
579
580 prot[TCP_BPF_TX] = prot[TCP_BPF_BASE];
581 prot[TCP_BPF_TX].sendmsg = tcp_bpf_sendmsg;
582 prot[TCP_BPF_TX].sendpage = tcp_bpf_sendpage;
583 }
584
tcp_bpf_check_v6_needs_rebuild(struct proto * ops)585 static void tcp_bpf_check_v6_needs_rebuild(struct proto *ops)
586 {
587 if (unlikely(ops != smp_load_acquire(&tcpv6_prot_saved))) {
588 spin_lock_bh(&tcpv6_prot_lock);
589 if (likely(ops != tcpv6_prot_saved)) {
590 tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV6], ops);
591 smp_store_release(&tcpv6_prot_saved, ops);
592 }
593 spin_unlock_bh(&tcpv6_prot_lock);
594 }
595 }
596
tcp_bpf_v4_build_proto(void)597 static int __init tcp_bpf_v4_build_proto(void)
598 {
599 tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV4], &tcp_prot);
600 return 0;
601 }
602 late_initcall(tcp_bpf_v4_build_proto);
603
tcp_bpf_assert_proto_ops(struct proto * ops)604 static int tcp_bpf_assert_proto_ops(struct proto *ops)
605 {
606 /* In order to avoid retpoline, we make assumptions when we call
607 * into ops if e.g. a psock is not present. Make sure they are
608 * indeed valid assumptions.
609 */
610 return ops->recvmsg == tcp_recvmsg &&
611 ops->sendmsg == tcp_sendmsg &&
612 ops->sendpage == tcp_sendpage ? 0 : -ENOTSUPP;
613 }
614
tcp_bpf_get_proto(struct sock * sk,struct sk_psock * psock)615 struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock)
616 {
617 int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4;
618 int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE;
619
620 if (sk->sk_family == AF_INET6) {
621 if (tcp_bpf_assert_proto_ops(psock->sk_proto))
622 return ERR_PTR(-EINVAL);
623
624 tcp_bpf_check_v6_needs_rebuild(psock->sk_proto);
625 }
626
627 return &tcp_bpf_prots[family][config];
628 }
629
630 /* If a child got cloned from a listening socket that had tcp_bpf
631 * protocol callbacks installed, we need to restore the callbacks to
632 * the default ones because the child does not inherit the psock state
633 * that tcp_bpf callbacks expect.
634 */
tcp_bpf_clone(const struct sock * sk,struct sock * newsk)635 void tcp_bpf_clone(const struct sock *sk, struct sock *newsk)
636 {
637 int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4;
638 struct proto *prot = newsk->sk_prot;
639
640 if (prot == &tcp_bpf_prots[family][TCP_BPF_BASE])
641 newsk->sk_prot = sk->sk_prot_creator;
642 }
643 #endif /* CONFIG_BPF_STREAM_PARSER */
644