Lines Matching refs:msg
11 static bool sk_msg_try_coalesce_ok(struct sk_msg *msg, int elem_first_coalesce) in sk_msg_try_coalesce_ok() argument
13 if (msg->sg.end > msg->sg.start && in sk_msg_try_coalesce_ok()
14 elem_first_coalesce < msg->sg.end) in sk_msg_try_coalesce_ok()
17 if (msg->sg.end < msg->sg.start && in sk_msg_try_coalesce_ok()
18 (elem_first_coalesce > msg->sg.start || in sk_msg_try_coalesce_ok()
19 elem_first_coalesce < msg->sg.end)) in sk_msg_try_coalesce_ok()
25 int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, in sk_msg_alloc() argument
31 len -= msg->sg.size; in sk_msg_alloc()
45 i = msg->sg.end; in sk_msg_alloc()
47 sge = &msg->sg.data[i]; in sk_msg_alloc()
49 if (sk_msg_try_coalesce_ok(msg, elem_first_coalesce) && in sk_msg_alloc()
54 if (sk_msg_full(msg)) { in sk_msg_alloc()
59 sge = &msg->sg.data[msg->sg.end]; in sk_msg_alloc()
63 sk_msg_iter_next(msg, end); in sk_msg_alloc()
67 msg->sg.size += use; in sk_msg_alloc()
127 void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes) in sk_msg_return_zero() argument
129 int i = msg->sg.start; in sk_msg_return_zero()
132 struct scatterlist *sge = sk_msg_elem(msg, i); in sk_msg_return_zero()
146 } while (bytes && i != msg->sg.end); in sk_msg_return_zero()
147 msg->sg.start = i; in sk_msg_return_zero()
151 void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes) in sk_msg_return() argument
153 int i = msg->sg.start; in sk_msg_return()
156 struct scatterlist *sge = &msg->sg.data[i]; in sk_msg_return()
162 } while (i != msg->sg.end); in sk_msg_return()
166 static int sk_msg_free_elem(struct sock *sk, struct sk_msg *msg, u32 i, in sk_msg_free_elem() argument
169 struct scatterlist *sge = sk_msg_elem(msg, i); in sk_msg_free_elem()
174 if (!msg->skb) in sk_msg_free_elem()
180 static int __sk_msg_free(struct sock *sk, struct sk_msg *msg, u32 i, in __sk_msg_free() argument
183 struct scatterlist *sge = sk_msg_elem(msg, i); in __sk_msg_free()
186 while (msg->sg.size) { in __sk_msg_free()
187 msg->sg.size -= sge->length; in __sk_msg_free()
188 freed += sk_msg_free_elem(sk, msg, i, charge); in __sk_msg_free()
190 sk_msg_check_to_free(msg, i, msg->sg.size); in __sk_msg_free()
191 sge = sk_msg_elem(msg, i); in __sk_msg_free()
193 consume_skb(msg->skb); in __sk_msg_free()
194 sk_msg_init(msg); in __sk_msg_free()
198 int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg) in sk_msg_free_nocharge() argument
200 return __sk_msg_free(sk, msg, msg->sg.start, false); in sk_msg_free_nocharge()
204 int sk_msg_free(struct sock *sk, struct sk_msg *msg) in sk_msg_free() argument
206 return __sk_msg_free(sk, msg, msg->sg.start, true); in sk_msg_free()
210 static void __sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, in __sk_msg_free_partial() argument
214 u32 i = msg->sg.start; in __sk_msg_free_partial()
217 sge = sk_msg_elem(msg, i); in __sk_msg_free_partial()
225 msg->sg.size -= bytes; in __sk_msg_free_partial()
229 msg->sg.size -= sge->length; in __sk_msg_free_partial()
231 sk_msg_free_elem(sk, msg, i, charge); in __sk_msg_free_partial()
233 sk_msg_check_to_free(msg, i, bytes); in __sk_msg_free_partial()
235 msg->sg.start = i; in __sk_msg_free_partial()
238 void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes) in sk_msg_free_partial() argument
240 __sk_msg_free_partial(sk, msg, bytes, true); in sk_msg_free_partial()
244 void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg, in sk_msg_free_partial_nocharge() argument
247 __sk_msg_free_partial(sk, msg, bytes, false); in sk_msg_free_partial_nocharge()
250 void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len) in sk_msg_trim() argument
252 int trim = msg->sg.size - len; in sk_msg_trim()
253 u32 i = msg->sg.end; in sk_msg_trim()
261 msg->sg.size = len; in sk_msg_trim()
262 while (msg->sg.data[i].length && in sk_msg_trim()
263 trim >= msg->sg.data[i].length) { in sk_msg_trim()
264 trim -= msg->sg.data[i].length; in sk_msg_trim()
265 sk_msg_free_elem(sk, msg, i, true); in sk_msg_trim()
271 msg->sg.data[i].length -= trim; in sk_msg_trim()
274 if (msg->sg.curr == i && msg->sg.copybreak > msg->sg.data[i].length) in sk_msg_trim()
275 msg->sg.copybreak = msg->sg.data[i].length; in sk_msg_trim()
278 msg->sg.end = i; in sk_msg_trim()
286 if (!msg->sg.size) { in sk_msg_trim()
287 msg->sg.curr = msg->sg.start; in sk_msg_trim()
288 msg->sg.copybreak = 0; in sk_msg_trim()
289 } else if (sk_msg_iter_dist(msg->sg.start, msg->sg.curr) >= in sk_msg_trim()
290 sk_msg_iter_dist(msg->sg.start, msg->sg.end)) { in sk_msg_trim()
292 msg->sg.curr = i; in sk_msg_trim()
293 msg->sg.copybreak = msg->sg.data[i].length; in sk_msg_trim()
299 struct sk_msg *msg, u32 bytes) in sk_msg_zerocopy_from_iter() argument
301 int i, maxpages, ret = 0, num_elems = sk_msg_elem_used(msg); in sk_msg_zerocopy_from_iter()
306 orig = msg->sg.size; in sk_msg_zerocopy_from_iter()
324 msg->sg.size += copied; in sk_msg_zerocopy_from_iter()
328 sg_set_page(&msg->sg.data[msg->sg.end], in sk_msg_zerocopy_from_iter()
330 sg_unmark_end(&msg->sg.data[msg->sg.end]); in sk_msg_zerocopy_from_iter()
335 sk_msg_iter_next(msg, end); in sk_msg_zerocopy_from_iter()
343 msg->sg.copybreak = 0; in sk_msg_zerocopy_from_iter()
344 msg->sg.curr = msg->sg.end; in sk_msg_zerocopy_from_iter()
351 iov_iter_revert(from, msg->sg.size - orig); in sk_msg_zerocopy_from_iter()
357 struct sk_msg *msg, u32 bytes) in sk_msg_memcopy_from_iter() argument
359 int ret = -ENOSPC, i = msg->sg.curr; in sk_msg_memcopy_from_iter()
365 sge = sk_msg_elem(msg, i); in sk_msg_memcopy_from_iter()
367 if (msg->sg.copybreak >= sge->length) { in sk_msg_memcopy_from_iter()
368 msg->sg.copybreak = 0; in sk_msg_memcopy_from_iter()
370 if (i == msg->sg.end) in sk_msg_memcopy_from_iter()
372 sge = sk_msg_elem(msg, i); in sk_msg_memcopy_from_iter()
375 buf_size = sge->length - msg->sg.copybreak; in sk_msg_memcopy_from_iter()
377 to = sg_virt(sge) + msg->sg.copybreak; in sk_msg_memcopy_from_iter()
378 msg->sg.copybreak += copy; in sk_msg_memcopy_from_iter()
390 msg->sg.copybreak = 0; in sk_msg_memcopy_from_iter()
392 } while (i != msg->sg.end); in sk_msg_memcopy_from_iter()
394 msg->sg.curr = i; in sk_msg_memcopy_from_iter()
403 struct sk_msg *msg; in sk_psock_skb_ingress() local
405 msg = kzalloc(sizeof(*msg), __GFP_NOWARN | GFP_ATOMIC); in sk_psock_skb_ingress()
406 if (unlikely(!msg)) in sk_psock_skb_ingress()
409 kfree(msg); in sk_psock_skb_ingress()
413 sk_msg_init(msg); in sk_psock_skb_ingress()
414 num_sge = skb_to_sgvec(skb, msg->sg.data, 0, skb->len); in sk_psock_skb_ingress()
416 kfree(msg); in sk_psock_skb_ingress()
422 msg->sg.start = 0; in sk_psock_skb_ingress()
423 msg->sg.size = copied; in sk_psock_skb_ingress()
424 msg->sg.end = num_sge; in sk_psock_skb_ingress()
425 msg->skb = skb; in sk_psock_skb_ingress()
427 sk_psock_queue_msg(psock, msg); in sk_psock_skb_ingress()
537 struct sk_msg *msg, *tmp; in __sk_psock_purge_ingress_msg() local
539 list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) { in __sk_psock_purge_ingress_msg()
540 list_del(&msg->list); in __sk_psock_purge_ingress_msg()
541 sk_msg_free(psock->sk, msg); in __sk_psock_purge_ingress_msg()
542 kfree(msg); in __sk_psock_purge_ingress_msg()
626 struct sk_msg *msg) in sk_psock_msg_verdict() argument
639 sk_msg_compute_data_pointers(msg); in sk_psock_msg_verdict()
640 msg->sk = sk; in sk_psock_msg_verdict()
641 ret = BPF_PROG_RUN(prog, msg); in sk_psock_msg_verdict()
642 ret = sk_psock_map_verd(ret, msg->sk_redir); in sk_psock_msg_verdict()
643 psock->apply_bytes = msg->apply_bytes; in sk_psock_msg_verdict()
647 psock->sk_redir = msg->sk_redir; in sk_psock_msg_verdict()