1 #include <linux/kernel.h>
2 #include <linux/errno.h>
3 #include <linux/file.h>
4 #include <linux/slab.h>
5 #include <linux/net.h>
6 #include <linux/io_uring.h>
7
8 #include "io_uring.h"
9 #include "notif.h"
10 #include "rsrc.h"
11
__io_notif_complete_tw(struct io_kiocb * notif,bool * locked)12 static void __io_notif_complete_tw(struct io_kiocb *notif, bool *locked)
13 {
14 struct io_notif_data *nd = io_notif_to_data(notif);
15 struct io_ring_ctx *ctx = notif->ctx;
16
17 if (nd->account_pages && ctx->user) {
18 __io_unaccount_mem(ctx->user, nd->account_pages);
19 nd->account_pages = 0;
20 }
21
22 if (nd->zc_report && (nd->zc_copied || !nd->zc_used))
23 notif->cqe.res |= IORING_NOTIF_USAGE_ZC_COPIED;
24
25 io_req_task_complete(notif, locked);
26 }
27
io_uring_tx_zerocopy_callback(struct sk_buff * skb,struct ubuf_info * uarg,bool success)28 static void io_uring_tx_zerocopy_callback(struct sk_buff *skb,
29 struct ubuf_info *uarg,
30 bool success)
31 {
32 struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg);
33 struct io_kiocb *notif = cmd_to_io_kiocb(nd);
34
35 if (nd->zc_report) {
36 if (success && !nd->zc_used && skb)
37 WRITE_ONCE(nd->zc_used, true);
38 else if (!success && !nd->zc_copied)
39 WRITE_ONCE(nd->zc_copied, true);
40 }
41
42 if (refcount_dec_and_test(&uarg->refcnt)) {
43 notif->io_task_work.func = __io_notif_complete_tw;
44 io_req_task_work_add(notif);
45 }
46 }
47
io_alloc_notif(struct io_ring_ctx * ctx)48 struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
49 __must_hold(&ctx->uring_lock)
50 {
51 struct io_kiocb *notif;
52 struct io_notif_data *nd;
53
54 if (unlikely(!io_alloc_req_refill(ctx)))
55 return NULL;
56 notif = io_alloc_req(ctx);
57 notif->opcode = IORING_OP_NOP;
58 notif->flags = 0;
59 notif->file = NULL;
60 notif->task = current;
61 io_get_task_refs(1);
62 notif->rsrc_node = NULL;
63 io_req_set_rsrc_node(notif, ctx, 0);
64
65 nd = io_notif_to_data(notif);
66 nd->account_pages = 0;
67 nd->uarg.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;
68 nd->uarg.callback = io_uring_tx_zerocopy_callback;
69 nd->zc_report = nd->zc_used = nd->zc_copied = false;
70 refcount_set(&nd->uarg.refcnt, 1);
71 return notif;
72 }
73
io_notif_flush(struct io_kiocb * notif)74 void io_notif_flush(struct io_kiocb *notif)
75 __must_hold(&slot->notif->ctx->uring_lock)
76 {
77 struct io_notif_data *nd = io_notif_to_data(notif);
78
79 /* drop slot's master ref */
80 if (refcount_dec_and_test(&nd->uarg.refcnt)) {
81 notif->io_task_work.func = __io_notif_complete_tw;
82 io_req_task_work_add(notif);
83 }
84 }
85