1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * io_uring opcode handling table
4 */
5 #include <linux/kernel.h>
6 #include <linux/errno.h>
7 #include <linux/fs.h>
8 #include <linux/file.h>
9 #include <linux/io_uring.h>
10
11 #include "io_uring.h"
12 #include "opdef.h"
13 #include "refs.h"
14 #include "tctx.h"
15 #include "sqpoll.h"
16 #include "fdinfo.h"
17 #include "kbuf.h"
18 #include "rsrc.h"
19
20 #include "xattr.h"
21 #include "nop.h"
22 #include "fs.h"
23 #include "splice.h"
24 #include "sync.h"
25 #include "advise.h"
26 #include "openclose.h"
27 #include "uring_cmd.h"
28 #include "epoll.h"
29 #include "statx.h"
30 #include "net.h"
31 #include "msg_ring.h"
32 #include "timeout.h"
33 #include "poll.h"
34 #include "cancel.h"
35 #include "rw.h"
36
io_no_issue(struct io_kiocb * req,unsigned int issue_flags)37 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
38 {
39 WARN_ON_ONCE(1);
40 return -ECANCELED;
41 }
42
io_eopnotsupp_prep(struct io_kiocb * kiocb,const struct io_uring_sqe * sqe)43 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
44 const struct io_uring_sqe *sqe)
45 {
46 return -EOPNOTSUPP;
47 }
48
49 const struct io_issue_def io_issue_defs[] = {
50 [IORING_OP_NOP] = {
51 .audit_skip = 1,
52 .iopoll = 1,
53 .prep = io_nop_prep,
54 .issue = io_nop,
55 },
56 [IORING_OP_READV] = {
57 .needs_file = 1,
58 .unbound_nonreg_file = 1,
59 .pollin = 1,
60 .buffer_select = 1,
61 .plug = 1,
62 .audit_skip = 1,
63 .ioprio = 1,
64 .iopoll = 1,
65 .iopoll_queue = 1,
66 .prep = io_prep_rw,
67 .issue = io_read,
68 },
69 [IORING_OP_WRITEV] = {
70 .needs_file = 1,
71 .hash_reg_file = 1,
72 .unbound_nonreg_file = 1,
73 .pollout = 1,
74 .plug = 1,
75 .audit_skip = 1,
76 .ioprio = 1,
77 .iopoll = 1,
78 .iopoll_queue = 1,
79 .prep = io_prep_rw,
80 .issue = io_write,
81 },
82 [IORING_OP_FSYNC] = {
83 .needs_file = 1,
84 .audit_skip = 1,
85 .prep = io_fsync_prep,
86 .issue = io_fsync,
87 },
88 [IORING_OP_READ_FIXED] = {
89 .needs_file = 1,
90 .unbound_nonreg_file = 1,
91 .pollin = 1,
92 .plug = 1,
93 .audit_skip = 1,
94 .ioprio = 1,
95 .iopoll = 1,
96 .iopoll_queue = 1,
97 .prep = io_prep_rw,
98 .issue = io_read,
99 },
100 [IORING_OP_WRITE_FIXED] = {
101 .needs_file = 1,
102 .hash_reg_file = 1,
103 .unbound_nonreg_file = 1,
104 .pollout = 1,
105 .plug = 1,
106 .audit_skip = 1,
107 .ioprio = 1,
108 .iopoll = 1,
109 .iopoll_queue = 1,
110 .prep = io_prep_rw,
111 .issue = io_write,
112 },
113 [IORING_OP_POLL_ADD] = {
114 .needs_file = 1,
115 .unbound_nonreg_file = 1,
116 .audit_skip = 1,
117 .prep = io_poll_add_prep,
118 .issue = io_poll_add,
119 },
120 [IORING_OP_POLL_REMOVE] = {
121 .audit_skip = 1,
122 .prep = io_poll_remove_prep,
123 .issue = io_poll_remove,
124 },
125 [IORING_OP_SYNC_FILE_RANGE] = {
126 .needs_file = 1,
127 .audit_skip = 1,
128 .prep = io_sfr_prep,
129 .issue = io_sync_file_range,
130 },
131 [IORING_OP_SENDMSG] = {
132 .needs_file = 1,
133 .unbound_nonreg_file = 1,
134 .pollout = 1,
135 .ioprio = 1,
136 .manual_alloc = 1,
137 #if defined(CONFIG_NET)
138 .prep = io_sendmsg_prep,
139 .issue = io_sendmsg,
140 #else
141 .prep = io_eopnotsupp_prep,
142 #endif
143 },
144 [IORING_OP_RECVMSG] = {
145 .needs_file = 1,
146 .unbound_nonreg_file = 1,
147 .pollin = 1,
148 .buffer_select = 1,
149 .ioprio = 1,
150 .manual_alloc = 1,
151 #if defined(CONFIG_NET)
152 .prep = io_recvmsg_prep,
153 .issue = io_recvmsg,
154 #else
155 .prep = io_eopnotsupp_prep,
156 #endif
157 },
158 [IORING_OP_TIMEOUT] = {
159 .audit_skip = 1,
160 .prep = io_timeout_prep,
161 .issue = io_timeout,
162 },
163 [IORING_OP_TIMEOUT_REMOVE] = {
164 /* used by timeout updates' prep() */
165 .audit_skip = 1,
166 .prep = io_timeout_remove_prep,
167 .issue = io_timeout_remove,
168 },
169 [IORING_OP_ACCEPT] = {
170 .needs_file = 1,
171 .unbound_nonreg_file = 1,
172 .pollin = 1,
173 .poll_exclusive = 1,
174 .ioprio = 1, /* used for flags */
175 #if defined(CONFIG_NET)
176 .prep = io_accept_prep,
177 .issue = io_accept,
178 #else
179 .prep = io_eopnotsupp_prep,
180 #endif
181 },
182 [IORING_OP_ASYNC_CANCEL] = {
183 .audit_skip = 1,
184 .prep = io_async_cancel_prep,
185 .issue = io_async_cancel,
186 },
187 [IORING_OP_LINK_TIMEOUT] = {
188 .audit_skip = 1,
189 .prep = io_link_timeout_prep,
190 .issue = io_no_issue,
191 },
192 [IORING_OP_CONNECT] = {
193 .needs_file = 1,
194 .unbound_nonreg_file = 1,
195 .pollout = 1,
196 #if defined(CONFIG_NET)
197 .prep = io_connect_prep,
198 .issue = io_connect,
199 #else
200 .prep = io_eopnotsupp_prep,
201 #endif
202 },
203 [IORING_OP_FALLOCATE] = {
204 .needs_file = 1,
205 .hash_reg_file = 1,
206 .prep = io_fallocate_prep,
207 .issue = io_fallocate,
208 },
209 [IORING_OP_OPENAT] = {
210 .prep = io_openat_prep,
211 .issue = io_openat,
212 },
213 [IORING_OP_CLOSE] = {
214 .prep = io_close_prep,
215 .issue = io_close,
216 },
217 [IORING_OP_FILES_UPDATE] = {
218 .audit_skip = 1,
219 .iopoll = 1,
220 .prep = io_files_update_prep,
221 .issue = io_files_update,
222 },
223 [IORING_OP_STATX] = {
224 .audit_skip = 1,
225 .prep = io_statx_prep,
226 .issue = io_statx,
227 },
228 [IORING_OP_READ] = {
229 .needs_file = 1,
230 .unbound_nonreg_file = 1,
231 .pollin = 1,
232 .buffer_select = 1,
233 .plug = 1,
234 .audit_skip = 1,
235 .ioprio = 1,
236 .iopoll = 1,
237 .iopoll_queue = 1,
238 .prep = io_prep_rw,
239 .issue = io_read,
240 },
241 [IORING_OP_WRITE] = {
242 .needs_file = 1,
243 .hash_reg_file = 1,
244 .unbound_nonreg_file = 1,
245 .pollout = 1,
246 .plug = 1,
247 .audit_skip = 1,
248 .ioprio = 1,
249 .iopoll = 1,
250 .iopoll_queue = 1,
251 .prep = io_prep_rw,
252 .issue = io_write,
253 },
254 [IORING_OP_FADVISE] = {
255 .needs_file = 1,
256 .audit_skip = 1,
257 .prep = io_fadvise_prep,
258 .issue = io_fadvise,
259 },
260 [IORING_OP_MADVISE] = {
261 .audit_skip = 1,
262 .prep = io_madvise_prep,
263 .issue = io_madvise,
264 },
265 [IORING_OP_SEND] = {
266 .needs_file = 1,
267 .unbound_nonreg_file = 1,
268 .pollout = 1,
269 .audit_skip = 1,
270 .ioprio = 1,
271 .manual_alloc = 1,
272 #if defined(CONFIG_NET)
273 .prep = io_sendmsg_prep,
274 .issue = io_send,
275 #else
276 .prep = io_eopnotsupp_prep,
277 #endif
278 },
279 [IORING_OP_RECV] = {
280 .needs_file = 1,
281 .unbound_nonreg_file = 1,
282 .pollin = 1,
283 .buffer_select = 1,
284 .audit_skip = 1,
285 .ioprio = 1,
286 #if defined(CONFIG_NET)
287 .prep = io_recvmsg_prep,
288 .issue = io_recv,
289 #else
290 .prep = io_eopnotsupp_prep,
291 #endif
292 },
293 [IORING_OP_OPENAT2] = {
294 .prep = io_openat2_prep,
295 .issue = io_openat2,
296 },
297 [IORING_OP_EPOLL_CTL] = {
298 .unbound_nonreg_file = 1,
299 .audit_skip = 1,
300 #if defined(CONFIG_EPOLL)
301 .prep = io_epoll_ctl_prep,
302 .issue = io_epoll_ctl,
303 #else
304 .prep = io_eopnotsupp_prep,
305 #endif
306 },
307 [IORING_OP_SPLICE] = {
308 .needs_file = 1,
309 .hash_reg_file = 1,
310 .unbound_nonreg_file = 1,
311 .audit_skip = 1,
312 .prep = io_splice_prep,
313 .issue = io_splice,
314 },
315 [IORING_OP_PROVIDE_BUFFERS] = {
316 .audit_skip = 1,
317 .iopoll = 1,
318 .prep = io_provide_buffers_prep,
319 .issue = io_provide_buffers,
320 },
321 [IORING_OP_REMOVE_BUFFERS] = {
322 .audit_skip = 1,
323 .iopoll = 1,
324 .prep = io_remove_buffers_prep,
325 .issue = io_remove_buffers,
326 },
327 [IORING_OP_TEE] = {
328 .needs_file = 1,
329 .hash_reg_file = 1,
330 .unbound_nonreg_file = 1,
331 .audit_skip = 1,
332 .prep = io_tee_prep,
333 .issue = io_tee,
334 },
335 [IORING_OP_SHUTDOWN] = {
336 .needs_file = 1,
337 #if defined(CONFIG_NET)
338 .prep = io_shutdown_prep,
339 .issue = io_shutdown,
340 #else
341 .prep = io_eopnotsupp_prep,
342 #endif
343 },
344 [IORING_OP_RENAMEAT] = {
345 .prep = io_renameat_prep,
346 .issue = io_renameat,
347 },
348 [IORING_OP_UNLINKAT] = {
349 .prep = io_unlinkat_prep,
350 .issue = io_unlinkat,
351 },
352 [IORING_OP_MKDIRAT] = {
353 .prep = io_mkdirat_prep,
354 .issue = io_mkdirat,
355 },
356 [IORING_OP_SYMLINKAT] = {
357 .prep = io_symlinkat_prep,
358 .issue = io_symlinkat,
359 },
360 [IORING_OP_LINKAT] = {
361 .prep = io_linkat_prep,
362 .issue = io_linkat,
363 },
364 [IORING_OP_MSG_RING] = {
365 .needs_file = 1,
366 .iopoll = 1,
367 .prep = io_msg_ring_prep,
368 .issue = io_msg_ring,
369 },
370 [IORING_OP_FSETXATTR] = {
371 .needs_file = 1,
372 .prep = io_fsetxattr_prep,
373 .issue = io_fsetxattr,
374 },
375 [IORING_OP_SETXATTR] = {
376 .prep = io_setxattr_prep,
377 .issue = io_setxattr,
378 },
379 [IORING_OP_FGETXATTR] = {
380 .needs_file = 1,
381 .prep = io_fgetxattr_prep,
382 .issue = io_fgetxattr,
383 },
384 [IORING_OP_GETXATTR] = {
385 .prep = io_getxattr_prep,
386 .issue = io_getxattr,
387 },
388 [IORING_OP_SOCKET] = {
389 .audit_skip = 1,
390 #if defined(CONFIG_NET)
391 .prep = io_socket_prep,
392 .issue = io_socket,
393 #else
394 .prep = io_eopnotsupp_prep,
395 #endif
396 },
397 [IORING_OP_URING_CMD] = {
398 .needs_file = 1,
399 .plug = 1,
400 .iopoll = 1,
401 .iopoll_queue = 1,
402 .prep = io_uring_cmd_prep,
403 .issue = io_uring_cmd,
404 },
405 [IORING_OP_SEND_ZC] = {
406 .needs_file = 1,
407 .unbound_nonreg_file = 1,
408 .pollout = 1,
409 .audit_skip = 1,
410 .ioprio = 1,
411 .manual_alloc = 1,
412 #if defined(CONFIG_NET)
413 .prep = io_send_zc_prep,
414 .issue = io_send_zc,
415 #else
416 .prep = io_eopnotsupp_prep,
417 #endif
418 },
419 [IORING_OP_SENDMSG_ZC] = {
420 .needs_file = 1,
421 .unbound_nonreg_file = 1,
422 .pollout = 1,
423 .ioprio = 1,
424 .manual_alloc = 1,
425 #if defined(CONFIG_NET)
426 .prep = io_send_zc_prep,
427 .issue = io_sendmsg_zc,
428 #else
429 .prep = io_eopnotsupp_prep,
430 #endif
431 },
432 };
433
434
435 const struct io_cold_def io_cold_defs[] = {
436 [IORING_OP_NOP] = {
437 .name = "NOP",
438 },
439 [IORING_OP_READV] = {
440 .async_size = sizeof(struct io_async_rw),
441 .name = "READV",
442 .prep_async = io_readv_prep_async,
443 .cleanup = io_readv_writev_cleanup,
444 .fail = io_rw_fail,
445 },
446 [IORING_OP_WRITEV] = {
447 .async_size = sizeof(struct io_async_rw),
448 .name = "WRITEV",
449 .prep_async = io_writev_prep_async,
450 .cleanup = io_readv_writev_cleanup,
451 .fail = io_rw_fail,
452 },
453 [IORING_OP_FSYNC] = {
454 .name = "FSYNC",
455 },
456 [IORING_OP_READ_FIXED] = {
457 .async_size = sizeof(struct io_async_rw),
458 .name = "READ_FIXED",
459 .fail = io_rw_fail,
460 },
461 [IORING_OP_WRITE_FIXED] = {
462 .async_size = sizeof(struct io_async_rw),
463 .name = "WRITE_FIXED",
464 .fail = io_rw_fail,
465 },
466 [IORING_OP_POLL_ADD] = {
467 .name = "POLL_ADD",
468 },
469 [IORING_OP_POLL_REMOVE] = {
470 .name = "POLL_REMOVE",
471 },
472 [IORING_OP_SYNC_FILE_RANGE] = {
473 .name = "SYNC_FILE_RANGE",
474 },
475 [IORING_OP_SENDMSG] = {
476 .name = "SENDMSG",
477 #if defined(CONFIG_NET)
478 .async_size = sizeof(struct io_async_msghdr),
479 .prep_async = io_sendmsg_prep_async,
480 .cleanup = io_sendmsg_recvmsg_cleanup,
481 .fail = io_sendrecv_fail,
482 #endif
483 },
484 [IORING_OP_RECVMSG] = {
485 .name = "RECVMSG",
486 #if defined(CONFIG_NET)
487 .async_size = sizeof(struct io_async_msghdr),
488 .prep_async = io_recvmsg_prep_async,
489 .cleanup = io_sendmsg_recvmsg_cleanup,
490 .fail = io_sendrecv_fail,
491 #endif
492 },
493 [IORING_OP_TIMEOUT] = {
494 .async_size = sizeof(struct io_timeout_data),
495 .name = "TIMEOUT",
496 },
497 [IORING_OP_TIMEOUT_REMOVE] = {
498 .name = "TIMEOUT_REMOVE",
499 },
500 [IORING_OP_ACCEPT] = {
501 .name = "ACCEPT",
502 },
503 [IORING_OP_ASYNC_CANCEL] = {
504 .name = "ASYNC_CANCEL",
505 },
506 [IORING_OP_LINK_TIMEOUT] = {
507 .async_size = sizeof(struct io_timeout_data),
508 .name = "LINK_TIMEOUT",
509 },
510 [IORING_OP_CONNECT] = {
511 .name = "CONNECT",
512 #if defined(CONFIG_NET)
513 .async_size = sizeof(struct io_async_connect),
514 .prep_async = io_connect_prep_async,
515 #endif
516 },
517 [IORING_OP_FALLOCATE] = {
518 .name = "FALLOCATE",
519 },
520 [IORING_OP_OPENAT] = {
521 .name = "OPENAT",
522 .cleanup = io_open_cleanup,
523 },
524 [IORING_OP_CLOSE] = {
525 .name = "CLOSE",
526 },
527 [IORING_OP_FILES_UPDATE] = {
528 .name = "FILES_UPDATE",
529 },
530 [IORING_OP_STATX] = {
531 .name = "STATX",
532 .cleanup = io_statx_cleanup,
533 },
534 [IORING_OP_READ] = {
535 .async_size = sizeof(struct io_async_rw),
536 .name = "READ",
537 .fail = io_rw_fail,
538 },
539 [IORING_OP_WRITE] = {
540 .async_size = sizeof(struct io_async_rw),
541 .name = "WRITE",
542 .fail = io_rw_fail,
543 },
544 [IORING_OP_FADVISE] = {
545 .name = "FADVISE",
546 },
547 [IORING_OP_MADVISE] = {
548 .name = "MADVISE",
549 },
550 [IORING_OP_SEND] = {
551 .name = "SEND",
552 #if defined(CONFIG_NET)
553 .async_size = sizeof(struct io_async_msghdr),
554 .fail = io_sendrecv_fail,
555 .prep_async = io_send_prep_async,
556 #endif
557 },
558 [IORING_OP_RECV] = {
559 .name = "RECV",
560 #if defined(CONFIG_NET)
561 .fail = io_sendrecv_fail,
562 #endif
563 },
564 [IORING_OP_OPENAT2] = {
565 .name = "OPENAT2",
566 .cleanup = io_open_cleanup,
567 },
568 [IORING_OP_EPOLL_CTL] = {
569 .name = "EPOLL",
570 },
571 [IORING_OP_SPLICE] = {
572 .name = "SPLICE",
573 },
574 [IORING_OP_PROVIDE_BUFFERS] = {
575 .name = "PROVIDE_BUFFERS",
576 },
577 [IORING_OP_REMOVE_BUFFERS] = {
578 .name = "REMOVE_BUFFERS",
579 },
580 [IORING_OP_TEE] = {
581 .name = "TEE",
582 },
583 [IORING_OP_SHUTDOWN] = {
584 .name = "SHUTDOWN",
585 },
586 [IORING_OP_RENAMEAT] = {
587 .name = "RENAMEAT",
588 .cleanup = io_renameat_cleanup,
589 },
590 [IORING_OP_UNLINKAT] = {
591 .name = "UNLINKAT",
592 .cleanup = io_unlinkat_cleanup,
593 },
594 [IORING_OP_MKDIRAT] = {
595 .name = "MKDIRAT",
596 .cleanup = io_mkdirat_cleanup,
597 },
598 [IORING_OP_SYMLINKAT] = {
599 .name = "SYMLINKAT",
600 .cleanup = io_link_cleanup,
601 },
602 [IORING_OP_LINKAT] = {
603 .name = "LINKAT",
604 .cleanup = io_link_cleanup,
605 },
606 [IORING_OP_MSG_RING] = {
607 .name = "MSG_RING",
608 .cleanup = io_msg_ring_cleanup,
609 },
610 [IORING_OP_FSETXATTR] = {
611 .name = "FSETXATTR",
612 .cleanup = io_xattr_cleanup,
613 },
614 [IORING_OP_SETXATTR] = {
615 .name = "SETXATTR",
616 .cleanup = io_xattr_cleanup,
617 },
618 [IORING_OP_FGETXATTR] = {
619 .name = "FGETXATTR",
620 .cleanup = io_xattr_cleanup,
621 },
622 [IORING_OP_GETXATTR] = {
623 .name = "GETXATTR",
624 .cleanup = io_xattr_cleanup,
625 },
626 [IORING_OP_SOCKET] = {
627 .name = "SOCKET",
628 },
629 [IORING_OP_URING_CMD] = {
630 .name = "URING_CMD",
631 .async_size = 2 * sizeof(struct io_uring_sqe),
632 .prep_async = io_uring_cmd_prep_async,
633 },
634 [IORING_OP_SEND_ZC] = {
635 .name = "SEND_ZC",
636 #if defined(CONFIG_NET)
637 .async_size = sizeof(struct io_async_msghdr),
638 .prep_async = io_send_prep_async,
639 .cleanup = io_send_zc_cleanup,
640 .fail = io_sendrecv_fail,
641 #endif
642 },
643 [IORING_OP_SENDMSG_ZC] = {
644 .name = "SENDMSG_ZC",
645 #if defined(CONFIG_NET)
646 .async_size = sizeof(struct io_async_msghdr),
647 .prep_async = io_sendmsg_prep_async,
648 .cleanup = io_send_zc_cleanup,
649 .fail = io_sendrecv_fail,
650 #endif
651 },
652 };
653
io_uring_get_opcode(u8 opcode)654 const char *io_uring_get_opcode(u8 opcode)
655 {
656 if (opcode < IORING_OP_LAST)
657 return io_cold_defs[opcode].name;
658 return "INVALID";
659 }
660
io_uring_optable_init(void)661 void __init io_uring_optable_init(void)
662 {
663 int i;
664
665 BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
666 BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
667
668 for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
669 BUG_ON(!io_issue_defs[i].prep);
670 if (io_issue_defs[i].prep != io_eopnotsupp_prep)
671 BUG_ON(!io_issue_defs[i].issue);
672 WARN_ON_ONCE(!io_cold_defs[i].name);
673 }
674 }
675