io_uring: get rid of double locking
[platform/kernel/linux-starfive.git] / io_uring / opdef.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * io_uring opcode handling table
4  */
5 #include <linux/kernel.h>
6 #include <linux/errno.h>
7 #include <linux/fs.h>
8 #include <linux/file.h>
9 #include <linux/io_uring.h>
10
11 #include "io_uring.h"
12 #include "opdef.h"
13 #include "refs.h"
14 #include "tctx.h"
15 #include "sqpoll.h"
16 #include "fdinfo.h"
17 #include "kbuf.h"
18 #include "rsrc.h"
19
20 #include "xattr.h"
21 #include "nop.h"
22 #include "fs.h"
23 #include "splice.h"
24 #include "sync.h"
25 #include "advise.h"
26 #include "openclose.h"
27 #include "uring_cmd.h"
28 #include "epoll.h"
29 #include "statx.h"
30 #include "net.h"
31 #include "msg_ring.h"
32 #include "timeout.h"
33 #include "poll.h"
34 #include "cancel.h"
35 #include "rw.h"
36
37 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
38 {
39         WARN_ON_ONCE(1);
40         return -ECANCELED;
41 }
42
43 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
44                                              const struct io_uring_sqe *sqe)
45 {
46         return -EOPNOTSUPP;
47 }
48
49 const struct io_op_def io_op_defs[] = {
50         [IORING_OP_NOP] = {
51                 .audit_skip             = 1,
52                 .iopoll                 = 1,
53                 .name                   = "NOP",
54                 .prep                   = io_nop_prep,
55                 .issue                  = io_nop,
56         },
57         [IORING_OP_READV] = {
58                 .needs_file             = 1,
59                 .unbound_nonreg_file    = 1,
60                 .pollin                 = 1,
61                 .buffer_select          = 1,
62                 .plug                   = 1,
63                 .audit_skip             = 1,
64                 .ioprio                 = 1,
65                 .iopoll                 = 1,
66                 .iopoll_queue           = 1,
67                 .async_size             = sizeof(struct io_async_rw),
68                 .name                   = "READV",
69                 .prep                   = io_prep_rw,
70                 .issue                  = io_read,
71                 .prep_async             = io_readv_prep_async,
72                 .cleanup                = io_readv_writev_cleanup,
73                 .fail                   = io_rw_fail,
74         },
75         [IORING_OP_WRITEV] = {
76                 .needs_file             = 1,
77                 .hash_reg_file          = 1,
78                 .unbound_nonreg_file    = 1,
79                 .pollout                = 1,
80                 .plug                   = 1,
81                 .audit_skip             = 1,
82                 .ioprio                 = 1,
83                 .iopoll                 = 1,
84                 .iopoll_queue           = 1,
85                 .async_size             = sizeof(struct io_async_rw),
86                 .name                   = "WRITEV",
87                 .prep                   = io_prep_rw,
88                 .issue                  = io_write,
89                 .prep_async             = io_writev_prep_async,
90                 .cleanup                = io_readv_writev_cleanup,
91                 .fail                   = io_rw_fail,
92         },
93         [IORING_OP_FSYNC] = {
94                 .needs_file             = 1,
95                 .audit_skip             = 1,
96                 .name                   = "FSYNC",
97                 .prep                   = io_fsync_prep,
98                 .issue                  = io_fsync,
99         },
100         [IORING_OP_READ_FIXED] = {
101                 .needs_file             = 1,
102                 .unbound_nonreg_file    = 1,
103                 .pollin                 = 1,
104                 .plug                   = 1,
105                 .audit_skip             = 1,
106                 .ioprio                 = 1,
107                 .iopoll                 = 1,
108                 .iopoll_queue           = 1,
109                 .async_size             = sizeof(struct io_async_rw),
110                 .name                   = "READ_FIXED",
111                 .prep                   = io_prep_rw,
112                 .issue                  = io_read,
113                 .fail                   = io_rw_fail,
114         },
115         [IORING_OP_WRITE_FIXED] = {
116                 .needs_file             = 1,
117                 .hash_reg_file          = 1,
118                 .unbound_nonreg_file    = 1,
119                 .pollout                = 1,
120                 .plug                   = 1,
121                 .audit_skip             = 1,
122                 .ioprio                 = 1,
123                 .iopoll                 = 1,
124                 .iopoll_queue           = 1,
125                 .async_size             = sizeof(struct io_async_rw),
126                 .name                   = "WRITE_FIXED",
127                 .prep                   = io_prep_rw,
128                 .issue                  = io_write,
129                 .fail                   = io_rw_fail,
130         },
131         [IORING_OP_POLL_ADD] = {
132                 .needs_file             = 1,
133                 .unbound_nonreg_file    = 1,
134                 .audit_skip             = 1,
135                 .name                   = "POLL_ADD",
136                 .prep                   = io_poll_add_prep,
137                 .issue                  = io_poll_add,
138         },
139         [IORING_OP_POLL_REMOVE] = {
140                 .audit_skip             = 1,
141                 .name                   = "POLL_REMOVE",
142                 .prep                   = io_poll_remove_prep,
143                 .issue                  = io_poll_remove,
144         },
145         [IORING_OP_SYNC_FILE_RANGE] = {
146                 .needs_file             = 1,
147                 .audit_skip             = 1,
148                 .name                   = "SYNC_FILE_RANGE",
149                 .prep                   = io_sfr_prep,
150                 .issue                  = io_sync_file_range,
151         },
152         [IORING_OP_SENDMSG] = {
153                 .needs_file             = 1,
154                 .unbound_nonreg_file    = 1,
155                 .pollout                = 1,
156                 .ioprio                 = 1,
157                 .manual_alloc           = 1,
158                 .name                   = "SENDMSG",
159 #if defined(CONFIG_NET)
160                 .async_size             = sizeof(struct io_async_msghdr),
161                 .prep                   = io_sendmsg_prep,
162                 .issue                  = io_sendmsg,
163                 .prep_async             = io_sendmsg_prep_async,
164                 .cleanup                = io_sendmsg_recvmsg_cleanup,
165                 .fail                   = io_sendrecv_fail,
166 #else
167                 .prep                   = io_eopnotsupp_prep,
168 #endif
169         },
170         [IORING_OP_RECVMSG] = {
171                 .needs_file             = 1,
172                 .unbound_nonreg_file    = 1,
173                 .pollin                 = 1,
174                 .buffer_select          = 1,
175                 .ioprio                 = 1,
176                 .manual_alloc           = 1,
177                 .name                   = "RECVMSG",
178 #if defined(CONFIG_NET)
179                 .async_size             = sizeof(struct io_async_msghdr),
180                 .prep                   = io_recvmsg_prep,
181                 .issue                  = io_recvmsg,
182                 .prep_async             = io_recvmsg_prep_async,
183                 .cleanup                = io_sendmsg_recvmsg_cleanup,
184                 .fail                   = io_sendrecv_fail,
185 #else
186                 .prep                   = io_eopnotsupp_prep,
187 #endif
188         },
189         [IORING_OP_TIMEOUT] = {
190                 .audit_skip             = 1,
191                 .async_size             = sizeof(struct io_timeout_data),
192                 .name                   = "TIMEOUT",
193                 .prep                   = io_timeout_prep,
194                 .issue                  = io_timeout,
195         },
196         [IORING_OP_TIMEOUT_REMOVE] = {
197                 /* used by timeout updates' prep() */
198                 .audit_skip             = 1,
199                 .name                   = "TIMEOUT_REMOVE",
200                 .prep                   = io_timeout_remove_prep,
201                 .issue                  = io_timeout_remove,
202         },
203         [IORING_OP_ACCEPT] = {
204                 .needs_file             = 1,
205                 .unbound_nonreg_file    = 1,
206                 .pollin                 = 1,
207                 .poll_exclusive         = 1,
208                 .ioprio                 = 1,    /* used for flags */
209                 .name                   = "ACCEPT",
210 #if defined(CONFIG_NET)
211                 .prep                   = io_accept_prep,
212                 .issue                  = io_accept,
213 #else
214                 .prep                   = io_eopnotsupp_prep,
215 #endif
216         },
217         [IORING_OP_ASYNC_CANCEL] = {
218                 .audit_skip             = 1,
219                 .name                   = "ASYNC_CANCEL",
220                 .prep                   = io_async_cancel_prep,
221                 .issue                  = io_async_cancel,
222         },
223         [IORING_OP_LINK_TIMEOUT] = {
224                 .audit_skip             = 1,
225                 .async_size             = sizeof(struct io_timeout_data),
226                 .name                   = "LINK_TIMEOUT",
227                 .prep                   = io_link_timeout_prep,
228                 .issue                  = io_no_issue,
229         },
230         [IORING_OP_CONNECT] = {
231                 .needs_file             = 1,
232                 .unbound_nonreg_file    = 1,
233                 .pollout                = 1,
234                 .name                   = "CONNECT",
235 #if defined(CONFIG_NET)
236                 .async_size             = sizeof(struct io_async_connect),
237                 .prep                   = io_connect_prep,
238                 .issue                  = io_connect,
239                 .prep_async             = io_connect_prep_async,
240 #else
241                 .prep                   = io_eopnotsupp_prep,
242 #endif
243         },
244         [IORING_OP_FALLOCATE] = {
245                 .needs_file             = 1,
246                 .name                   = "FALLOCATE",
247                 .prep                   = io_fallocate_prep,
248                 .issue                  = io_fallocate,
249         },
250         [IORING_OP_OPENAT] = {
251                 .name                   = "OPENAT",
252                 .prep                   = io_openat_prep,
253                 .issue                  = io_openat,
254                 .cleanup                = io_open_cleanup,
255         },
256         [IORING_OP_CLOSE] = {
257                 .name                   = "CLOSE",
258                 .prep                   = io_close_prep,
259                 .issue                  = io_close,
260         },
261         [IORING_OP_FILES_UPDATE] = {
262                 .audit_skip             = 1,
263                 .iopoll                 = 1,
264                 .name                   = "FILES_UPDATE",
265                 .prep                   = io_files_update_prep,
266                 .issue                  = io_files_update,
267         },
268         [IORING_OP_STATX] = {
269                 .audit_skip             = 1,
270                 .name                   = "STATX",
271                 .prep                   = io_statx_prep,
272                 .issue                  = io_statx,
273                 .cleanup                = io_statx_cleanup,
274         },
275         [IORING_OP_READ] = {
276                 .needs_file             = 1,
277                 .unbound_nonreg_file    = 1,
278                 .pollin                 = 1,
279                 .buffer_select          = 1,
280                 .plug                   = 1,
281                 .audit_skip             = 1,
282                 .ioprio                 = 1,
283                 .iopoll                 = 1,
284                 .iopoll_queue           = 1,
285                 .async_size             = sizeof(struct io_async_rw),
286                 .name                   = "READ",
287                 .prep                   = io_prep_rw,
288                 .issue                  = io_read,
289                 .fail                   = io_rw_fail,
290         },
291         [IORING_OP_WRITE] = {
292                 .needs_file             = 1,
293                 .hash_reg_file          = 1,
294                 .unbound_nonreg_file    = 1,
295                 .pollout                = 1,
296                 .plug                   = 1,
297                 .audit_skip             = 1,
298                 .ioprio                 = 1,
299                 .iopoll                 = 1,
300                 .iopoll_queue           = 1,
301                 .async_size             = sizeof(struct io_async_rw),
302                 .name                   = "WRITE",
303                 .prep                   = io_prep_rw,
304                 .issue                  = io_write,
305                 .fail                   = io_rw_fail,
306         },
307         [IORING_OP_FADVISE] = {
308                 .needs_file             = 1,
309                 .audit_skip             = 1,
310                 .name                   = "FADVISE",
311                 .prep                   = io_fadvise_prep,
312                 .issue                  = io_fadvise,
313         },
314         [IORING_OP_MADVISE] = {
315                 .name                   = "MADVISE",
316                 .prep                   = io_madvise_prep,
317                 .issue                  = io_madvise,
318         },
319         [IORING_OP_SEND] = {
320                 .needs_file             = 1,
321                 .unbound_nonreg_file    = 1,
322                 .pollout                = 1,
323                 .audit_skip             = 1,
324                 .ioprio                 = 1,
325                 .manual_alloc           = 1,
326                 .name                   = "SEND",
327 #if defined(CONFIG_NET)
328                 .async_size             = sizeof(struct io_async_msghdr),
329                 .prep                   = io_sendmsg_prep,
330                 .issue                  = io_send,
331                 .fail                   = io_sendrecv_fail,
332                 .prep_async             = io_send_prep_async,
333 #else
334                 .prep                   = io_eopnotsupp_prep,
335 #endif
336         },
337         [IORING_OP_RECV] = {
338                 .needs_file             = 1,
339                 .unbound_nonreg_file    = 1,
340                 .pollin                 = 1,
341                 .buffer_select          = 1,
342                 .audit_skip             = 1,
343                 .ioprio                 = 1,
344                 .name                   = "RECV",
345 #if defined(CONFIG_NET)
346                 .prep                   = io_recvmsg_prep,
347                 .issue                  = io_recv,
348                 .fail                   = io_sendrecv_fail,
349 #else
350                 .prep                   = io_eopnotsupp_prep,
351 #endif
352         },
353         [IORING_OP_OPENAT2] = {
354                 .name                   = "OPENAT2",
355                 .prep                   = io_openat2_prep,
356                 .issue                  = io_openat2,
357                 .cleanup                = io_open_cleanup,
358         },
359         [IORING_OP_EPOLL_CTL] = {
360                 .unbound_nonreg_file    = 1,
361                 .audit_skip             = 1,
362                 .name                   = "EPOLL",
363 #if defined(CONFIG_EPOLL)
364                 .prep                   = io_epoll_ctl_prep,
365                 .issue                  = io_epoll_ctl,
366 #else
367                 .prep                   = io_eopnotsupp_prep,
368 #endif
369         },
370         [IORING_OP_SPLICE] = {
371                 .needs_file             = 1,
372                 .hash_reg_file          = 1,
373                 .unbound_nonreg_file    = 1,
374                 .audit_skip             = 1,
375                 .name                   = "SPLICE",
376                 .prep                   = io_splice_prep,
377                 .issue                  = io_splice,
378         },
379         [IORING_OP_PROVIDE_BUFFERS] = {
380                 .audit_skip             = 1,
381                 .iopoll                 = 1,
382                 .name                   = "PROVIDE_BUFFERS",
383                 .prep                   = io_provide_buffers_prep,
384                 .issue                  = io_provide_buffers,
385         },
386         [IORING_OP_REMOVE_BUFFERS] = {
387                 .audit_skip             = 1,
388                 .iopoll                 = 1,
389                 .name                   = "REMOVE_BUFFERS",
390                 .prep                   = io_remove_buffers_prep,
391                 .issue                  = io_remove_buffers,
392         },
393         [IORING_OP_TEE] = {
394                 .needs_file             = 1,
395                 .hash_reg_file          = 1,
396                 .unbound_nonreg_file    = 1,
397                 .audit_skip             = 1,
398                 .name                   = "TEE",
399                 .prep                   = io_tee_prep,
400                 .issue                  = io_tee,
401         },
402         [IORING_OP_SHUTDOWN] = {
403                 .needs_file             = 1,
404                 .name                   = "SHUTDOWN",
405 #if defined(CONFIG_NET)
406                 .prep                   = io_shutdown_prep,
407                 .issue                  = io_shutdown,
408 #else
409                 .prep                   = io_eopnotsupp_prep,
410 #endif
411         },
412         [IORING_OP_RENAMEAT] = {
413                 .name                   = "RENAMEAT",
414                 .prep                   = io_renameat_prep,
415                 .issue                  = io_renameat,
416                 .cleanup                = io_renameat_cleanup,
417         },
418         [IORING_OP_UNLINKAT] = {
419                 .name                   = "UNLINKAT",
420                 .prep                   = io_unlinkat_prep,
421                 .issue                  = io_unlinkat,
422                 .cleanup                = io_unlinkat_cleanup,
423         },
424         [IORING_OP_MKDIRAT] = {
425                 .name                   = "MKDIRAT",
426                 .prep                   = io_mkdirat_prep,
427                 .issue                  = io_mkdirat,
428                 .cleanup                = io_mkdirat_cleanup,
429         },
430         [IORING_OP_SYMLINKAT] = {
431                 .name                   = "SYMLINKAT",
432                 .prep                   = io_symlinkat_prep,
433                 .issue                  = io_symlinkat,
434                 .cleanup                = io_link_cleanup,
435         },
436         [IORING_OP_LINKAT] = {
437                 .name                   = "LINKAT",
438                 .prep                   = io_linkat_prep,
439                 .issue                  = io_linkat,
440                 .cleanup                = io_link_cleanup,
441         },
442         [IORING_OP_MSG_RING] = {
443                 .needs_file             = 1,
444                 .iopoll                 = 1,
445                 .name                   = "MSG_RING",
446                 .prep                   = io_msg_ring_prep,
447                 .issue                  = io_msg_ring,
448                 .cleanup                = io_msg_ring_cleanup,
449         },
450         [IORING_OP_FSETXATTR] = {
451                 .needs_file = 1,
452                 .name                   = "FSETXATTR",
453                 .prep                   = io_fsetxattr_prep,
454                 .issue                  = io_fsetxattr,
455                 .cleanup                = io_xattr_cleanup,
456         },
457         [IORING_OP_SETXATTR] = {
458                 .name                   = "SETXATTR",
459                 .prep                   = io_setxattr_prep,
460                 .issue                  = io_setxattr,
461                 .cleanup                = io_xattr_cleanup,
462         },
463         [IORING_OP_FGETXATTR] = {
464                 .needs_file = 1,
465                 .name                   = "FGETXATTR",
466                 .prep                   = io_fgetxattr_prep,
467                 .issue                  = io_fgetxattr,
468                 .cleanup                = io_xattr_cleanup,
469         },
470         [IORING_OP_GETXATTR] = {
471                 .name                   = "GETXATTR",
472                 .prep                   = io_getxattr_prep,
473                 .issue                  = io_getxattr,
474                 .cleanup                = io_xattr_cleanup,
475         },
476         [IORING_OP_SOCKET] = {
477                 .audit_skip             = 1,
478                 .name                   = "SOCKET",
479 #if defined(CONFIG_NET)
480                 .prep                   = io_socket_prep,
481                 .issue                  = io_socket,
482 #else
483                 .prep                   = io_eopnotsupp_prep,
484 #endif
485         },
486         [IORING_OP_URING_CMD] = {
487                 .needs_file             = 1,
488                 .plug                   = 1,
489                 .name                   = "URING_CMD",
490                 .iopoll                 = 1,
491                 .iopoll_queue           = 1,
492                 .async_size             = uring_cmd_pdu_size(1),
493                 .prep                   = io_uring_cmd_prep,
494                 .issue                  = io_uring_cmd,
495                 .prep_async             = io_uring_cmd_prep_async,
496         },
497         [IORING_OP_SEND_ZC] = {
498                 .name                   = "SEND_ZC",
499                 .needs_file             = 1,
500                 .unbound_nonreg_file    = 1,
501                 .pollout                = 1,
502                 .audit_skip             = 1,
503                 .ioprio                 = 1,
504                 .manual_alloc           = 1,
505 #if defined(CONFIG_NET)
506                 .async_size             = sizeof(struct io_async_msghdr),
507                 .prep                   = io_send_zc_prep,
508                 .issue                  = io_send_zc,
509                 .prep_async             = io_send_prep_async,
510                 .cleanup                = io_send_zc_cleanup,
511                 .fail                   = io_sendrecv_fail,
512 #else
513                 .prep                   = io_eopnotsupp_prep,
514 #endif
515         },
516         [IORING_OP_SENDMSG_ZC] = {
517                 .name                   = "SENDMSG_ZC",
518                 .needs_file             = 1,
519                 .unbound_nonreg_file    = 1,
520                 .pollout                = 1,
521                 .ioprio                 = 1,
522                 .manual_alloc           = 1,
523 #if defined(CONFIG_NET)
524                 .async_size             = sizeof(struct io_async_msghdr),
525                 .prep                   = io_send_zc_prep,
526                 .issue                  = io_sendmsg_zc,
527                 .prep_async             = io_sendmsg_prep_async,
528                 .cleanup                = io_send_zc_cleanup,
529                 .fail                   = io_sendrecv_fail,
530 #else
531                 .prep                   = io_eopnotsupp_prep,
532 #endif
533         },
534 };
535
536 const char *io_uring_get_opcode(u8 opcode)
537 {
538         if (opcode < IORING_OP_LAST)
539                 return io_op_defs[opcode].name;
540         return "INVALID";
541 }
542
543 void __init io_uring_optable_init(void)
544 {
545         int i;
546
547         BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
548
549         for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) {
550                 BUG_ON(!io_op_defs[i].prep);
551                 if (io_op_defs[i].prep != io_eopnotsupp_prep)
552                         BUG_ON(!io_op_defs[i].issue);
553                 WARN_ON_ONCE(!io_op_defs[i].name);
554         }
555 }