Merge tag 'rpmsg-v5.20' of git://git.kernel.org/pub/scm/linux/kernel/git/remoteproc...
[platform/kernel/linux-rpi.git] / io_uring / opdef.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * io_uring opcode handling table
4  */
5 #include <linux/kernel.h>
6 #include <linux/errno.h>
7 #include <linux/fs.h>
8 #include <linux/file.h>
9 #include <linux/io_uring.h>
10
11 #include "io_uring.h"
12 #include "opdef.h"
13 #include "refs.h"
14 #include "tctx.h"
15 #include "sqpoll.h"
16 #include "fdinfo.h"
17 #include "kbuf.h"
18 #include "rsrc.h"
19
20 #include "xattr.h"
21 #include "nop.h"
22 #include "fs.h"
23 #include "splice.h"
24 #include "sync.h"
25 #include "advise.h"
26 #include "openclose.h"
27 #include "uring_cmd.h"
28 #include "epoll.h"
29 #include "statx.h"
30 #include "net.h"
31 #include "msg_ring.h"
32 #include "timeout.h"
33 #include "poll.h"
34 #include "cancel.h"
35 #include "rw.h"
36
37 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
38 {
39         WARN_ON_ONCE(1);
40         return -ECANCELED;
41 }
42
43 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
44                                              const struct io_uring_sqe *sqe)
45 {
46         return -EOPNOTSUPP;
47 }
48
49 const struct io_op_def io_op_defs[] = {
50         [IORING_OP_NOP] = {
51                 .audit_skip             = 1,
52                 .iopoll                 = 1,
53                 .name                   = "NOP",
54                 .prep                   = io_nop_prep,
55                 .issue                  = io_nop,
56         },
57         [IORING_OP_READV] = {
58                 .needs_file             = 1,
59                 .unbound_nonreg_file    = 1,
60                 .pollin                 = 1,
61                 .buffer_select          = 1,
62                 .plug                   = 1,
63                 .audit_skip             = 1,
64                 .ioprio                 = 1,
65                 .iopoll                 = 1,
66                 .async_size             = sizeof(struct io_async_rw),
67                 .name                   = "READV",
68                 .prep                   = io_prep_rw,
69                 .issue                  = io_read,
70                 .prep_async             = io_readv_prep_async,
71                 .cleanup                = io_readv_writev_cleanup,
72         },
73         [IORING_OP_WRITEV] = {
74                 .needs_file             = 1,
75                 .hash_reg_file          = 1,
76                 .unbound_nonreg_file    = 1,
77                 .pollout                = 1,
78                 .plug                   = 1,
79                 .audit_skip             = 1,
80                 .ioprio                 = 1,
81                 .iopoll                 = 1,
82                 .async_size             = sizeof(struct io_async_rw),
83                 .name                   = "WRITEV",
84                 .prep                   = io_prep_rw,
85                 .issue                  = io_write,
86                 .prep_async             = io_writev_prep_async,
87                 .cleanup                = io_readv_writev_cleanup,
88         },
89         [IORING_OP_FSYNC] = {
90                 .needs_file             = 1,
91                 .audit_skip             = 1,
92                 .name                   = "FSYNC",
93                 .prep                   = io_fsync_prep,
94                 .issue                  = io_fsync,
95         },
96         [IORING_OP_READ_FIXED] = {
97                 .needs_file             = 1,
98                 .unbound_nonreg_file    = 1,
99                 .pollin                 = 1,
100                 .plug                   = 1,
101                 .audit_skip             = 1,
102                 .ioprio                 = 1,
103                 .iopoll                 = 1,
104                 .async_size             = sizeof(struct io_async_rw),
105                 .name                   = "READ_FIXED",
106                 .prep                   = io_prep_rw,
107                 .issue                  = io_read,
108         },
109         [IORING_OP_WRITE_FIXED] = {
110                 .needs_file             = 1,
111                 .hash_reg_file          = 1,
112                 .unbound_nonreg_file    = 1,
113                 .pollout                = 1,
114                 .plug                   = 1,
115                 .audit_skip             = 1,
116                 .ioprio                 = 1,
117                 .iopoll                 = 1,
118                 .async_size             = sizeof(struct io_async_rw),
119                 .name                   = "WRITE_FIXED",
120                 .prep                   = io_prep_rw,
121                 .issue                  = io_write,
122         },
123         [IORING_OP_POLL_ADD] = {
124                 .needs_file             = 1,
125                 .unbound_nonreg_file    = 1,
126                 .audit_skip             = 1,
127                 .name                   = "POLL_ADD",
128                 .prep                   = io_poll_add_prep,
129                 .issue                  = io_poll_add,
130         },
131         [IORING_OP_POLL_REMOVE] = {
132                 .audit_skip             = 1,
133                 .name                   = "POLL_REMOVE",
134                 .prep                   = io_poll_remove_prep,
135                 .issue                  = io_poll_remove,
136         },
137         [IORING_OP_SYNC_FILE_RANGE] = {
138                 .needs_file             = 1,
139                 .audit_skip             = 1,
140                 .name                   = "SYNC_FILE_RANGE",
141                 .prep                   = io_sfr_prep,
142                 .issue                  = io_sync_file_range,
143         },
144         [IORING_OP_SENDMSG] = {
145                 .needs_file             = 1,
146                 .unbound_nonreg_file    = 1,
147                 .pollout                = 1,
148                 .ioprio                 = 1,
149                 .name                   = "SENDMSG",
150 #if defined(CONFIG_NET)
151                 .async_size             = sizeof(struct io_async_msghdr),
152                 .prep                   = io_sendmsg_prep,
153                 .issue                  = io_sendmsg,
154                 .prep_async             = io_sendmsg_prep_async,
155                 .cleanup                = io_sendmsg_recvmsg_cleanup,
156 #else
157                 .prep                   = io_eopnotsupp_prep,
158 #endif
159         },
160         [IORING_OP_RECVMSG] = {
161                 .needs_file             = 1,
162                 .unbound_nonreg_file    = 1,
163                 .pollin                 = 1,
164                 .buffer_select          = 1,
165                 .ioprio                 = 1,
166                 .name                   = "RECVMSG",
167 #if defined(CONFIG_NET)
168                 .async_size             = sizeof(struct io_async_msghdr),
169                 .prep                   = io_recvmsg_prep,
170                 .issue                  = io_recvmsg,
171                 .prep_async             = io_recvmsg_prep_async,
172                 .cleanup                = io_sendmsg_recvmsg_cleanup,
173 #else
174                 .prep                   = io_eopnotsupp_prep,
175 #endif
176         },
177         [IORING_OP_TIMEOUT] = {
178                 .audit_skip             = 1,
179                 .async_size             = sizeof(struct io_timeout_data),
180                 .name                   = "TIMEOUT",
181                 .prep                   = io_timeout_prep,
182                 .issue                  = io_timeout,
183         },
184         [IORING_OP_TIMEOUT_REMOVE] = {
185                 /* used by timeout updates' prep() */
186                 .audit_skip             = 1,
187                 .name                   = "TIMEOUT_REMOVE",
188                 .prep                   = io_timeout_remove_prep,
189                 .issue                  = io_timeout_remove,
190         },
191         [IORING_OP_ACCEPT] = {
192                 .needs_file             = 1,
193                 .unbound_nonreg_file    = 1,
194                 .pollin                 = 1,
195                 .poll_exclusive         = 1,
196                 .ioprio                 = 1,    /* used for flags */
197                 .name                   = "ACCEPT",
198 #if defined(CONFIG_NET)
199                 .prep                   = io_accept_prep,
200                 .issue                  = io_accept,
201 #else
202                 .prep                   = io_eopnotsupp_prep,
203 #endif
204         },
205         [IORING_OP_ASYNC_CANCEL] = {
206                 .audit_skip             = 1,
207                 .name                   = "ASYNC_CANCEL",
208                 .prep                   = io_async_cancel_prep,
209                 .issue                  = io_async_cancel,
210         },
211         [IORING_OP_LINK_TIMEOUT] = {
212                 .audit_skip             = 1,
213                 .async_size             = sizeof(struct io_timeout_data),
214                 .name                   = "LINK_TIMEOUT",
215                 .prep                   = io_link_timeout_prep,
216                 .issue                  = io_no_issue,
217         },
218         [IORING_OP_CONNECT] = {
219                 .needs_file             = 1,
220                 .unbound_nonreg_file    = 1,
221                 .pollout                = 1,
222                 .name                   = "CONNECT",
223 #if defined(CONFIG_NET)
224                 .async_size             = sizeof(struct io_async_connect),
225                 .prep                   = io_connect_prep,
226                 .issue                  = io_connect,
227                 .prep_async             = io_connect_prep_async,
228 #else
229                 .prep                   = io_eopnotsupp_prep,
230 #endif
231         },
232         [IORING_OP_FALLOCATE] = {
233                 .needs_file             = 1,
234                 .name                   = "FALLOCATE",
235                 .prep                   = io_fallocate_prep,
236                 .issue                  = io_fallocate,
237         },
238         [IORING_OP_OPENAT] = {
239                 .name                   = "OPENAT",
240                 .prep                   = io_openat_prep,
241                 .issue                  = io_openat,
242                 .cleanup                = io_open_cleanup,
243         },
244         [IORING_OP_CLOSE] = {
245                 .name                   = "CLOSE",
246                 .prep                   = io_close_prep,
247                 .issue                  = io_close,
248         },
249         [IORING_OP_RSRC_UPDATE] = {
250                 .audit_skip             = 1,
251                 .iopoll                 = 1,
252                 .name                   = "RSRC_UPDATE",
253                 .prep                   = io_rsrc_update_prep,
254                 .issue                  = io_rsrc_update,
255                 .ioprio                 = 1,
256         },
257         [IORING_OP_STATX] = {
258                 .audit_skip             = 1,
259                 .name                   = "STATX",
260                 .prep                   = io_statx_prep,
261                 .issue                  = io_statx,
262                 .cleanup                = io_statx_cleanup,
263         },
264         [IORING_OP_READ] = {
265                 .needs_file             = 1,
266                 .unbound_nonreg_file    = 1,
267                 .pollin                 = 1,
268                 .buffer_select          = 1,
269                 .plug                   = 1,
270                 .audit_skip             = 1,
271                 .ioprio                 = 1,
272                 .iopoll                 = 1,
273                 .async_size             = sizeof(struct io_async_rw),
274                 .name                   = "READ",
275                 .prep                   = io_prep_rw,
276                 .issue                  = io_read,
277         },
278         [IORING_OP_WRITE] = {
279                 .needs_file             = 1,
280                 .hash_reg_file          = 1,
281                 .unbound_nonreg_file    = 1,
282                 .pollout                = 1,
283                 .plug                   = 1,
284                 .audit_skip             = 1,
285                 .ioprio                 = 1,
286                 .iopoll                 = 1,
287                 .async_size             = sizeof(struct io_async_rw),
288                 .name                   = "WRITE",
289                 .prep                   = io_prep_rw,
290                 .issue                  = io_write,
291         },
292         [IORING_OP_FADVISE] = {
293                 .needs_file             = 1,
294                 .audit_skip             = 1,
295                 .name                   = "FADVISE",
296                 .prep                   = io_fadvise_prep,
297                 .issue                  = io_fadvise,
298         },
299         [IORING_OP_MADVISE] = {
300                 .name                   = "MADVISE",
301                 .prep                   = io_madvise_prep,
302                 .issue                  = io_madvise,
303         },
304         [IORING_OP_SEND] = {
305                 .needs_file             = 1,
306                 .unbound_nonreg_file    = 1,
307                 .pollout                = 1,
308                 .audit_skip             = 1,
309                 .ioprio                 = 1,
310                 .name                   = "SEND",
311 #if defined(CONFIG_NET)
312                 .prep                   = io_sendmsg_prep,
313                 .issue                  = io_send,
314 #else
315                 .prep                   = io_eopnotsupp_prep,
316 #endif
317         },
318         [IORING_OP_RECV] = {
319                 .needs_file             = 1,
320                 .unbound_nonreg_file    = 1,
321                 .pollin                 = 1,
322                 .buffer_select          = 1,
323                 .audit_skip             = 1,
324                 .ioprio                 = 1,
325                 .name                   = "RECV",
326 #if defined(CONFIG_NET)
327                 .prep                   = io_recvmsg_prep,
328                 .issue                  = io_recv,
329 #else
330                 .prep                   = io_eopnotsupp_prep,
331 #endif
332         },
333         [IORING_OP_OPENAT2] = {
334                 .name                   = "OPENAT2",
335                 .prep                   = io_openat2_prep,
336                 .issue                  = io_openat2,
337                 .cleanup                = io_open_cleanup,
338         },
339         [IORING_OP_EPOLL_CTL] = {
340                 .unbound_nonreg_file    = 1,
341                 .audit_skip             = 1,
342                 .name                   = "EPOLL",
343 #if defined(CONFIG_EPOLL)
344                 .prep                   = io_epoll_ctl_prep,
345                 .issue                  = io_epoll_ctl,
346 #else
347                 .prep                   = io_eopnotsupp_prep,
348 #endif
349         },
350         [IORING_OP_SPLICE] = {
351                 .needs_file             = 1,
352                 .hash_reg_file          = 1,
353                 .unbound_nonreg_file    = 1,
354                 .audit_skip             = 1,
355                 .name                   = "SPLICE",
356                 .prep                   = io_splice_prep,
357                 .issue                  = io_splice,
358         },
359         [IORING_OP_PROVIDE_BUFFERS] = {
360                 .audit_skip             = 1,
361                 .iopoll                 = 1,
362                 .name                   = "PROVIDE_BUFFERS",
363                 .prep                   = io_provide_buffers_prep,
364                 .issue                  = io_provide_buffers,
365         },
366         [IORING_OP_REMOVE_BUFFERS] = {
367                 .audit_skip             = 1,
368                 .iopoll                 = 1,
369                 .name                   = "REMOVE_BUFFERS",
370                 .prep                   = io_remove_buffers_prep,
371                 .issue                  = io_remove_buffers,
372         },
373         [IORING_OP_TEE] = {
374                 .needs_file             = 1,
375                 .hash_reg_file          = 1,
376                 .unbound_nonreg_file    = 1,
377                 .audit_skip             = 1,
378                 .name                   = "TEE",
379                 .prep                   = io_tee_prep,
380                 .issue                  = io_tee,
381         },
382         [IORING_OP_SHUTDOWN] = {
383                 .needs_file             = 1,
384                 .name                   = "SHUTDOWN",
385 #if defined(CONFIG_NET)
386                 .prep                   = io_shutdown_prep,
387                 .issue                  = io_shutdown,
388 #else
389                 .prep                   = io_eopnotsupp_prep,
390 #endif
391         },
392         [IORING_OP_RENAMEAT] = {
393                 .name                   = "RENAMEAT",
394                 .prep                   = io_renameat_prep,
395                 .issue                  = io_renameat,
396                 .cleanup                = io_renameat_cleanup,
397         },
398         [IORING_OP_UNLINKAT] = {
399                 .name                   = "UNLINKAT",
400                 .prep                   = io_unlinkat_prep,
401                 .issue                  = io_unlinkat,
402                 .cleanup                = io_unlinkat_cleanup,
403         },
404         [IORING_OP_MKDIRAT] = {
405                 .name                   = "MKDIRAT",
406                 .prep                   = io_mkdirat_prep,
407                 .issue                  = io_mkdirat,
408                 .cleanup                = io_mkdirat_cleanup,
409         },
410         [IORING_OP_SYMLINKAT] = {
411                 .name                   = "SYMLINKAT",
412                 .prep                   = io_symlinkat_prep,
413                 .issue                  = io_symlinkat,
414                 .cleanup                = io_link_cleanup,
415         },
416         [IORING_OP_LINKAT] = {
417                 .name                   = "LINKAT",
418                 .prep                   = io_linkat_prep,
419                 .issue                  = io_linkat,
420                 .cleanup                = io_link_cleanup,
421         },
422         [IORING_OP_MSG_RING] = {
423                 .needs_file             = 1,
424                 .iopoll                 = 1,
425                 .name                   = "MSG_RING",
426                 .prep                   = io_msg_ring_prep,
427                 .issue                  = io_msg_ring,
428         },
429         [IORING_OP_FSETXATTR] = {
430                 .needs_file = 1,
431                 .name                   = "FSETXATTR",
432                 .prep                   = io_fsetxattr_prep,
433                 .issue                  = io_fsetxattr,
434                 .cleanup                = io_xattr_cleanup,
435         },
436         [IORING_OP_SETXATTR] = {
437                 .name                   = "SETXATTR",
438                 .prep                   = io_setxattr_prep,
439                 .issue                  = io_setxattr,
440                 .cleanup                = io_xattr_cleanup,
441         },
442         [IORING_OP_FGETXATTR] = {
443                 .needs_file = 1,
444                 .name                   = "FGETXATTR",
445                 .prep                   = io_fgetxattr_prep,
446                 .issue                  = io_fgetxattr,
447                 .cleanup                = io_xattr_cleanup,
448         },
449         [IORING_OP_GETXATTR] = {
450                 .name                   = "GETXATTR",
451                 .prep                   = io_getxattr_prep,
452                 .issue                  = io_getxattr,
453                 .cleanup                = io_xattr_cleanup,
454         },
455         [IORING_OP_SOCKET] = {
456                 .audit_skip             = 1,
457                 .name                   = "SOCKET",
458 #if defined(CONFIG_NET)
459                 .prep                   = io_socket_prep,
460                 .issue                  = io_socket,
461 #else
462                 .prep                   = io_eopnotsupp_prep,
463 #endif
464         },
465         [IORING_OP_URING_CMD] = {
466                 .needs_file             = 1,
467                 .plug                   = 1,
468                 .name                   = "URING_CMD",
469                 .async_size             = uring_cmd_pdu_size(1),
470                 .prep                   = io_uring_cmd_prep,
471                 .issue                  = io_uring_cmd,
472                 .prep_async             = io_uring_cmd_prep_async,
473         },
474         [IORING_OP_SENDZC_NOTIF] = {
475                 .name                   = "SENDZC_NOTIF",
476                 .needs_file             = 1,
477                 .unbound_nonreg_file    = 1,
478                 .pollout                = 1,
479                 .audit_skip             = 1,
480                 .ioprio                 = 1,
481 #if defined(CONFIG_NET)
482                 .prep                   = io_sendzc_prep,
483                 .issue                  = io_sendzc,
484 #else
485                 .prep                   = io_eopnotsupp_prep,
486 #endif
487
488         },
489 };
490
491 const char *io_uring_get_opcode(u8 opcode)
492 {
493         if (opcode < IORING_OP_LAST)
494                 return io_op_defs[opcode].name;
495         return "INVALID";
496 }
497
498 void __init io_uring_optable_init(void)
499 {
500         int i;
501
502         BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
503
504         for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) {
505                 BUG_ON(!io_op_defs[i].prep);
506                 if (io_op_defs[i].prep != io_eopnotsupp_prep)
507                         BUG_ON(!io_op_defs[i].issue);
508                 WARN_ON_ONCE(!io_op_defs[i].name);
509         }
510 }