io_uring/opdef.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * io_uring opcode handling table
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/errno.h>
   7 #include <linux/fs.h>
   8 #include <linux/file.h>
   9 #include <linux/io_uring.h>
  10
  11 #include "io_uring.h"
  12 #include "opdef.h"
  13 #include "refs.h"
  14 #include "tctx.h"
  15 #include "sqpoll.h"
  16 #include "fdinfo.h"
  17 #include "kbuf.h"
  18 #include "rsrc.h"
  19
  20 #include "xattr.h"
  21 #include "nop.h"
  22 #include "fs.h"
  23 #include "splice.h"
  24 #include "sync.h"
  25 #include "advise.h"
  26 #include "openclose.h"
  27 #include "uring_cmd.h"
  28 #include "epoll.h"
  29 #include "statx.h"
  30 #include "net.h"
  31 #include "msg_ring.h"
  32 #include "timeout.h"
  33 #include "poll.h"
  34 #include "cancel.h"
  35 #include "rw.h"
  36
  37 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
  38 {
  39         WARN_ON_ONCE(1);
  40         return -ECANCELED;
  41 }
  42
  43 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
  44                                              const struct io_uring_sqe *sqe)
  45 {
  46         return -EOPNOTSUPP;
  47 }
  48
  49 const struct io_op_def io_op_defs[] = {
  50         [IORING_OP_NOP] = {
  51                 .audit_skip             = 1,
  52                 .iopoll                 = 1,
  53                 .name                   = "NOP",
  54                 .prep                   = io_nop_prep,
  55                 .issue                  = io_nop,
  56         },
  57         [IORING_OP_READV] = {
  58                 .needs_file             = 1,
  59                 .unbound_nonreg_file    = 1,
  60                 .pollin                 = 1,
  61                 .buffer_select          = 1,
  62                 .plug                   = 1,
  63                 .audit_skip             = 1,
  64                 .ioprio                 = 1,
  65                 .iopoll                 = 1,
  66                 .iopoll_queue           = 1,
  67                 .async_size             = sizeof(struct io_async_rw),
  68                 .name                   = "READV",
  69                 .prep                   = io_prep_rw,
  70                 .issue                  = io_read,
  71                 .prep_async             = io_readv_prep_async,
  72                 .cleanup                = io_readv_writev_cleanup,
  73                 .fail                   = io_rw_fail,
  74         },
  75         [IORING_OP_WRITEV] = {
  76                 .needs_file             = 1,
  77                 .hash_reg_file          = 1,
  78                 .unbound_nonreg_file    = 1,
  79                 .pollout                = 1,
  80                 .plug                   = 1,
  81                 .audit_skip             = 1,
  82                 .ioprio                 = 1,
  83                 .iopoll                 = 1,
  84                 .iopoll_queue           = 1,
  85                 .async_size             = sizeof(struct io_async_rw),
  86                 .name                   = "WRITEV",
  87                 .prep                   = io_prep_rw,
  88                 .issue                  = io_write,
  89                 .prep_async             = io_writev_prep_async,
  90                 .cleanup                = io_readv_writev_cleanup,
  91                 .fail                   = io_rw_fail,
  92         },
  93         [IORING_OP_FSYNC] = {
  94                 .needs_file             = 1,
  95                 .audit_skip             = 1,
  96                 .name                   = "FSYNC",
  97                 .prep                   = io_fsync_prep,
  98                 .issue                  = io_fsync,
  99         },
 100         [IORING_OP_READ_FIXED] = {
 101                 .needs_file             = 1,
 102                 .unbound_nonreg_file    = 1,
 103                 .pollin                 = 1,
 104                 .plug                   = 1,
 105                 .audit_skip             = 1,
 106                 .ioprio                 = 1,
 107                 .iopoll                 = 1,
 108                 .iopoll_queue           = 1,
 109                 .async_size             = sizeof(struct io_async_rw),
 110                 .name                   = "READ_FIXED",
 111                 .prep                   = io_prep_rw,
 112                 .issue                  = io_read,
 113                 .fail                   = io_rw_fail,
 114         },
 115         [IORING_OP_WRITE_FIXED] = {
 116                 .needs_file             = 1,
 117                 .hash_reg_file          = 1,
 118                 .unbound_nonreg_file    = 1,
 119                 .pollout                = 1,
 120                 .plug                   = 1,
 121                 .audit_skip             = 1,
 122                 .ioprio                 = 1,
 123                 .iopoll                 = 1,
 124                 .iopoll_queue           = 1,
 125                 .async_size             = sizeof(struct io_async_rw),
 126                 .name                   = "WRITE_FIXED",
 127                 .prep                   = io_prep_rw,
 128                 .issue                  = io_write,
 129                 .fail                   = io_rw_fail,
 130         },
 131         [IORING_OP_POLL_ADD] = {
 132                 .needs_file             = 1,
 133                 .unbound_nonreg_file    = 1,
 134                 .audit_skip             = 1,
 135                 .name                   = "POLL_ADD",
 136                 .prep                   = io_poll_add_prep,
 137                 .issue                  = io_poll_add,
 138         },
 139         [IORING_OP_POLL_REMOVE] = {
 140                 .audit_skip             = 1,
 141                 .name                   = "POLL_REMOVE",
 142                 .prep                   = io_poll_remove_prep,
 143                 .issue                  = io_poll_remove,
 144         },
 145         [IORING_OP_SYNC_FILE_RANGE] = {
 146                 .needs_file             = 1,
 147                 .audit_skip             = 1,
 148                 .name                   = "SYNC_FILE_RANGE",
 149                 .prep                   = io_sfr_prep,
 150                 .issue                  = io_sync_file_range,
 151         },
 152         [IORING_OP_SENDMSG] = {
 153                 .needs_file             = 1,
 154                 .unbound_nonreg_file    = 1,
 155                 .pollout                = 1,
 156                 .ioprio                 = 1,
 157                 .manual_alloc           = 1,
 158                 .name                   = "SENDMSG",
 159 #if defined(CONFIG_NET)
 160                 .async_size             = sizeof(struct io_async_msghdr),
 161                 .prep                   = io_sendmsg_prep,
 162                 .issue                  = io_sendmsg,
 163                 .prep_async             = io_sendmsg_prep_async,
 164                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 165                 .fail                   = io_sendrecv_fail,
 166 #else
 167                 .prep                   = io_eopnotsupp_prep,
 168 #endif
 169         },
 170         [IORING_OP_RECVMSG] = {
 171                 .needs_file             = 1,
 172                 .unbound_nonreg_file    = 1,
 173                 .pollin                 = 1,
 174                 .buffer_select          = 1,
 175                 .ioprio                 = 1,
 176                 .manual_alloc           = 1,
 177                 .name                   = "RECVMSG",
 178 #if defined(CONFIG_NET)
 179                 .async_size             = sizeof(struct io_async_msghdr),
 180                 .prep                   = io_recvmsg_prep,
 181                 .issue                  = io_recvmsg,
 182                 .prep_async             = io_recvmsg_prep_async,
 183                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 184                 .fail                   = io_sendrecv_fail,
 185 #else
 186                 .prep                   = io_eopnotsupp_prep,
 187 #endif
 188         },
 189         [IORING_OP_TIMEOUT] = {
 190                 .audit_skip             = 1,
 191                 .async_size             = sizeof(struct io_timeout_data),
 192                 .name                   = "TIMEOUT",
 193                 .prep                   = io_timeout_prep,
 194                 .issue                  = io_timeout,
 195         },
 196         [IORING_OP_TIMEOUT_REMOVE] = {
 197                 /* used by timeout updates' prep() */
 198                 .audit_skip             = 1,
 199                 .name                   = "TIMEOUT_REMOVE",
 200                 .prep                   = io_timeout_remove_prep,
 201                 .issue                  = io_timeout_remove,
 202         },
 203         [IORING_OP_ACCEPT] = {
 204                 .needs_file             = 1,
 205                 .unbound_nonreg_file    = 1,
 206                 .pollin                 = 1,
 207                 .poll_exclusive         = 1,
 208                 .ioprio                 = 1,    /* used for flags */
 209                 .name                   = "ACCEPT",
 210 #if defined(CONFIG_NET)
 211                 .prep                   = io_accept_prep,
 212                 .issue                  = io_accept,
 213 #else
 214                 .prep                   = io_eopnotsupp_prep,
 215 #endif
 216         },
 217         [IORING_OP_ASYNC_CANCEL] = {
 218                 .audit_skip             = 1,
 219                 .name                   = "ASYNC_CANCEL",
 220                 .prep                   = io_async_cancel_prep,
 221                 .issue                  = io_async_cancel,
 222         },
 223         [IORING_OP_LINK_TIMEOUT] = {
 224                 .audit_skip             = 1,
 225                 .async_size             = sizeof(struct io_timeout_data),
 226                 .name                   = "LINK_TIMEOUT",
 227                 .prep                   = io_link_timeout_prep,
 228                 .issue                  = io_no_issue,
 229         },
 230         [IORING_OP_CONNECT] = {
 231                 .needs_file             = 1,
 232                 .unbound_nonreg_file    = 1,
 233                 .pollout                = 1,
 234                 .name                   = "CONNECT",
 235 #if defined(CONFIG_NET)
 236                 .async_size             = sizeof(struct io_async_connect),
 237                 .prep                   = io_connect_prep,
 238                 .issue                  = io_connect,
 239                 .prep_async             = io_connect_prep_async,
 240 #else
 241                 .prep                   = io_eopnotsupp_prep,
 242 #endif
 243         },
 244         [IORING_OP_FALLOCATE] = {
 245                 .needs_file             = 1,
 246                 .name                   = "FALLOCATE",
 247                 .prep                   = io_fallocate_prep,
 248                 .issue                  = io_fallocate,
 249         },
 250         [IORING_OP_OPENAT] = {
 251                 .name                   = "OPENAT",
 252                 .prep                   = io_openat_prep,
 253                 .issue                  = io_openat,
 254                 .cleanup                = io_open_cleanup,
 255         },
 256         [IORING_OP_CLOSE] = {
 257                 .name                   = "CLOSE",
 258                 .prep                   = io_close_prep,
 259                 .issue                  = io_close,
 260         },
 261         [IORING_OP_FILES_UPDATE] = {
 262                 .audit_skip             = 1,
 263                 .iopoll                 = 1,
 264                 .name                   = "FILES_UPDATE",
 265                 .prep                   = io_files_update_prep,
 266                 .issue                  = io_files_update,
 267         },
 268         [IORING_OP_STATX] = {
 269                 .audit_skip             = 1,
 270                 .name                   = "STATX",
 271                 .prep                   = io_statx_prep,
 272                 .issue                  = io_statx,
 273                 .cleanup                = io_statx_cleanup,
 274         },
 275         [IORING_OP_READ] = {
 276                 .needs_file             = 1,
 277                 .unbound_nonreg_file    = 1,
 278                 .pollin                 = 1,
 279                 .buffer_select          = 1,
 280                 .plug                   = 1,
 281                 .audit_skip             = 1,
 282                 .ioprio                 = 1,
 283                 .iopoll                 = 1,
 284                 .iopoll_queue           = 1,
 285                 .async_size             = sizeof(struct io_async_rw),
 286                 .name                   = "READ",
 287                 .prep                   = io_prep_rw,
 288                 .issue                  = io_read,
 289                 .fail                   = io_rw_fail,
 290         },
 291         [IORING_OP_WRITE] = {
 292                 .needs_file             = 1,
 293                 .hash_reg_file          = 1,
 294                 .unbound_nonreg_file    = 1,
 295                 .pollout                = 1,
 296                 .plug                   = 1,
 297                 .audit_skip             = 1,
 298                 .ioprio                 = 1,
 299                 .iopoll                 = 1,
 300                 .iopoll_queue           = 1,
 301                 .async_size             = sizeof(struct io_async_rw),
 302                 .name                   = "WRITE",
 303                 .prep                   = io_prep_rw,
 304                 .issue                  = io_write,
 305                 .fail                   = io_rw_fail,
 306         },
 307         [IORING_OP_FADVISE] = {
 308                 .needs_file             = 1,
 309                 .audit_skip             = 1,
 310                 .name                   = "FADVISE",
 311                 .prep                   = io_fadvise_prep,
 312                 .issue                  = io_fadvise,
 313         },
 314         [IORING_OP_MADVISE] = {
 315                 .name                   = "MADVISE",
 316                 .prep                   = io_madvise_prep,
 317                 .issue                  = io_madvise,
 318         },
 319         [IORING_OP_SEND] = {
 320                 .needs_file             = 1,
 321                 .unbound_nonreg_file    = 1,
 322                 .pollout                = 1,
 323                 .audit_skip             = 1,
 324                 .ioprio                 = 1,
 325                 .manual_alloc           = 1,
 326                 .name                   = "SEND",
 327 #if defined(CONFIG_NET)
 328                 .async_size             = sizeof(struct io_async_msghdr),
 329                 .prep                   = io_sendmsg_prep,
 330                 .issue                  = io_send,
 331                 .fail                   = io_sendrecv_fail,
 332                 .prep_async             = io_send_prep_async,
 333 #else
 334                 .prep                   = io_eopnotsupp_prep,
 335 #endif
 336         },
 337         [IORING_OP_RECV] = {
 338                 .needs_file             = 1,
 339                 .unbound_nonreg_file    = 1,
 340                 .pollin                 = 1,
 341                 .buffer_select          = 1,
 342                 .audit_skip             = 1,
 343                 .ioprio                 = 1,
 344                 .name                   = "RECV",
 345 #if defined(CONFIG_NET)
 346                 .prep                   = io_recvmsg_prep,
 347                 .issue                  = io_recv,
 348                 .fail                   = io_sendrecv_fail,
 349 #else
 350                 .prep                   = io_eopnotsupp_prep,
 351 #endif
 352         },
 353         [IORING_OP_OPENAT2] = {
 354                 .name                   = "OPENAT2",
 355                 .prep                   = io_openat2_prep,
 356                 .issue                  = io_openat2,
 357                 .cleanup                = io_open_cleanup,
 358         },
 359         [IORING_OP_EPOLL_CTL] = {
 360                 .unbound_nonreg_file    = 1,
 361                 .audit_skip             = 1,
 362                 .name                   = "EPOLL",
 363 #if defined(CONFIG_EPOLL)
 364                 .prep                   = io_epoll_ctl_prep,
 365                 .issue                  = io_epoll_ctl,
 366 #else
 367                 .prep                   = io_eopnotsupp_prep,
 368 #endif
 369         },
 370         [IORING_OP_SPLICE] = {
 371                 .needs_file             = 1,
 372                 .hash_reg_file          = 1,
 373                 .unbound_nonreg_file    = 1,
 374                 .audit_skip             = 1,
 375                 .name                   = "SPLICE",
 376                 .prep                   = io_splice_prep,
 377                 .issue                  = io_splice,
 378         },
 379         [IORING_OP_PROVIDE_BUFFERS] = {
 380                 .audit_skip             = 1,
 381                 .iopoll                 = 1,
 382                 .name                   = "PROVIDE_BUFFERS",
 383                 .prep                   = io_provide_buffers_prep,
 384                 .issue                  = io_provide_buffers,
 385         },
 386         [IORING_OP_REMOVE_BUFFERS] = {
 387                 .audit_skip             = 1,
 388                 .iopoll                 = 1,
 389                 .name                   = "REMOVE_BUFFERS",
 390                 .prep                   = io_remove_buffers_prep,
 391                 .issue                  = io_remove_buffers,
 392         },
 393         [IORING_OP_TEE] = {
 394                 .needs_file             = 1,
 395                 .hash_reg_file          = 1,
 396                 .unbound_nonreg_file    = 1,
 397                 .audit_skip             = 1,
 398                 .name                   = "TEE",
 399                 .prep                   = io_tee_prep,
 400                 .issue                  = io_tee,
 401         },
 402         [IORING_OP_SHUTDOWN] = {
 403                 .needs_file             = 1,
 404                 .name                   = "SHUTDOWN",
 405 #if defined(CONFIG_NET)
 406                 .prep                   = io_shutdown_prep,
 407                 .issue                  = io_shutdown,
 408 #else
 409                 .prep                   = io_eopnotsupp_prep,
 410 #endif
 411         },
 412         [IORING_OP_RENAMEAT] = {
 413                 .name                   = "RENAMEAT",
 414                 .prep                   = io_renameat_prep,
 415                 .issue                  = io_renameat,
 416                 .cleanup                = io_renameat_cleanup,
 417         },
 418         [IORING_OP_UNLINKAT] = {
 419                 .name                   = "UNLINKAT",
 420                 .prep                   = io_unlinkat_prep,
 421                 .issue                  = io_unlinkat,
 422                 .cleanup                = io_unlinkat_cleanup,
 423         },
 424         [IORING_OP_MKDIRAT] = {
 425                 .name                   = "MKDIRAT",
 426                 .prep                   = io_mkdirat_prep,
 427                 .issue                  = io_mkdirat,
 428                 .cleanup                = io_mkdirat_cleanup,
 429         },
 430         [IORING_OP_SYMLINKAT] = {
 431                 .name                   = "SYMLINKAT",
 432                 .prep                   = io_symlinkat_prep,
 433                 .issue                  = io_symlinkat,
 434                 .cleanup                = io_link_cleanup,
 435         },
 436         [IORING_OP_LINKAT] = {
 437                 .name                   = "LINKAT",
 438                 .prep                   = io_linkat_prep,
 439                 .issue                  = io_linkat,
 440                 .cleanup                = io_link_cleanup,
 441         },
 442         [IORING_OP_MSG_RING] = {
 443                 .needs_file             = 1,
 444                 .iopoll                 = 1,
 445                 .name                   = "MSG_RING",
 446                 .prep                   = io_msg_ring_prep,
 447                 .issue                  = io_msg_ring,
 448                 .cleanup                = io_msg_ring_cleanup,
 449         },
 450         [IORING_OP_FSETXATTR] = {
 451                 .needs_file = 1,
 452                 .name                   = "FSETXATTR",
 453                 .prep                   = io_fsetxattr_prep,
 454                 .issue                  = io_fsetxattr,
 455                 .cleanup                = io_xattr_cleanup,
 456         },
 457         [IORING_OP_SETXATTR] = {
 458                 .name                   = "SETXATTR",
 459                 .prep                   = io_setxattr_prep,
 460                 .issue                  = io_setxattr,
 461                 .cleanup                = io_xattr_cleanup,
 462         },
 463         [IORING_OP_FGETXATTR] = {
 464                 .needs_file = 1,
 465                 .name                   = "FGETXATTR",
 466                 .prep                   = io_fgetxattr_prep,
 467                 .issue                  = io_fgetxattr,
 468                 .cleanup                = io_xattr_cleanup,
 469         },
 470         [IORING_OP_GETXATTR] = {
 471                 .name                   = "GETXATTR",
 472                 .prep                   = io_getxattr_prep,
 473                 .issue                  = io_getxattr,
 474                 .cleanup                = io_xattr_cleanup,
 475         },
 476         [IORING_OP_SOCKET] = {
 477                 .audit_skip             = 1,
 478                 .name                   = "SOCKET",
 479 #if defined(CONFIG_NET)
 480                 .prep                   = io_socket_prep,
 481                 .issue                  = io_socket,
 482 #else
 483                 .prep                   = io_eopnotsupp_prep,
 484 #endif
 485         },
 486         [IORING_OP_URING_CMD] = {
 487                 .needs_file             = 1,
 488                 .plug                   = 1,
 489                 .name                   = "URING_CMD",
 490                 .iopoll                 = 1,
 491                 .iopoll_queue           = 1,
 492                 .async_size             = uring_cmd_pdu_size(1),
 493                 .prep                   = io_uring_cmd_prep,
 494                 .issue                  = io_uring_cmd,
 495                 .prep_async             = io_uring_cmd_prep_async,
 496         },
 497         [IORING_OP_SEND_ZC] = {
 498                 .name                   = "SEND_ZC",
 499                 .needs_file             = 1,
 500                 .unbound_nonreg_file    = 1,
 501                 .pollout                = 1,
 502                 .audit_skip             = 1,
 503                 .ioprio                 = 1,
 504                 .manual_alloc           = 1,
 505 #if defined(CONFIG_NET)
 506                 .async_size             = sizeof(struct io_async_msghdr),
 507                 .prep                   = io_send_zc_prep,
 508                 .issue                  = io_send_zc,
 509                 .prep_async             = io_send_prep_async,
 510                 .cleanup                = io_send_zc_cleanup,
 511                 .fail                   = io_sendrecv_fail,
 512 #else
 513                 .prep                   = io_eopnotsupp_prep,
 514 #endif
 515         },
 516         [IORING_OP_SENDMSG_ZC] = {
 517                 .name                   = "SENDMSG_ZC",
 518                 .needs_file             = 1,
 519                 .unbound_nonreg_file    = 1,
 520                 .pollout                = 1,
 521                 .ioprio                 = 1,
 522                 .manual_alloc           = 1,
 523 #if defined(CONFIG_NET)
 524                 .async_size             = sizeof(struct io_async_msghdr),
 525                 .prep                   = io_send_zc_prep,
 526                 .issue                  = io_sendmsg_zc,
 527                 .prep_async             = io_sendmsg_prep_async,
 528                 .cleanup                = io_send_zc_cleanup,
 529                 .fail                   = io_sendrecv_fail,
 530 #else
 531                 .prep                   = io_eopnotsupp_prep,
 532 #endif
 533         },
 534 };
 535
 536 const char *io_uring_get_opcode(u8 opcode)
 537 {
 538         if (opcode < IORING_OP_LAST)
 539                 return io_op_defs[opcode].name;
 540         return "INVALID";
 541 }
 542
 543 void __init io_uring_optable_init(void)
 544 {
 545         int i;
 546
 547         BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
 548
 549         for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) {
 550                 BUG_ON(!io_op_defs[i].prep);
 551                 if (io_op_defs[i].prep != io_eopnotsupp_prep)
 552                         BUG_ON(!io_op_defs[i].issue);
 553                 WARN_ON_ONCE(!io_op_defs[i].name);
 554         }
 555 }