io_uring/opdef.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * io_uring opcode handling table
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/errno.h>
   7 #include <linux/fs.h>
   8 #include <linux/file.h>
   9 #include <linux/io_uring.h>
  10
  11 #include "io_uring.h"
  12 #include "opdef.h"
  13 #include "refs.h"
  14 #include "tctx.h"
  15 #include "sqpoll.h"
  16 #include "fdinfo.h"
  17 #include "kbuf.h"
  18 #include "rsrc.h"
  19
  20 #include "xattr.h"
  21 #include "nop.h"
  22 #include "fs.h"
  23 #include "splice.h"
  24 #include "sync.h"
  25 #include "advise.h"
  26 #include "openclose.h"
  27 #include "uring_cmd.h"
  28 #include "epoll.h"
  29 #include "statx.h"
  30 #include "net.h"
  31 #include "msg_ring.h"
  32 #include "timeout.h"
  33 #include "poll.h"
  34 #include "cancel.h"
  35 #include "rw.h"
  36
  37 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
  38 {
  39         WARN_ON_ONCE(1);
  40         return -ECANCELED;
  41 }
  42
  43 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
  44                                              const struct io_uring_sqe *sqe)
  45 {
  46         return -EOPNOTSUPP;
  47 }
  48
  49 const struct io_issue_def io_issue_defs[] = {
  50         [IORING_OP_NOP] = {
  51                 .audit_skip             = 1,
  52                 .iopoll                 = 1,
  53                 .prep                   = io_nop_prep,
  54                 .issue                  = io_nop,
  55         },
  56         [IORING_OP_READV] = {
  57                 .needs_file             = 1,
  58                 .unbound_nonreg_file    = 1,
  59                 .pollin                 = 1,
  60                 .buffer_select          = 1,
  61                 .plug                   = 1,
  62                 .audit_skip             = 1,
  63                 .ioprio                 = 1,
  64                 .iopoll                 = 1,
  65                 .iopoll_queue           = 1,
  66                 .prep                   = io_prep_rw,
  67                 .issue                  = io_read,
  68         },
  69         [IORING_OP_WRITEV] = {
  70                 .needs_file             = 1,
  71                 .hash_reg_file          = 1,
  72                 .unbound_nonreg_file    = 1,
  73                 .pollout                = 1,
  74                 .plug                   = 1,
  75                 .audit_skip             = 1,
  76                 .ioprio                 = 1,
  77                 .iopoll                 = 1,
  78                 .iopoll_queue           = 1,
  79                 .prep                   = io_prep_rw,
  80                 .issue                  = io_write,
  81         },
  82         [IORING_OP_FSYNC] = {
  83                 .needs_file             = 1,
  84                 .audit_skip             = 1,
  85                 .prep                   = io_fsync_prep,
  86                 .issue                  = io_fsync,
  87         },
  88         [IORING_OP_READ_FIXED] = {
  89                 .needs_file             = 1,
  90                 .unbound_nonreg_file    = 1,
  91                 .pollin                 = 1,
  92                 .plug                   = 1,
  93                 .audit_skip             = 1,
  94                 .ioprio                 = 1,
  95                 .iopoll                 = 1,
  96                 .iopoll_queue           = 1,
  97                 .prep                   = io_prep_rw,
  98                 .issue                  = io_read,
  99         },
 100         [IORING_OP_WRITE_FIXED] = {
 101                 .needs_file             = 1,
 102                 .hash_reg_file          = 1,
 103                 .unbound_nonreg_file    = 1,
 104                 .pollout                = 1,
 105                 .plug                   = 1,
 106                 .audit_skip             = 1,
 107                 .ioprio                 = 1,
 108                 .iopoll                 = 1,
 109                 .iopoll_queue           = 1,
 110                 .prep                   = io_prep_rw,
 111                 .issue                  = io_write,
 112         },
 113         [IORING_OP_POLL_ADD] = {
 114                 .needs_file             = 1,
 115                 .unbound_nonreg_file    = 1,
 116                 .audit_skip             = 1,
 117                 .prep                   = io_poll_add_prep,
 118                 .issue                  = io_poll_add,
 119         },
 120         [IORING_OP_POLL_REMOVE] = {
 121                 .audit_skip             = 1,
 122                 .prep                   = io_poll_remove_prep,
 123                 .issue                  = io_poll_remove,
 124         },
 125         [IORING_OP_SYNC_FILE_RANGE] = {
 126                 .needs_file             = 1,
 127                 .audit_skip             = 1,
 128                 .prep                   = io_sfr_prep,
 129                 .issue                  = io_sync_file_range,
 130         },
 131         [IORING_OP_SENDMSG] = {
 132                 .needs_file             = 1,
 133                 .unbound_nonreg_file    = 1,
 134                 .pollout                = 1,
 135                 .ioprio                 = 1,
 136                 .manual_alloc           = 1,
 137 #if defined(CONFIG_NET)
 138                 .prep                   = io_sendmsg_prep,
 139                 .issue                  = io_sendmsg,
 140 #else
 141                 .prep                   = io_eopnotsupp_prep,
 142 #endif
 143         },
 144         [IORING_OP_RECVMSG] = {
 145                 .needs_file             = 1,
 146                 .unbound_nonreg_file    = 1,
 147                 .pollin                 = 1,
 148                 .buffer_select          = 1,
 149                 .ioprio                 = 1,
 150                 .manual_alloc           = 1,
 151 #if defined(CONFIG_NET)
 152                 .prep                   = io_recvmsg_prep,
 153                 .issue                  = io_recvmsg,
 154 #else
 155                 .prep                   = io_eopnotsupp_prep,
 156 #endif
 157         },
 158         [IORING_OP_TIMEOUT] = {
 159                 .audit_skip             = 1,
 160                 .prep                   = io_timeout_prep,
 161                 .issue                  = io_timeout,
 162         },
 163         [IORING_OP_TIMEOUT_REMOVE] = {
 164                 /* used by timeout updates' prep() */
 165                 .audit_skip             = 1,
 166                 .prep                   = io_timeout_remove_prep,
 167                 .issue                  = io_timeout_remove,
 168         },
 169         [IORING_OP_ACCEPT] = {
 170                 .needs_file             = 1,
 171                 .unbound_nonreg_file    = 1,
 172                 .pollin                 = 1,
 173                 .poll_exclusive         = 1,
 174                 .ioprio                 = 1,    /* used for flags */
 175 #if defined(CONFIG_NET)
 176                 .prep                   = io_accept_prep,
 177                 .issue                  = io_accept,
 178 #else
 179                 .prep                   = io_eopnotsupp_prep,
 180 #endif
 181         },
 182         [IORING_OP_ASYNC_CANCEL] = {
 183                 .audit_skip             = 1,
 184                 .prep                   = io_async_cancel_prep,
 185                 .issue                  = io_async_cancel,
 186         },
 187         [IORING_OP_LINK_TIMEOUT] = {
 188                 .audit_skip             = 1,
 189                 .prep                   = io_link_timeout_prep,
 190                 .issue                  = io_no_issue,
 191         },
 192         [IORING_OP_CONNECT] = {
 193                 .needs_file             = 1,
 194                 .unbound_nonreg_file    = 1,
 195                 .pollout                = 1,
 196 #if defined(CONFIG_NET)
 197                 .prep                   = io_connect_prep,
 198                 .issue                  = io_connect,
 199 #else
 200                 .prep                   = io_eopnotsupp_prep,
 201 #endif
 202         },
 203         [IORING_OP_FALLOCATE] = {
 204                 .needs_file             = 1,
 205                 .prep                   = io_fallocate_prep,
 206                 .issue                  = io_fallocate,
 207         },
 208         [IORING_OP_OPENAT] = {
 209                 .prep                   = io_openat_prep,
 210                 .issue                  = io_openat,
 211         },
 212         [IORING_OP_CLOSE] = {
 213                 .prep                   = io_close_prep,
 214                 .issue                  = io_close,
 215         },
 216         [IORING_OP_FILES_UPDATE] = {
 217                 .audit_skip             = 1,
 218                 .iopoll                 = 1,
 219                 .prep                   = io_files_update_prep,
 220                 .issue                  = io_files_update,
 221         },
 222         [IORING_OP_STATX] = {
 223                 .audit_skip             = 1,
 224                 .prep                   = io_statx_prep,
 225                 .issue                  = io_statx,
 226         },
 227         [IORING_OP_READ] = {
 228                 .needs_file             = 1,
 229                 .unbound_nonreg_file    = 1,
 230                 .pollin                 = 1,
 231                 .buffer_select          = 1,
 232                 .plug                   = 1,
 233                 .audit_skip             = 1,
 234                 .ioprio                 = 1,
 235                 .iopoll                 = 1,
 236                 .iopoll_queue           = 1,
 237                 .prep                   = io_prep_rw,
 238                 .issue                  = io_read,
 239         },
 240         [IORING_OP_WRITE] = {
 241                 .needs_file             = 1,
 242                 .hash_reg_file          = 1,
 243                 .unbound_nonreg_file    = 1,
 244                 .pollout                = 1,
 245                 .plug                   = 1,
 246                 .audit_skip             = 1,
 247                 .ioprio                 = 1,
 248                 .iopoll                 = 1,
 249                 .iopoll_queue           = 1,
 250                 .prep                   = io_prep_rw,
 251                 .issue                  = io_write,
 252         },
 253         [IORING_OP_FADVISE] = {
 254                 .needs_file             = 1,
 255                 .audit_skip             = 1,
 256                 .prep                   = io_fadvise_prep,
 257                 .issue                  = io_fadvise,
 258         },
 259         [IORING_OP_MADVISE] = {
 260                 .audit_skip             = 1,
 261                 .prep                   = io_madvise_prep,
 262                 .issue                  = io_madvise,
 263         },
 264         [IORING_OP_SEND] = {
 265                 .needs_file             = 1,
 266                 .unbound_nonreg_file    = 1,
 267                 .pollout                = 1,
 268                 .audit_skip             = 1,
 269                 .ioprio                 = 1,
 270                 .manual_alloc           = 1,
 271 #if defined(CONFIG_NET)
 272                 .prep                   = io_sendmsg_prep,
 273                 .issue                  = io_send,
 274 #else
 275                 .prep                   = io_eopnotsupp_prep,
 276 #endif
 277         },
 278         [IORING_OP_RECV] = {
 279                 .needs_file             = 1,
 280                 .unbound_nonreg_file    = 1,
 281                 .pollin                 = 1,
 282                 .buffer_select          = 1,
 283                 .audit_skip             = 1,
 284                 .ioprio                 = 1,
 285 #if defined(CONFIG_NET)
 286                 .prep                   = io_recvmsg_prep,
 287                 .issue                  = io_recv,
 288 #else
 289                 .prep                   = io_eopnotsupp_prep,
 290 #endif
 291         },
 292         [IORING_OP_OPENAT2] = {
 293                 .prep                   = io_openat2_prep,
 294                 .issue                  = io_openat2,
 295         },
 296         [IORING_OP_EPOLL_CTL] = {
 297                 .unbound_nonreg_file    = 1,
 298                 .audit_skip             = 1,
 299 #if defined(CONFIG_EPOLL)
 300                 .prep                   = io_epoll_ctl_prep,
 301                 .issue                  = io_epoll_ctl,
 302 #else
 303                 .prep                   = io_eopnotsupp_prep,
 304 #endif
 305         },
 306         [IORING_OP_SPLICE] = {
 307                 .needs_file             = 1,
 308                 .hash_reg_file          = 1,
 309                 .unbound_nonreg_file    = 1,
 310                 .audit_skip             = 1,
 311                 .prep                   = io_splice_prep,
 312                 .issue                  = io_splice,
 313         },
 314         [IORING_OP_PROVIDE_BUFFERS] = {
 315                 .audit_skip             = 1,
 316                 .iopoll                 = 1,
 317                 .prep                   = io_provide_buffers_prep,
 318                 .issue                  = io_provide_buffers,
 319         },
 320         [IORING_OP_REMOVE_BUFFERS] = {
 321                 .audit_skip             = 1,
 322                 .iopoll                 = 1,
 323                 .prep                   = io_remove_buffers_prep,
 324                 .issue                  = io_remove_buffers,
 325         },
 326         [IORING_OP_TEE] = {
 327                 .needs_file             = 1,
 328                 .hash_reg_file          = 1,
 329                 .unbound_nonreg_file    = 1,
 330                 .audit_skip             = 1,
 331                 .prep                   = io_tee_prep,
 332                 .issue                  = io_tee,
 333         },
 334         [IORING_OP_SHUTDOWN] = {
 335                 .needs_file             = 1,
 336 #if defined(CONFIG_NET)
 337                 .prep                   = io_shutdown_prep,
 338                 .issue                  = io_shutdown,
 339 #else
 340                 .prep                   = io_eopnotsupp_prep,
 341 #endif
 342         },
 343         [IORING_OP_RENAMEAT] = {
 344                 .prep                   = io_renameat_prep,
 345                 .issue                  = io_renameat,
 346         },
 347         [IORING_OP_UNLINKAT] = {
 348                 .prep                   = io_unlinkat_prep,
 349                 .issue                  = io_unlinkat,
 350         },
 351         [IORING_OP_MKDIRAT] = {
 352                 .prep                   = io_mkdirat_prep,
 353                 .issue                  = io_mkdirat,
 354         },
 355         [IORING_OP_SYMLINKAT] = {
 356                 .prep                   = io_symlinkat_prep,
 357                 .issue                  = io_symlinkat,
 358         },
 359         [IORING_OP_LINKAT] = {
 360                 .prep                   = io_linkat_prep,
 361                 .issue                  = io_linkat,
 362         },
 363         [IORING_OP_MSG_RING] = {
 364                 .needs_file             = 1,
 365                 .iopoll                 = 1,
 366                 .prep                   = io_msg_ring_prep,
 367                 .issue                  = io_msg_ring,
 368         },
 369         [IORING_OP_FSETXATTR] = {
 370                 .needs_file = 1,
 371                 .prep                   = io_fsetxattr_prep,
 372                 .issue                  = io_fsetxattr,
 373         },
 374         [IORING_OP_SETXATTR] = {
 375                 .prep                   = io_setxattr_prep,
 376                 .issue                  = io_setxattr,
 377         },
 378         [IORING_OP_FGETXATTR] = {
 379                 .needs_file = 1,
 380                 .prep                   = io_fgetxattr_prep,
 381                 .issue                  = io_fgetxattr,
 382         },
 383         [IORING_OP_GETXATTR] = {
 384                 .prep                   = io_getxattr_prep,
 385                 .issue                  = io_getxattr,
 386         },
 387         [IORING_OP_SOCKET] = {
 388                 .audit_skip             = 1,
 389 #if defined(CONFIG_NET)
 390                 .prep                   = io_socket_prep,
 391                 .issue                  = io_socket,
 392 #else
 393                 .prep                   = io_eopnotsupp_prep,
 394 #endif
 395         },
 396         [IORING_OP_URING_CMD] = {
 397                 .needs_file             = 1,
 398                 .plug                   = 1,
 399                 .iopoll                 = 1,
 400                 .iopoll_queue           = 1,
 401                 .prep                   = io_uring_cmd_prep,
 402                 .issue                  = io_uring_cmd,
 403         },
 404         [IORING_OP_SEND_ZC] = {
 405                 .needs_file             = 1,
 406                 .unbound_nonreg_file    = 1,
 407                 .pollout                = 1,
 408                 .audit_skip             = 1,
 409                 .ioprio                 = 1,
 410                 .manual_alloc           = 1,
 411 #if defined(CONFIG_NET)
 412                 .prep                   = io_send_zc_prep,
 413                 .issue                  = io_send_zc,
 414 #else
 415                 .prep                   = io_eopnotsupp_prep,
 416 #endif
 417         },
 418         [IORING_OP_SENDMSG_ZC] = {
 419                 .needs_file             = 1,
 420                 .unbound_nonreg_file    = 1,
 421                 .pollout                = 1,
 422                 .ioprio                 = 1,
 423                 .manual_alloc           = 1,
 424 #if defined(CONFIG_NET)
 425                 .prep                   = io_send_zc_prep,
 426                 .issue                  = io_sendmsg_zc,
 427 #else
 428                 .prep                   = io_eopnotsupp_prep,
 429 #endif
 430         },
 431 };
 432
 433
 434 const struct io_cold_def io_cold_defs[] = {
 435         [IORING_OP_NOP] = {
 436                 .name                   = "NOP",
 437         },
 438         [IORING_OP_READV] = {
 439                 .async_size             = sizeof(struct io_async_rw),
 440                 .name                   = "READV",
 441                 .prep_async             = io_readv_prep_async,
 442                 .cleanup                = io_readv_writev_cleanup,
 443                 .fail                   = io_rw_fail,
 444         },
 445         [IORING_OP_WRITEV] = {
 446                 .async_size             = sizeof(struct io_async_rw),
 447                 .name                   = "WRITEV",
 448                 .prep_async             = io_writev_prep_async,
 449                 .cleanup                = io_readv_writev_cleanup,
 450                 .fail                   = io_rw_fail,
 451         },
 452         [IORING_OP_FSYNC] = {
 453                 .name                   = "FSYNC",
 454         },
 455         [IORING_OP_READ_FIXED] = {
 456                 .async_size             = sizeof(struct io_async_rw),
 457                 .name                   = "READ_FIXED",
 458                 .fail                   = io_rw_fail,
 459         },
 460         [IORING_OP_WRITE_FIXED] = {
 461                 .async_size             = sizeof(struct io_async_rw),
 462                 .name                   = "WRITE_FIXED",
 463                 .fail                   = io_rw_fail,
 464         },
 465         [IORING_OP_POLL_ADD] = {
 466                 .name                   = "POLL_ADD",
 467         },
 468         [IORING_OP_POLL_REMOVE] = {
 469                 .name                   = "POLL_REMOVE",
 470         },
 471         [IORING_OP_SYNC_FILE_RANGE] = {
 472                 .name                   = "SYNC_FILE_RANGE",
 473         },
 474         [IORING_OP_SENDMSG] = {
 475                 .name                   = "SENDMSG",
 476 #if defined(CONFIG_NET)
 477                 .async_size             = sizeof(struct io_async_msghdr),
 478                 .prep_async             = io_sendmsg_prep_async,
 479                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 480                 .fail                   = io_sendrecv_fail,
 481 #endif
 482         },
 483         [IORING_OP_RECVMSG] = {
 484                 .name                   = "RECVMSG",
 485 #if defined(CONFIG_NET)
 486                 .async_size             = sizeof(struct io_async_msghdr),
 487                 .prep_async             = io_recvmsg_prep_async,
 488                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 489                 .fail                   = io_sendrecv_fail,
 490 #endif
 491         },
 492         [IORING_OP_TIMEOUT] = {
 493                 .async_size             = sizeof(struct io_timeout_data),
 494                 .name                   = "TIMEOUT",
 495         },
 496         [IORING_OP_TIMEOUT_REMOVE] = {
 497                 .name                   = "TIMEOUT_REMOVE",
 498         },
 499         [IORING_OP_ACCEPT] = {
 500                 .name                   = "ACCEPT",
 501         },
 502         [IORING_OP_ASYNC_CANCEL] = {
 503                 .name                   = "ASYNC_CANCEL",
 504         },
 505         [IORING_OP_LINK_TIMEOUT] = {
 506                 .async_size             = sizeof(struct io_timeout_data),
 507                 .name                   = "LINK_TIMEOUT",
 508         },
 509         [IORING_OP_CONNECT] = {
 510                 .name                   = "CONNECT",
 511 #if defined(CONFIG_NET)
 512                 .async_size             = sizeof(struct io_async_connect),
 513                 .prep_async             = io_connect_prep_async,
 514 #endif
 515         },
 516         [IORING_OP_FALLOCATE] = {
 517                 .name                   = "FALLOCATE",
 518         },
 519         [IORING_OP_OPENAT] = {
 520                 .name                   = "OPENAT",
 521                 .cleanup                = io_open_cleanup,
 522         },
 523         [IORING_OP_CLOSE] = {
 524                 .name                   = "CLOSE",
 525         },
 526         [IORING_OP_FILES_UPDATE] = {
 527                 .name                   = "FILES_UPDATE",
 528         },
 529         [IORING_OP_STATX] = {
 530                 .name                   = "STATX",
 531                 .cleanup                = io_statx_cleanup,
 532         },
 533         [IORING_OP_READ] = {
 534                 .async_size             = sizeof(struct io_async_rw),
 535                 .name                   = "READ",
 536                 .fail                   = io_rw_fail,
 537         },
 538         [IORING_OP_WRITE] = {
 539                 .async_size             = sizeof(struct io_async_rw),
 540                 .name                   = "WRITE",
 541                 .fail                   = io_rw_fail,
 542         },
 543         [IORING_OP_FADVISE] = {
 544                 .name                   = "FADVISE",
 545         },
 546         [IORING_OP_MADVISE] = {
 547                 .name                   = "MADVISE",
 548         },
 549         [IORING_OP_SEND] = {
 550                 .name                   = "SEND",
 551 #if defined(CONFIG_NET)
 552                 .async_size             = sizeof(struct io_async_msghdr),
 553                 .fail                   = io_sendrecv_fail,
 554                 .prep_async             = io_send_prep_async,
 555 #endif
 556         },
 557         [IORING_OP_RECV] = {
 558                 .name                   = "RECV",
 559 #if defined(CONFIG_NET)
 560                 .fail                   = io_sendrecv_fail,
 561 #endif
 562         },
 563         [IORING_OP_OPENAT2] = {
 564                 .name                   = "OPENAT2",
 565                 .cleanup                = io_open_cleanup,
 566         },
 567         [IORING_OP_EPOLL_CTL] = {
 568                 .name                   = "EPOLL",
 569         },
 570         [IORING_OP_SPLICE] = {
 571                 .name                   = "SPLICE",
 572         },
 573         [IORING_OP_PROVIDE_BUFFERS] = {
 574                 .name                   = "PROVIDE_BUFFERS",
 575         },
 576         [IORING_OP_REMOVE_BUFFERS] = {
 577                 .name                   = "REMOVE_BUFFERS",
 578         },
 579         [IORING_OP_TEE] = {
 580                 .name                   = "TEE",
 581         },
 582         [IORING_OP_SHUTDOWN] = {
 583                 .name                   = "SHUTDOWN",
 584         },
 585         [IORING_OP_RENAMEAT] = {
 586                 .name                   = "RENAMEAT",
 587                 .cleanup                = io_renameat_cleanup,
 588         },
 589         [IORING_OP_UNLINKAT] = {
 590                 .name                   = "UNLINKAT",
 591                 .cleanup                = io_unlinkat_cleanup,
 592         },
 593         [IORING_OP_MKDIRAT] = {
 594                 .name                   = "MKDIRAT",
 595                 .cleanup                = io_mkdirat_cleanup,
 596         },
 597         [IORING_OP_SYMLINKAT] = {
 598                 .name                   = "SYMLINKAT",
 599                 .cleanup                = io_link_cleanup,
 600         },
 601         [IORING_OP_LINKAT] = {
 602                 .name                   = "LINKAT",
 603                 .cleanup                = io_link_cleanup,
 604         },
 605         [IORING_OP_MSG_RING] = {
 606                 .name                   = "MSG_RING",
 607                 .cleanup                = io_msg_ring_cleanup,
 608         },
 609         [IORING_OP_FSETXATTR] = {
 610                 .name                   = "FSETXATTR",
 611                 .cleanup                = io_xattr_cleanup,
 612         },
 613         [IORING_OP_SETXATTR] = {
 614                 .name                   = "SETXATTR",
 615                 .cleanup                = io_xattr_cleanup,
 616         },
 617         [IORING_OP_FGETXATTR] = {
 618                 .name                   = "FGETXATTR",
 619                 .cleanup                = io_xattr_cleanup,
 620         },
 621         [IORING_OP_GETXATTR] = {
 622                 .name                   = "GETXATTR",
 623                 .cleanup                = io_xattr_cleanup,
 624         },
 625         [IORING_OP_SOCKET] = {
 626                 .name                   = "SOCKET",
 627         },
 628         [IORING_OP_URING_CMD] = {
 629                 .name                   = "URING_CMD",
 630                 .async_size             = 2 * sizeof(struct io_uring_sqe),
 631                 .prep_async             = io_uring_cmd_prep_async,
 632         },
 633         [IORING_OP_SEND_ZC] = {
 634                 .name                   = "SEND_ZC",
 635 #if defined(CONFIG_NET)
 636                 .async_size             = sizeof(struct io_async_msghdr),
 637                 .prep_async             = io_send_prep_async,
 638                 .cleanup                = io_send_zc_cleanup,
 639                 .fail                   = io_sendrecv_fail,
 640 #endif
 641         },
 642         [IORING_OP_SENDMSG_ZC] = {
 643                 .name                   = "SENDMSG_ZC",
 644 #if defined(CONFIG_NET)
 645                 .async_size             = sizeof(struct io_async_msghdr),
 646                 .prep_async             = io_sendmsg_prep_async,
 647                 .cleanup                = io_send_zc_cleanup,
 648                 .fail                   = io_sendrecv_fail,
 649 #endif
 650         },
 651 };
 652
 653 const char *io_uring_get_opcode(u8 opcode)
 654 {
 655         if (opcode < IORING_OP_LAST)
 656                 return io_cold_defs[opcode].name;
 657         return "INVALID";
 658 }
 659
 660 void __init io_uring_optable_init(void)
 661 {
 662         int i;
 663
 664         BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
 665         BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
 666
 667         for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
 668                 BUG_ON(!io_issue_defs[i].prep);
 669                 if (io_issue_defs[i].prep != io_eopnotsupp_prep)
 670                         BUG_ON(!io_issue_defs[i].issue);
 671                 WARN_ON_ONCE(!io_cold_defs[i].name);
 672         }
 673 }