io_uring/opdef.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * io_uring opcode handling table
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/errno.h>
   7 #include <linux/fs.h>
   8 #include <linux/file.h>
   9 #include <linux/io_uring.h>
  10
  11 #include "io_uring.h"
  12 #include "opdef.h"
  13 #include "refs.h"
  14 #include "tctx.h"
  15 #include "sqpoll.h"
  16 #include "fdinfo.h"
  17 #include "kbuf.h"
  18 #include "rsrc.h"
  19
  20 #include "xattr.h"
  21 #include "nop.h"
  22 #include "fs.h"
  23 #include "splice.h"
  24 #include "sync.h"
  25 #include "advise.h"
  26 #include "openclose.h"
  27 #include "uring_cmd.h"
  28 #include "epoll.h"
  29 #include "statx.h"
  30 #include "net.h"
  31 #include "msg_ring.h"
  32 #include "timeout.h"
  33 #include "poll.h"
  34 #include "cancel.h"
  35 #include "rw.h"
  36 #include "waitid.h"
  37 #include "futex.h"
  38 #include "truncate.h"
  39
  40 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
  41 {
  42         WARN_ON_ONCE(1);
  43         return -ECANCELED;
  44 }
  45
  46 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
  47                                              const struct io_uring_sqe *sqe)
  48 {
  49         return -EOPNOTSUPP;
  50 }
  51
  52 const struct io_issue_def io_issue_defs[] = {
  53         [IORING_OP_NOP] = {
  54                 .audit_skip             = 1,
  55                 .iopoll                 = 1,
  56                 .prep                   = io_nop_prep,
  57                 .issue                  = io_nop,
  58         },
  59         [IORING_OP_READV] = {
  60                 .needs_file             = 1,
  61                 .unbound_nonreg_file    = 1,
  62                 .pollin                 = 1,
  63                 .buffer_select          = 1,
  64                 .plug                   = 1,
  65                 .audit_skip             = 1,
  66                 .ioprio                 = 1,
  67                 .iopoll                 = 1,
  68                 .iopoll_queue           = 1,
  69                 .vectored               = 1,
  70                 .async_size             = sizeof(struct io_async_rw),
  71                 .prep                   = io_prep_readv,
  72                 .issue                  = io_read,
  73         },
  74         [IORING_OP_WRITEV] = {
  75                 .needs_file             = 1,
  76                 .hash_reg_file          = 1,
  77                 .unbound_nonreg_file    = 1,
  78                 .pollout                = 1,
  79                 .plug                   = 1,
  80                 .audit_skip             = 1,
  81                 .ioprio                 = 1,
  82                 .iopoll                 = 1,
  83                 .iopoll_queue           = 1,
  84                 .vectored               = 1,
  85                 .async_size             = sizeof(struct io_async_rw),
  86                 .prep                   = io_prep_writev,
  87                 .issue                  = io_write,
  88         },
  89         [IORING_OP_FSYNC] = {
  90                 .needs_file             = 1,
  91                 .audit_skip             = 1,
  92                 .prep                   = io_fsync_prep,
  93                 .issue                  = io_fsync,
  94         },
  95         [IORING_OP_READ_FIXED] = {
  96                 .needs_file             = 1,
  97                 .unbound_nonreg_file    = 1,
  98                 .pollin                 = 1,
  99                 .plug                   = 1,
 100                 .audit_skip             = 1,
 101                 .ioprio                 = 1,
 102                 .iopoll                 = 1,
 103                 .iopoll_queue           = 1,
 104                 .async_size             = sizeof(struct io_async_rw),
 105                 .prep                   = io_prep_read_fixed,
 106                 .issue                  = io_read,
 107         },
 108         [IORING_OP_WRITE_FIXED] = {
 109                 .needs_file             = 1,
 110                 .hash_reg_file          = 1,
 111                 .unbound_nonreg_file    = 1,
 112                 .pollout                = 1,
 113                 .plug                   = 1,
 114                 .audit_skip             = 1,
 115                 .ioprio                 = 1,
 116                 .iopoll                 = 1,
 117                 .iopoll_queue           = 1,
 118                 .async_size             = sizeof(struct io_async_rw),
 119                 .prep                   = io_prep_write_fixed,
 120                 .issue                  = io_write,
 121         },
 122         [IORING_OP_POLL_ADD] = {
 123                 .needs_file             = 1,
 124                 .unbound_nonreg_file    = 1,
 125                 .audit_skip             = 1,
 126                 .prep                   = io_poll_add_prep,
 127                 .issue                  = io_poll_add,
 128         },
 129         [IORING_OP_POLL_REMOVE] = {
 130                 .audit_skip             = 1,
 131                 .prep                   = io_poll_remove_prep,
 132                 .issue                  = io_poll_remove,
 133         },
 134         [IORING_OP_SYNC_FILE_RANGE] = {
 135                 .needs_file             = 1,
 136                 .audit_skip             = 1,
 137                 .prep                   = io_sfr_prep,
 138                 .issue                  = io_sync_file_range,
 139         },
 140         [IORING_OP_SENDMSG] = {
 141                 .needs_file             = 1,
 142                 .unbound_nonreg_file    = 1,
 143                 .pollout                = 1,
 144                 .ioprio                 = 1,
 145 #if defined(CONFIG_NET)
 146                 .async_size             = sizeof(struct io_async_msghdr),
 147                 .prep                   = io_sendmsg_prep,
 148                 .issue                  = io_sendmsg,
 149 #else
 150                 .prep                   = io_eopnotsupp_prep,
 151 #endif
 152         },
 153         [IORING_OP_RECVMSG] = {
 154                 .needs_file             = 1,
 155                 .unbound_nonreg_file    = 1,
 156                 .pollin                 = 1,
 157                 .buffer_select          = 1,
 158                 .ioprio                 = 1,
 159 #if defined(CONFIG_NET)
 160                 .async_size             = sizeof(struct io_async_msghdr),
 161                 .prep                   = io_recvmsg_prep,
 162                 .issue                  = io_recvmsg,
 163 #else
 164                 .prep                   = io_eopnotsupp_prep,
 165 #endif
 166         },
 167         [IORING_OP_TIMEOUT] = {
 168                 .audit_skip             = 1,
 169                 .async_size             = sizeof(struct io_timeout_data),
 170                 .prep                   = io_timeout_prep,
 171                 .issue                  = io_timeout,
 172         },
 173         [IORING_OP_TIMEOUT_REMOVE] = {
 174                 /* used by timeout updates' prep() */
 175                 .audit_skip             = 1,
 176                 .prep                   = io_timeout_remove_prep,
 177                 .issue                  = io_timeout_remove,
 178         },
 179         [IORING_OP_ACCEPT] = {
 180                 .needs_file             = 1,
 181                 .unbound_nonreg_file    = 1,
 182                 .pollin                 = 1,
 183                 .poll_exclusive         = 1,
 184                 .ioprio                 = 1,    /* used for flags */
 185 #if defined(CONFIG_NET)
 186                 .prep                   = io_accept_prep,
 187                 .issue                  = io_accept,
 188 #else
 189                 .prep                   = io_eopnotsupp_prep,
 190 #endif
 191         },
 192         [IORING_OP_ASYNC_CANCEL] = {
 193                 .audit_skip             = 1,
 194                 .prep                   = io_async_cancel_prep,
 195                 .issue                  = io_async_cancel,
 196         },
 197         [IORING_OP_LINK_TIMEOUT] = {
 198                 .audit_skip             = 1,
 199                 .async_size             = sizeof(struct io_timeout_data),
 200                 .prep                   = io_link_timeout_prep,
 201                 .issue                  = io_no_issue,
 202         },
 203         [IORING_OP_CONNECT] = {
 204                 .needs_file             = 1,
 205                 .unbound_nonreg_file    = 1,
 206                 .pollout                = 1,
 207 #if defined(CONFIG_NET)
 208                 .async_size             = sizeof(struct io_async_msghdr),
 209                 .prep                   = io_connect_prep,
 210                 .issue                  = io_connect,
 211 #else
 212                 .prep                   = io_eopnotsupp_prep,
 213 #endif
 214         },
 215         [IORING_OP_FALLOCATE] = {
 216                 .needs_file             = 1,
 217                 .prep                   = io_fallocate_prep,
 218                 .issue                  = io_fallocate,
 219         },
 220         [IORING_OP_OPENAT] = {
 221                 .prep                   = io_openat_prep,
 222                 .issue                  = io_openat,
 223         },
 224         [IORING_OP_CLOSE] = {
 225                 .prep                   = io_close_prep,
 226                 .issue                  = io_close,
 227         },
 228         [IORING_OP_FILES_UPDATE] = {
 229                 .audit_skip             = 1,
 230                 .iopoll                 = 1,
 231                 .prep                   = io_files_update_prep,
 232                 .issue                  = io_files_update,
 233         },
 234         [IORING_OP_STATX] = {
 235                 .audit_skip             = 1,
 236                 .prep                   = io_statx_prep,
 237                 .issue                  = io_statx,
 238         },
 239         [IORING_OP_READ] = {
 240                 .needs_file             = 1,
 241                 .unbound_nonreg_file    = 1,
 242                 .pollin                 = 1,
 243                 .buffer_select          = 1,
 244                 .plug                   = 1,
 245                 .audit_skip             = 1,
 246                 .ioprio                 = 1,
 247                 .iopoll                 = 1,
 248                 .iopoll_queue           = 1,
 249                 .async_size             = sizeof(struct io_async_rw),
 250                 .prep                   = io_prep_read,
 251                 .issue                  = io_read,
 252         },
 253         [IORING_OP_WRITE] = {
 254                 .needs_file             = 1,
 255                 .hash_reg_file          = 1,
 256                 .unbound_nonreg_file    = 1,
 257                 .pollout                = 1,
 258                 .plug                   = 1,
 259                 .audit_skip             = 1,
 260                 .ioprio                 = 1,
 261                 .iopoll                 = 1,
 262                 .iopoll_queue           = 1,
 263                 .async_size             = sizeof(struct io_async_rw),
 264                 .prep                   = io_prep_write,
 265                 .issue                  = io_write,
 266         },
 267         [IORING_OP_FADVISE] = {
 268                 .needs_file             = 1,
 269                 .audit_skip             = 1,
 270                 .prep                   = io_fadvise_prep,
 271                 .issue                  = io_fadvise,
 272         },
 273         [IORING_OP_MADVISE] = {
 274                 .audit_skip             = 1,
 275                 .prep                   = io_madvise_prep,
 276                 .issue                  = io_madvise,
 277         },
 278         [IORING_OP_SEND] = {
 279                 .needs_file             = 1,
 280                 .unbound_nonreg_file    = 1,
 281                 .pollout                = 1,
 282                 .audit_skip             = 1,
 283                 .ioprio                 = 1,
 284                 .buffer_select          = 1,
 285 #if defined(CONFIG_NET)
 286                 .async_size             = sizeof(struct io_async_msghdr),
 287                 .prep                   = io_sendmsg_prep,
 288                 .issue                  = io_send,
 289 #else
 290                 .prep                   = io_eopnotsupp_prep,
 291 #endif
 292         },
 293         [IORING_OP_RECV] = {
 294                 .needs_file             = 1,
 295                 .unbound_nonreg_file    = 1,
 296                 .pollin                 = 1,
 297                 .buffer_select          = 1,
 298                 .audit_skip             = 1,
 299                 .ioprio                 = 1,
 300 #if defined(CONFIG_NET)
 301                 .async_size             = sizeof(struct io_async_msghdr),
 302                 .prep                   = io_recvmsg_prep,
 303                 .issue                  = io_recv,
 304 #else
 305                 .prep                   = io_eopnotsupp_prep,
 306 #endif
 307         },
 308         [IORING_OP_OPENAT2] = {
 309                 .prep                   = io_openat2_prep,
 310                 .issue                  = io_openat2,
 311         },
 312         [IORING_OP_EPOLL_CTL] = {
 313                 .unbound_nonreg_file    = 1,
 314                 .audit_skip             = 1,
 315 #if defined(CONFIG_EPOLL)
 316                 .prep                   = io_epoll_ctl_prep,
 317                 .issue                  = io_epoll_ctl,
 318 #else
 319                 .prep                   = io_eopnotsupp_prep,
 320 #endif
 321         },
 322         [IORING_OP_SPLICE] = {
 323                 .needs_file             = 1,
 324                 .hash_reg_file          = 1,
 325                 .unbound_nonreg_file    = 1,
 326                 .audit_skip             = 1,
 327                 .prep                   = io_splice_prep,
 328                 .issue                  = io_splice,
 329         },
 330         [IORING_OP_PROVIDE_BUFFERS] = {
 331                 .audit_skip             = 1,
 332                 .iopoll                 = 1,
 333                 .prep                   = io_provide_buffers_prep,
 334                 .issue                  = io_provide_buffers,
 335         },
 336         [IORING_OP_REMOVE_BUFFERS] = {
 337                 .audit_skip             = 1,
 338                 .iopoll                 = 1,
 339                 .prep                   = io_remove_buffers_prep,
 340                 .issue                  = io_remove_buffers,
 341         },
 342         [IORING_OP_TEE] = {
 343                 .needs_file             = 1,
 344                 .hash_reg_file          = 1,
 345                 .unbound_nonreg_file    = 1,
 346                 .audit_skip             = 1,
 347                 .prep                   = io_tee_prep,
 348                 .issue                  = io_tee,
 349         },
 350         [IORING_OP_SHUTDOWN] = {
 351                 .needs_file             = 1,
 352 #if defined(CONFIG_NET)
 353                 .prep                   = io_shutdown_prep,
 354                 .issue                  = io_shutdown,
 355 #else
 356                 .prep                   = io_eopnotsupp_prep,
 357 #endif
 358         },
 359         [IORING_OP_RENAMEAT] = {
 360                 .prep                   = io_renameat_prep,
 361                 .issue                  = io_renameat,
 362         },
 363         [IORING_OP_UNLINKAT] = {
 364                 .prep                   = io_unlinkat_prep,
 365                 .issue                  = io_unlinkat,
 366         },
 367         [IORING_OP_MKDIRAT] = {
 368                 .prep                   = io_mkdirat_prep,
 369                 .issue                  = io_mkdirat,
 370         },
 371         [IORING_OP_SYMLINKAT] = {
 372                 .prep                   = io_symlinkat_prep,
 373                 .issue                  = io_symlinkat,
 374         },
 375         [IORING_OP_LINKAT] = {
 376                 .prep                   = io_linkat_prep,
 377                 .issue                  = io_linkat,
 378         },
 379         [IORING_OP_MSG_RING] = {
 380                 .needs_file             = 1,
 381                 .iopoll                 = 1,
 382                 .prep                   = io_msg_ring_prep,
 383                 .issue                  = io_msg_ring,
 384         },
 385         [IORING_OP_FSETXATTR] = {
 386                 .needs_file = 1,
 387                 .prep                   = io_fsetxattr_prep,
 388                 .issue                  = io_fsetxattr,
 389         },
 390         [IORING_OP_SETXATTR] = {
 391                 .prep                   = io_setxattr_prep,
 392                 .issue                  = io_setxattr,
 393         },
 394         [IORING_OP_FGETXATTR] = {
 395                 .needs_file = 1,
 396                 .prep                   = io_fgetxattr_prep,
 397                 .issue                  = io_fgetxattr,
 398         },
 399         [IORING_OP_GETXATTR] = {
 400                 .prep                   = io_getxattr_prep,
 401                 .issue                  = io_getxattr,
 402         },
 403         [IORING_OP_SOCKET] = {
 404                 .audit_skip             = 1,
 405 #if defined(CONFIG_NET)
 406                 .prep                   = io_socket_prep,
 407                 .issue                  = io_socket,
 408 #else
 409                 .prep                   = io_eopnotsupp_prep,
 410 #endif
 411         },
 412         [IORING_OP_URING_CMD] = {
 413                 .needs_file             = 1,
 414                 .plug                   = 1,
 415                 .iopoll                 = 1,
 416                 .iopoll_queue           = 1,
 417                 .async_size             = 2 * sizeof(struct io_uring_sqe),
 418                 .prep                   = io_uring_cmd_prep,
 419                 .issue                  = io_uring_cmd,
 420         },
 421         [IORING_OP_SEND_ZC] = {
 422                 .needs_file             = 1,
 423                 .unbound_nonreg_file    = 1,
 424                 .pollout                = 1,
 425                 .audit_skip             = 1,
 426                 .ioprio                 = 1,
 427 #if defined(CONFIG_NET)
 428                 .async_size             = sizeof(struct io_async_msghdr),
 429                 .prep                   = io_send_zc_prep,
 430                 .issue                  = io_send_zc,
 431 #else
 432                 .prep                   = io_eopnotsupp_prep,
 433 #endif
 434         },
 435         [IORING_OP_SENDMSG_ZC] = {
 436                 .needs_file             = 1,
 437                 .unbound_nonreg_file    = 1,
 438                 .pollout                = 1,
 439                 .ioprio                 = 1,
 440 #if defined(CONFIG_NET)
 441                 .async_size             = sizeof(struct io_async_msghdr),
 442                 .prep                   = io_send_zc_prep,
 443                 .issue                  = io_sendmsg_zc,
 444 #else
 445                 .prep                   = io_eopnotsupp_prep,
 446 #endif
 447         },
 448         [IORING_OP_READ_MULTISHOT] = {
 449                 .needs_file             = 1,
 450                 .unbound_nonreg_file    = 1,
 451                 .pollin                 = 1,
 452                 .buffer_select          = 1,
 453                 .audit_skip             = 1,
 454                 .async_size             = sizeof(struct io_async_rw),
 455                 .prep                   = io_read_mshot_prep,
 456                 .issue                  = io_read_mshot,
 457         },
 458         [IORING_OP_WAITID] = {
 459                 .async_size             = sizeof(struct io_waitid_async),
 460                 .prep                   = io_waitid_prep,
 461                 .issue                  = io_waitid,
 462         },
 463         [IORING_OP_FUTEX_WAIT] = {
 464 #if defined(CONFIG_FUTEX)
 465                 .prep                   = io_futex_prep,
 466                 .issue                  = io_futex_wait,
 467 #else
 468                 .prep                   = io_eopnotsupp_prep,
 469 #endif
 470         },
 471         [IORING_OP_FUTEX_WAKE] = {
 472 #if defined(CONFIG_FUTEX)
 473                 .prep                   = io_futex_prep,
 474                 .issue                  = io_futex_wake,
 475 #else
 476                 .prep                   = io_eopnotsupp_prep,
 477 #endif
 478         },
 479         [IORING_OP_FUTEX_WAITV] = {
 480 #if defined(CONFIG_FUTEX)
 481                 .prep                   = io_futexv_prep,
 482                 .issue                  = io_futexv_wait,
 483 #else
 484                 .prep                   = io_eopnotsupp_prep,
 485 #endif
 486         },
 487         [IORING_OP_FIXED_FD_INSTALL] = {
 488                 .needs_file             = 1,
 489                 .prep                   = io_install_fixed_fd_prep,
 490                 .issue                  = io_install_fixed_fd,
 491         },
 492         [IORING_OP_FTRUNCATE] = {
 493                 .needs_file             = 1,
 494                 .hash_reg_file          = 1,
 495                 .prep                   = io_ftruncate_prep,
 496                 .issue                  = io_ftruncate,
 497         },
 498         [IORING_OP_BIND] = {
 499 #if defined(CONFIG_NET)
 500                 .needs_file             = 1,
 501                 .prep                   = io_bind_prep,
 502                 .issue                  = io_bind,
 503                 .async_size             = sizeof(struct io_async_msghdr),
 504 #else
 505                 .prep                   = io_eopnotsupp_prep,
 506 #endif
 507         },
 508         [IORING_OP_LISTEN] = {
 509 #if defined(CONFIG_NET)
 510                 .needs_file             = 1,
 511                 .prep                   = io_listen_prep,
 512                 .issue                  = io_listen,
 513                 .async_size             = sizeof(struct io_async_msghdr),
 514 #else
 515                 .prep                   = io_eopnotsupp_prep,
 516 #endif
 517         },
 518 };
 519
 520 const struct io_cold_def io_cold_defs[] = {
 521         [IORING_OP_NOP] = {
 522                 .name                   = "NOP",
 523         },
 524         [IORING_OP_READV] = {
 525                 .name                   = "READV",
 526                 .cleanup                = io_readv_writev_cleanup,
 527                 .fail                   = io_rw_fail,
 528         },
 529         [IORING_OP_WRITEV] = {
 530                 .name                   = "WRITEV",
 531                 .cleanup                = io_readv_writev_cleanup,
 532                 .fail                   = io_rw_fail,
 533         },
 534         [IORING_OP_FSYNC] = {
 535                 .name                   = "FSYNC",
 536         },
 537         [IORING_OP_READ_FIXED] = {
 538                 .name                   = "READ_FIXED",
 539                 .cleanup                = io_readv_writev_cleanup,
 540                 .fail                   = io_rw_fail,
 541         },
 542         [IORING_OP_WRITE_FIXED] = {
 543                 .name                   = "WRITE_FIXED",
 544                 .cleanup                = io_readv_writev_cleanup,
 545                 .fail                   = io_rw_fail,
 546         },
 547         [IORING_OP_POLL_ADD] = {
 548                 .name                   = "POLL_ADD",
 549         },
 550         [IORING_OP_POLL_REMOVE] = {
 551                 .name                   = "POLL_REMOVE",
 552         },
 553         [IORING_OP_SYNC_FILE_RANGE] = {
 554                 .name                   = "SYNC_FILE_RANGE",
 555         },
 556         [IORING_OP_SENDMSG] = {
 557                 .name                   = "SENDMSG",
 558 #if defined(CONFIG_NET)
 559                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 560                 .fail                   = io_sendrecv_fail,
 561 #endif
 562         },
 563         [IORING_OP_RECVMSG] = {
 564                 .name                   = "RECVMSG",
 565 #if defined(CONFIG_NET)
 566                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 567                 .fail                   = io_sendrecv_fail,
 568 #endif
 569         },
 570         [IORING_OP_TIMEOUT] = {
 571                 .name                   = "TIMEOUT",
 572         },
 573         [IORING_OP_TIMEOUT_REMOVE] = {
 574                 .name                   = "TIMEOUT_REMOVE",
 575         },
 576         [IORING_OP_ACCEPT] = {
 577                 .name                   = "ACCEPT",
 578         },
 579         [IORING_OP_ASYNC_CANCEL] = {
 580                 .name                   = "ASYNC_CANCEL",
 581         },
 582         [IORING_OP_LINK_TIMEOUT] = {
 583                 .name                   = "LINK_TIMEOUT",
 584         },
 585         [IORING_OP_CONNECT] = {
 586                 .name                   = "CONNECT",
 587         },
 588         [IORING_OP_FALLOCATE] = {
 589                 .name                   = "FALLOCATE",
 590         },
 591         [IORING_OP_OPENAT] = {
 592                 .name                   = "OPENAT",
 593                 .cleanup                = io_open_cleanup,
 594         },
 595         [IORING_OP_CLOSE] = {
 596                 .name                   = "CLOSE",
 597         },
 598         [IORING_OP_FILES_UPDATE] = {
 599                 .name                   = "FILES_UPDATE",
 600         },
 601         [IORING_OP_STATX] = {
 602                 .name                   = "STATX",
 603                 .cleanup                = io_statx_cleanup,
 604         },
 605         [IORING_OP_READ] = {
 606                 .name                   = "READ",
 607                 .cleanup                = io_readv_writev_cleanup,
 608                 .fail                   = io_rw_fail,
 609         },
 610         [IORING_OP_WRITE] = {
 611                 .name                   = "WRITE",
 612                 .cleanup                = io_readv_writev_cleanup,
 613                 .fail                   = io_rw_fail,
 614         },
 615         [IORING_OP_FADVISE] = {
 616                 .name                   = "FADVISE",
 617         },
 618         [IORING_OP_MADVISE] = {
 619                 .name                   = "MADVISE",
 620         },
 621         [IORING_OP_SEND] = {
 622                 .name                   = "SEND",
 623 #if defined(CONFIG_NET)
 624                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 625                 .fail                   = io_sendrecv_fail,
 626 #endif
 627         },
 628         [IORING_OP_RECV] = {
 629                 .name                   = "RECV",
 630 #if defined(CONFIG_NET)
 631                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 632                 .fail                   = io_sendrecv_fail,
 633 #endif
 634         },
 635         [IORING_OP_OPENAT2] = {
 636                 .name                   = "OPENAT2",
 637                 .cleanup                = io_open_cleanup,
 638         },
 639         [IORING_OP_EPOLL_CTL] = {
 640                 .name                   = "EPOLL",
 641         },
 642         [IORING_OP_SPLICE] = {
 643                 .name                   = "SPLICE",
 644         },
 645         [IORING_OP_PROVIDE_BUFFERS] = {
 646                 .name                   = "PROVIDE_BUFFERS",
 647         },
 648         [IORING_OP_REMOVE_BUFFERS] = {
 649                 .name                   = "REMOVE_BUFFERS",
 650         },
 651         [IORING_OP_TEE] = {
 652                 .name                   = "TEE",
 653         },
 654         [IORING_OP_SHUTDOWN] = {
 655                 .name                   = "SHUTDOWN",
 656         },
 657         [IORING_OP_RENAMEAT] = {
 658                 .name                   = "RENAMEAT",
 659                 .cleanup                = io_renameat_cleanup,
 660         },
 661         [IORING_OP_UNLINKAT] = {
 662                 .name                   = "UNLINKAT",
 663                 .cleanup                = io_unlinkat_cleanup,
 664         },
 665         [IORING_OP_MKDIRAT] = {
 666                 .name                   = "MKDIRAT",
 667                 .cleanup                = io_mkdirat_cleanup,
 668         },
 669         [IORING_OP_SYMLINKAT] = {
 670                 .name                   = "SYMLINKAT",
 671                 .cleanup                = io_link_cleanup,
 672         },
 673         [IORING_OP_LINKAT] = {
 674                 .name                   = "LINKAT",
 675                 .cleanup                = io_link_cleanup,
 676         },
 677         [IORING_OP_MSG_RING] = {
 678                 .name                   = "MSG_RING",
 679                 .cleanup                = io_msg_ring_cleanup,
 680         },
 681         [IORING_OP_FSETXATTR] = {
 682                 .name                   = "FSETXATTR",
 683                 .cleanup                = io_xattr_cleanup,
 684         },
 685         [IORING_OP_SETXATTR] = {
 686                 .name                   = "SETXATTR",
 687                 .cleanup                = io_xattr_cleanup,
 688         },
 689         [IORING_OP_FGETXATTR] = {
 690                 .name                   = "FGETXATTR",
 691                 .cleanup                = io_xattr_cleanup,
 692         },
 693         [IORING_OP_GETXATTR] = {
 694                 .name                   = "GETXATTR",
 695                 .cleanup                = io_xattr_cleanup,
 696         },
 697         [IORING_OP_SOCKET] = {
 698                 .name                   = "SOCKET",
 699         },
 700         [IORING_OP_URING_CMD] = {
 701                 .name                   = "URING_CMD",
 702         },
 703         [IORING_OP_SEND_ZC] = {
 704                 .name                   = "SEND_ZC",
 705 #if defined(CONFIG_NET)
 706                 .cleanup                = io_send_zc_cleanup,
 707                 .fail                   = io_sendrecv_fail,
 708 #endif
 709         },
 710         [IORING_OP_SENDMSG_ZC] = {
 711                 .name                   = "SENDMSG_ZC",
 712 #if defined(CONFIG_NET)
 713                 .cleanup                = io_send_zc_cleanup,
 714                 .fail                   = io_sendrecv_fail,
 715 #endif
 716         },
 717         [IORING_OP_READ_MULTISHOT] = {
 718                 .name                   = "READ_MULTISHOT",
 719                 .cleanup                = io_readv_writev_cleanup,
 720         },
 721         [IORING_OP_WAITID] = {
 722                 .name                   = "WAITID",
 723         },
 724         [IORING_OP_FUTEX_WAIT] = {
 725                 .name                   = "FUTEX_WAIT",
 726         },
 727         [IORING_OP_FUTEX_WAKE] = {
 728                 .name                   = "FUTEX_WAKE",
 729         },
 730         [IORING_OP_FUTEX_WAITV] = {
 731                 .name                   = "FUTEX_WAITV",
 732         },
 733         [IORING_OP_FIXED_FD_INSTALL] = {
 734                 .name                   = "FIXED_FD_INSTALL",
 735         },
 736         [IORING_OP_FTRUNCATE] = {
 737                 .name                   = "FTRUNCATE",
 738         },
 739         [IORING_OP_BIND] = {
 740                 .name                   = "BIND",
 741         },
 742         [IORING_OP_LISTEN] = {
 743                 .name                   = "LISTEN",
 744         },
 745 };
 746
 747 const char *io_uring_get_opcode(u8 opcode)
 748 {
 749         if (opcode < IORING_OP_LAST)
 750                 return io_cold_defs[opcode].name;
 751         return "INVALID";
 752 }
 753
 754 bool io_uring_op_supported(u8 opcode)
 755 {
 756         if (opcode < IORING_OP_LAST &&
 757             io_issue_defs[opcode].prep != io_eopnotsupp_prep)
 758                 return true;
 759         return false;
 760 }
 761
 762 void __init io_uring_optable_init(void)
 763 {
 764         int i;
 765
 766         BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
 767         BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
 768
 769         for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
 770                 BUG_ON(!io_issue_defs[i].prep);
 771                 if (io_issue_defs[i].prep != io_eopnotsupp_prep)
 772                         BUG_ON(!io_issue_defs[i].issue);
 773                 WARN_ON_ONCE(!io_cold_defs[i].name);
 774         }
 775 }