io_uring/opdef.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * io_uring opcode handling table
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/errno.h>
   7 #include <linux/fs.h>
   8 #include <linux/file.h>
   9 #include <linux/io_uring.h>
  10 #include <linux/io_uring/cmd.h>
  11
  12 #include "io_uring.h"
  13 #include "opdef.h"
  14 #include "refs.h"
  15 #include "tctx.h"
  16 #include "sqpoll.h"
  17 #include "fdinfo.h"
  18 #include "kbuf.h"
  19 #include "rsrc.h"
  20
  21 #include "xattr.h"
  22 #include "nop.h"
  23 #include "fs.h"
  24 #include "splice.h"
  25 #include "sync.h"
  26 #include "advise.h"
  27 #include "openclose.h"
  28 #include "uring_cmd.h"
  29 #include "epoll.h"
  30 #include "statx.h"
  31 #include "net.h"
  32 #include "msg_ring.h"
  33 #include "timeout.h"
  34 #include "poll.h"
  35 #include "cancel.h"
  36 #include "rw.h"
  37 #include "waitid.h"
  38 #include "futex.h"
  39 #include "truncate.h"
  40
  41 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
  42 {
  43         WARN_ON_ONCE(1);
  44         return -ECANCELED;
  45 }
  46
  47 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
  48                                              const struct io_uring_sqe *sqe)
  49 {
  50         return -EOPNOTSUPP;
  51 }
  52
  53 const struct io_issue_def io_issue_defs[] = {
  54         [IORING_OP_NOP] = {
  55                 .audit_skip             = 1,
  56                 .iopoll                 = 1,
  57                 .prep                   = io_nop_prep,
  58                 .issue                  = io_nop,
  59         },
  60         [IORING_OP_READV] = {
  61                 .needs_file             = 1,
  62                 .unbound_nonreg_file    = 1,
  63                 .pollin                 = 1,
  64                 .buffer_select          = 1,
  65                 .plug                   = 1,
  66                 .audit_skip             = 1,
  67                 .ioprio                 = 1,
  68                 .iopoll                 = 1,
  69                 .iopoll_queue           = 1,
  70                 .vectored               = 1,
  71                 .async_size             = sizeof(struct io_async_rw),
  72                 .prep                   = io_prep_readv,
  73                 .issue                  = io_read,
  74         },
  75         [IORING_OP_WRITEV] = {
  76                 .needs_file             = 1,
  77                 .hash_reg_file          = 1,
  78                 .unbound_nonreg_file    = 1,
  79                 .pollout                = 1,
  80                 .plug                   = 1,
  81                 .audit_skip             = 1,
  82                 .ioprio                 = 1,
  83                 .iopoll                 = 1,
  84                 .iopoll_queue           = 1,
  85                 .vectored               = 1,
  86                 .async_size             = sizeof(struct io_async_rw),
  87                 .prep                   = io_prep_writev,
  88                 .issue                  = io_write,
  89         },
  90         [IORING_OP_FSYNC] = {
  91                 .needs_file             = 1,
  92                 .audit_skip             = 1,
  93                 .prep                   = io_fsync_prep,
  94                 .issue                  = io_fsync,
  95         },
  96         [IORING_OP_READ_FIXED] = {
  97                 .needs_file             = 1,
  98                 .unbound_nonreg_file    = 1,
  99                 .pollin                 = 1,
 100                 .plug                   = 1,
 101                 .audit_skip             = 1,
 102                 .ioprio                 = 1,
 103                 .iopoll                 = 1,
 104                 .iopoll_queue           = 1,
 105                 .async_size             = sizeof(struct io_async_rw),
 106                 .prep                   = io_prep_read_fixed,
 107                 .issue                  = io_read,
 108         },
 109         [IORING_OP_WRITE_FIXED] = {
 110                 .needs_file             = 1,
 111                 .hash_reg_file          = 1,
 112                 .unbound_nonreg_file    = 1,
 113                 .pollout                = 1,
 114                 .plug                   = 1,
 115                 .audit_skip             = 1,
 116                 .ioprio                 = 1,
 117                 .iopoll                 = 1,
 118                 .iopoll_queue           = 1,
 119                 .async_size             = sizeof(struct io_async_rw),
 120                 .prep                   = io_prep_write_fixed,
 121                 .issue                  = io_write,
 122         },
 123         [IORING_OP_POLL_ADD] = {
 124                 .needs_file             = 1,
 125                 .unbound_nonreg_file    = 1,
 126                 .audit_skip             = 1,
 127                 .prep                   = io_poll_add_prep,
 128                 .issue                  = io_poll_add,
 129         },
 130         [IORING_OP_POLL_REMOVE] = {
 131                 .audit_skip             = 1,
 132                 .prep                   = io_poll_remove_prep,
 133                 .issue                  = io_poll_remove,
 134         },
 135         [IORING_OP_SYNC_FILE_RANGE] = {
 136                 .needs_file             = 1,
 137                 .audit_skip             = 1,
 138                 .prep                   = io_sfr_prep,
 139                 .issue                  = io_sync_file_range,
 140         },
 141         [IORING_OP_SENDMSG] = {
 142                 .needs_file             = 1,
 143                 .unbound_nonreg_file    = 1,
 144                 .pollout                = 1,
 145                 .ioprio                 = 1,
 146 #if defined(CONFIG_NET)
 147                 .async_size             = sizeof(struct io_async_msghdr),
 148                 .prep                   = io_sendmsg_prep,
 149                 .issue                  = io_sendmsg,
 150 #else
 151                 .prep                   = io_eopnotsupp_prep,
 152 #endif
 153         },
 154         [IORING_OP_RECVMSG] = {
 155                 .needs_file             = 1,
 156                 .unbound_nonreg_file    = 1,
 157                 .pollin                 = 1,
 158                 .buffer_select          = 1,
 159                 .ioprio                 = 1,
 160 #if defined(CONFIG_NET)
 161                 .async_size             = sizeof(struct io_async_msghdr),
 162                 .prep                   = io_recvmsg_prep,
 163                 .issue                  = io_recvmsg,
 164 #else
 165                 .prep                   = io_eopnotsupp_prep,
 166 #endif
 167         },
 168         [IORING_OP_TIMEOUT] = {
 169                 .audit_skip             = 1,
 170                 .async_size             = sizeof(struct io_timeout_data),
 171                 .prep                   = io_timeout_prep,
 172                 .issue                  = io_timeout,
 173         },
 174         [IORING_OP_TIMEOUT_REMOVE] = {
 175                 /* used by timeout updates' prep() */
 176                 .audit_skip             = 1,
 177                 .prep                   = io_timeout_remove_prep,
 178                 .issue                  = io_timeout_remove,
 179         },
 180         [IORING_OP_ACCEPT] = {
 181                 .needs_file             = 1,
 182                 .unbound_nonreg_file    = 1,
 183                 .pollin                 = 1,
 184                 .poll_exclusive         = 1,
 185                 .ioprio                 = 1,    /* used for flags */
 186 #if defined(CONFIG_NET)
 187                 .prep                   = io_accept_prep,
 188                 .issue                  = io_accept,
 189 #else
 190                 .prep                   = io_eopnotsupp_prep,
 191 #endif
 192         },
 193         [IORING_OP_ASYNC_CANCEL] = {
 194                 .audit_skip             = 1,
 195                 .prep                   = io_async_cancel_prep,
 196                 .issue                  = io_async_cancel,
 197         },
 198         [IORING_OP_LINK_TIMEOUT] = {
 199                 .audit_skip             = 1,
 200                 .async_size             = sizeof(struct io_timeout_data),
 201                 .prep                   = io_link_timeout_prep,
 202                 .issue                  = io_no_issue,
 203         },
 204         [IORING_OP_CONNECT] = {
 205                 .needs_file             = 1,
 206                 .unbound_nonreg_file    = 1,
 207                 .pollout                = 1,
 208 #if defined(CONFIG_NET)
 209                 .async_size             = sizeof(struct io_async_msghdr),
 210                 .prep                   = io_connect_prep,
 211                 .issue                  = io_connect,
 212 #else
 213                 .prep                   = io_eopnotsupp_prep,
 214 #endif
 215         },
 216         [IORING_OP_FALLOCATE] = {
 217                 .needs_file             = 1,
 218                 .prep                   = io_fallocate_prep,
 219                 .issue                  = io_fallocate,
 220         },
 221         [IORING_OP_OPENAT] = {
 222                 .prep                   = io_openat_prep,
 223                 .issue                  = io_openat,
 224         },
 225         [IORING_OP_CLOSE] = {
 226                 .prep                   = io_close_prep,
 227                 .issue                  = io_close,
 228         },
 229         [IORING_OP_FILES_UPDATE] = {
 230                 .audit_skip             = 1,
 231                 .iopoll                 = 1,
 232                 .prep                   = io_files_update_prep,
 233                 .issue                  = io_files_update,
 234         },
 235         [IORING_OP_STATX] = {
 236                 .audit_skip             = 1,
 237                 .prep                   = io_statx_prep,
 238                 .issue                  = io_statx,
 239         },
 240         [IORING_OP_READ] = {
 241                 .needs_file             = 1,
 242                 .unbound_nonreg_file    = 1,
 243                 .pollin                 = 1,
 244                 .buffer_select          = 1,
 245                 .plug                   = 1,
 246                 .audit_skip             = 1,
 247                 .ioprio                 = 1,
 248                 .iopoll                 = 1,
 249                 .iopoll_queue           = 1,
 250                 .async_size             = sizeof(struct io_async_rw),
 251                 .prep                   = io_prep_read,
 252                 .issue                  = io_read,
 253         },
 254         [IORING_OP_WRITE] = {
 255                 .needs_file             = 1,
 256                 .hash_reg_file          = 1,
 257                 .unbound_nonreg_file    = 1,
 258                 .pollout                = 1,
 259                 .plug                   = 1,
 260                 .audit_skip             = 1,
 261                 .ioprio                 = 1,
 262                 .iopoll                 = 1,
 263                 .iopoll_queue           = 1,
 264                 .async_size             = sizeof(struct io_async_rw),
 265                 .prep                   = io_prep_write,
 266                 .issue                  = io_write,
 267         },
 268         [IORING_OP_FADVISE] = {
 269                 .needs_file             = 1,
 270                 .audit_skip             = 1,
 271                 .prep                   = io_fadvise_prep,
 272                 .issue                  = io_fadvise,
 273         },
 274         [IORING_OP_MADVISE] = {
 275                 .audit_skip             = 1,
 276                 .prep                   = io_madvise_prep,
 277                 .issue                  = io_madvise,
 278         },
 279         [IORING_OP_SEND] = {
 280                 .needs_file             = 1,
 281                 .unbound_nonreg_file    = 1,
 282                 .pollout                = 1,
 283                 .audit_skip             = 1,
 284                 .ioprio                 = 1,
 285                 .buffer_select          = 1,
 286 #if defined(CONFIG_NET)
 287                 .async_size             = sizeof(struct io_async_msghdr),
 288                 .prep                   = io_sendmsg_prep,
 289                 .issue                  = io_send,
 290 #else
 291                 .prep                   = io_eopnotsupp_prep,
 292 #endif
 293         },
 294         [IORING_OP_RECV] = {
 295                 .needs_file             = 1,
 296                 .unbound_nonreg_file    = 1,
 297                 .pollin                 = 1,
 298                 .buffer_select          = 1,
 299                 .audit_skip             = 1,
 300                 .ioprio                 = 1,
 301 #if defined(CONFIG_NET)
 302                 .async_size             = sizeof(struct io_async_msghdr),
 303                 .prep                   = io_recvmsg_prep,
 304                 .issue                  = io_recv,
 305 #else
 306                 .prep                   = io_eopnotsupp_prep,
 307 #endif
 308         },
 309         [IORING_OP_OPENAT2] = {
 310                 .prep                   = io_openat2_prep,
 311                 .issue                  = io_openat2,
 312         },
 313         [IORING_OP_EPOLL_CTL] = {
 314                 .unbound_nonreg_file    = 1,
 315                 .audit_skip             = 1,
 316 #if defined(CONFIG_EPOLL)
 317                 .prep                   = io_epoll_ctl_prep,
 318                 .issue                  = io_epoll_ctl,
 319 #else
 320                 .prep                   = io_eopnotsupp_prep,
 321 #endif
 322         },
 323         [IORING_OP_SPLICE] = {
 324                 .needs_file             = 1,
 325                 .hash_reg_file          = 1,
 326                 .unbound_nonreg_file    = 1,
 327                 .audit_skip             = 1,
 328                 .prep                   = io_splice_prep,
 329                 .issue                  = io_splice,
 330         },
 331         [IORING_OP_PROVIDE_BUFFERS] = {
 332                 .audit_skip             = 1,
 333                 .iopoll                 = 1,
 334                 .prep                   = io_provide_buffers_prep,
 335                 .issue                  = io_provide_buffers,
 336         },
 337         [IORING_OP_REMOVE_BUFFERS] = {
 338                 .audit_skip             = 1,
 339                 .iopoll                 = 1,
 340                 .prep                   = io_remove_buffers_prep,
 341                 .issue                  = io_remove_buffers,
 342         },
 343         [IORING_OP_TEE] = {
 344                 .needs_file             = 1,
 345                 .hash_reg_file          = 1,
 346                 .unbound_nonreg_file    = 1,
 347                 .audit_skip             = 1,
 348                 .prep                   = io_tee_prep,
 349                 .issue                  = io_tee,
 350         },
 351         [IORING_OP_SHUTDOWN] = {
 352                 .needs_file             = 1,
 353 #if defined(CONFIG_NET)
 354                 .prep                   = io_shutdown_prep,
 355                 .issue                  = io_shutdown,
 356 #else
 357                 .prep                   = io_eopnotsupp_prep,
 358 #endif
 359         },
 360         [IORING_OP_RENAMEAT] = {
 361                 .prep                   = io_renameat_prep,
 362                 .issue                  = io_renameat,
 363         },
 364         [IORING_OP_UNLINKAT] = {
 365                 .prep                   = io_unlinkat_prep,
 366                 .issue                  = io_unlinkat,
 367         },
 368         [IORING_OP_MKDIRAT] = {
 369                 .prep                   = io_mkdirat_prep,
 370                 .issue                  = io_mkdirat,
 371         },
 372         [IORING_OP_SYMLINKAT] = {
 373                 .prep                   = io_symlinkat_prep,
 374                 .issue                  = io_symlinkat,
 375         },
 376         [IORING_OP_LINKAT] = {
 377                 .prep                   = io_linkat_prep,
 378                 .issue                  = io_linkat,
 379         },
 380         [IORING_OP_MSG_RING] = {
 381                 .needs_file             = 1,
 382                 .iopoll                 = 1,
 383                 .prep                   = io_msg_ring_prep,
 384                 .issue                  = io_msg_ring,
 385         },
 386         [IORING_OP_FSETXATTR] = {
 387                 .needs_file = 1,
 388                 .prep                   = io_fsetxattr_prep,
 389                 .issue                  = io_fsetxattr,
 390         },
 391         [IORING_OP_SETXATTR] = {
 392                 .prep                   = io_setxattr_prep,
 393                 .issue                  = io_setxattr,
 394         },
 395         [IORING_OP_FGETXATTR] = {
 396                 .needs_file = 1,
 397                 .prep                   = io_fgetxattr_prep,
 398                 .issue                  = io_fgetxattr,
 399         },
 400         [IORING_OP_GETXATTR] = {
 401                 .prep                   = io_getxattr_prep,
 402                 .issue                  = io_getxattr,
 403         },
 404         [IORING_OP_SOCKET] = {
 405                 .audit_skip             = 1,
 406 #if defined(CONFIG_NET)
 407                 .prep                   = io_socket_prep,
 408                 .issue                  = io_socket,
 409 #else
 410                 .prep                   = io_eopnotsupp_prep,
 411 #endif
 412         },
 413         [IORING_OP_URING_CMD] = {
 414                 .needs_file             = 1,
 415                 .plug                   = 1,
 416                 .iopoll                 = 1,
 417                 .iopoll_queue           = 1,
 418                 .async_size             = sizeof(struct io_uring_cmd_data),
 419                 .prep                   = io_uring_cmd_prep,
 420                 .issue                  = io_uring_cmd,
 421         },
 422         [IORING_OP_SEND_ZC] = {
 423                 .needs_file             = 1,
 424                 .unbound_nonreg_file    = 1,
 425                 .pollout                = 1,
 426                 .audit_skip             = 1,
 427                 .ioprio                 = 1,
 428 #if defined(CONFIG_NET)
 429                 .async_size             = sizeof(struct io_async_msghdr),
 430                 .prep                   = io_send_zc_prep,
 431                 .issue                  = io_send_zc,
 432 #else
 433                 .prep                   = io_eopnotsupp_prep,
 434 #endif
 435         },
 436         [IORING_OP_SENDMSG_ZC] = {
 437                 .needs_file             = 1,
 438                 .unbound_nonreg_file    = 1,
 439                 .pollout                = 1,
 440                 .ioprio                 = 1,
 441 #if defined(CONFIG_NET)
 442                 .async_size             = sizeof(struct io_async_msghdr),
 443                 .prep                   = io_send_zc_prep,
 444                 .issue                  = io_sendmsg_zc,
 445 #else
 446                 .prep                   = io_eopnotsupp_prep,
 447 #endif
 448         },
 449         [IORING_OP_READ_MULTISHOT] = {
 450                 .needs_file             = 1,
 451                 .unbound_nonreg_file    = 1,
 452                 .pollin                 = 1,
 453                 .buffer_select          = 1,
 454                 .audit_skip             = 1,
 455                 .async_size             = sizeof(struct io_async_rw),
 456                 .prep                   = io_read_mshot_prep,
 457                 .issue                  = io_read_mshot,
 458         },
 459         [IORING_OP_WAITID] = {
 460                 .async_size             = sizeof(struct io_waitid_async),
 461                 .prep                   = io_waitid_prep,
 462                 .issue                  = io_waitid,
 463         },
 464         [IORING_OP_FUTEX_WAIT] = {
 465 #if defined(CONFIG_FUTEX)
 466                 .prep                   = io_futex_prep,
 467                 .issue                  = io_futex_wait,
 468 #else
 469                 .prep                   = io_eopnotsupp_prep,
 470 #endif
 471         },
 472         [IORING_OP_FUTEX_WAKE] = {
 473 #if defined(CONFIG_FUTEX)
 474                 .prep                   = io_futex_prep,
 475                 .issue                  = io_futex_wake,
 476 #else
 477                 .prep                   = io_eopnotsupp_prep,
 478 #endif
 479         },
 480         [IORING_OP_FUTEX_WAITV] = {
 481 #if defined(CONFIG_FUTEX)
 482                 .prep                   = io_futexv_prep,
 483                 .issue                  = io_futexv_wait,
 484 #else
 485                 .prep                   = io_eopnotsupp_prep,
 486 #endif
 487         },
 488         [IORING_OP_FIXED_FD_INSTALL] = {
 489                 .needs_file             = 1,
 490                 .prep                   = io_install_fixed_fd_prep,
 491                 .issue                  = io_install_fixed_fd,
 492         },
 493         [IORING_OP_FTRUNCATE] = {
 494                 .needs_file             = 1,
 495                 .hash_reg_file          = 1,
 496                 .prep                   = io_ftruncate_prep,
 497                 .issue                  = io_ftruncate,
 498         },
 499         [IORING_OP_BIND] = {
 500 #if defined(CONFIG_NET)
 501                 .needs_file             = 1,
 502                 .prep                   = io_bind_prep,
 503                 .issue                  = io_bind,
 504                 .async_size             = sizeof(struct io_async_msghdr),
 505 #else
 506                 .prep                   = io_eopnotsupp_prep,
 507 #endif
 508         },
 509         [IORING_OP_LISTEN] = {
 510 #if defined(CONFIG_NET)
 511                 .needs_file             = 1,
 512                 .prep                   = io_listen_prep,
 513                 .issue                  = io_listen,
 514                 .async_size             = sizeof(struct io_async_msghdr),
 515 #else
 516                 .prep                   = io_eopnotsupp_prep,
 517 #endif
 518         },
 519 };
 520
 521 const struct io_cold_def io_cold_defs[] = {
 522         [IORING_OP_NOP] = {
 523                 .name                   = "NOP",
 524         },
 525         [IORING_OP_READV] = {
 526                 .name                   = "READV",
 527                 .cleanup                = io_readv_writev_cleanup,
 528                 .fail                   = io_rw_fail,
 529         },
 530         [IORING_OP_WRITEV] = {
 531                 .name                   = "WRITEV",
 532                 .cleanup                = io_readv_writev_cleanup,
 533                 .fail                   = io_rw_fail,
 534         },
 535         [IORING_OP_FSYNC] = {
 536                 .name                   = "FSYNC",
 537         },
 538         [IORING_OP_READ_FIXED] = {
 539                 .name                   = "READ_FIXED",
 540                 .cleanup                = io_readv_writev_cleanup,
 541                 .fail                   = io_rw_fail,
 542         },
 543         [IORING_OP_WRITE_FIXED] = {
 544                 .name                   = "WRITE_FIXED",
 545                 .cleanup                = io_readv_writev_cleanup,
 546                 .fail                   = io_rw_fail,
 547         },
 548         [IORING_OP_POLL_ADD] = {
 549                 .name                   = "POLL_ADD",
 550         },
 551         [IORING_OP_POLL_REMOVE] = {
 552                 .name                   = "POLL_REMOVE",
 553         },
 554         [IORING_OP_SYNC_FILE_RANGE] = {
 555                 .name                   = "SYNC_FILE_RANGE",
 556         },
 557         [IORING_OP_SENDMSG] = {
 558                 .name                   = "SENDMSG",
 559 #if defined(CONFIG_NET)
 560                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 561                 .fail                   = io_sendrecv_fail,
 562 #endif
 563         },
 564         [IORING_OP_RECVMSG] = {
 565                 .name                   = "RECVMSG",
 566 #if defined(CONFIG_NET)
 567                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 568                 .fail                   = io_sendrecv_fail,
 569 #endif
 570         },
 571         [IORING_OP_TIMEOUT] = {
 572                 .name                   = "TIMEOUT",
 573         },
 574         [IORING_OP_TIMEOUT_REMOVE] = {
 575                 .name                   = "TIMEOUT_REMOVE",
 576         },
 577         [IORING_OP_ACCEPT] = {
 578                 .name                   = "ACCEPT",
 579         },
 580         [IORING_OP_ASYNC_CANCEL] = {
 581                 .name                   = "ASYNC_CANCEL",
 582         },
 583         [IORING_OP_LINK_TIMEOUT] = {
 584                 .name                   = "LINK_TIMEOUT",
 585         },
 586         [IORING_OP_CONNECT] = {
 587                 .name                   = "CONNECT",
 588         },
 589         [IORING_OP_FALLOCATE] = {
 590                 .name                   = "FALLOCATE",
 591         },
 592         [IORING_OP_OPENAT] = {
 593                 .name                   = "OPENAT",
 594                 .cleanup                = io_open_cleanup,
 595         },
 596         [IORING_OP_CLOSE] = {
 597                 .name                   = "CLOSE",
 598         },
 599         [IORING_OP_FILES_UPDATE] = {
 600                 .name                   = "FILES_UPDATE",
 601         },
 602         [IORING_OP_STATX] = {
 603                 .name                   = "STATX",
 604                 .cleanup                = io_statx_cleanup,
 605         },
 606         [IORING_OP_READ] = {
 607                 .name                   = "READ",
 608                 .cleanup                = io_readv_writev_cleanup,
 609                 .fail                   = io_rw_fail,
 610         },
 611         [IORING_OP_WRITE] = {
 612                 .name                   = "WRITE",
 613                 .cleanup                = io_readv_writev_cleanup,
 614                 .fail                   = io_rw_fail,
 615         },
 616         [IORING_OP_FADVISE] = {
 617                 .name                   = "FADVISE",
 618         },
 619         [IORING_OP_MADVISE] = {
 620                 .name                   = "MADVISE",
 621         },
 622         [IORING_OP_SEND] = {
 623                 .name                   = "SEND",
 624 #if defined(CONFIG_NET)
 625                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 626                 .fail                   = io_sendrecv_fail,
 627 #endif
 628         },
 629         [IORING_OP_RECV] = {
 630                 .name                   = "RECV",
 631 #if defined(CONFIG_NET)
 632                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 633                 .fail                   = io_sendrecv_fail,
 634 #endif
 635         },
 636         [IORING_OP_OPENAT2] = {
 637                 .name                   = "OPENAT2",
 638                 .cleanup                = io_open_cleanup,
 639         },
 640         [IORING_OP_EPOLL_CTL] = {
 641                 .name                   = "EPOLL",
 642         },
 643         [IORING_OP_SPLICE] = {
 644                 .name                   = "SPLICE",
 645                 .cleanup                = io_splice_cleanup,
 646         },
 647         [IORING_OP_PROVIDE_BUFFERS] = {
 648                 .name                   = "PROVIDE_BUFFERS",
 649         },
 650         [IORING_OP_REMOVE_BUFFERS] = {
 651                 .name                   = "REMOVE_BUFFERS",
 652         },
 653         [IORING_OP_TEE] = {
 654                 .name                   = "TEE",
 655                 .cleanup                = io_splice_cleanup,
 656         },
 657         [IORING_OP_SHUTDOWN] = {
 658                 .name                   = "SHUTDOWN",
 659         },
 660         [IORING_OP_RENAMEAT] = {
 661                 .name                   = "RENAMEAT",
 662                 .cleanup                = io_renameat_cleanup,
 663         },
 664         [IORING_OP_UNLINKAT] = {
 665                 .name                   = "UNLINKAT",
 666                 .cleanup                = io_unlinkat_cleanup,
 667         },
 668         [IORING_OP_MKDIRAT] = {
 669                 .name                   = "MKDIRAT",
 670                 .cleanup                = io_mkdirat_cleanup,
 671         },
 672         [IORING_OP_SYMLINKAT] = {
 673                 .name                   = "SYMLINKAT",
 674                 .cleanup                = io_link_cleanup,
 675         },
 676         [IORING_OP_LINKAT] = {
 677                 .name                   = "LINKAT",
 678                 .cleanup                = io_link_cleanup,
 679         },
 680         [IORING_OP_MSG_RING] = {
 681                 .name                   = "MSG_RING",
 682                 .cleanup                = io_msg_ring_cleanup,
 683         },
 684         [IORING_OP_FSETXATTR] = {
 685                 .name                   = "FSETXATTR",
 686                 .cleanup                = io_xattr_cleanup,
 687         },
 688         [IORING_OP_SETXATTR] = {
 689                 .name                   = "SETXATTR",
 690                 .cleanup                = io_xattr_cleanup,
 691         },
 692         [IORING_OP_FGETXATTR] = {
 693                 .name                   = "FGETXATTR",
 694                 .cleanup                = io_xattr_cleanup,
 695         },
 696         [IORING_OP_GETXATTR] = {
 697                 .name                   = "GETXATTR",
 698                 .cleanup                = io_xattr_cleanup,
 699         },
 700         [IORING_OP_SOCKET] = {
 701                 .name                   = "SOCKET",
 702         },
 703         [IORING_OP_URING_CMD] = {
 704                 .name                   = "URING_CMD",
 705         },
 706         [IORING_OP_SEND_ZC] = {
 707                 .name                   = "SEND_ZC",
 708 #if defined(CONFIG_NET)
 709                 .cleanup                = io_send_zc_cleanup,
 710                 .fail                   = io_sendrecv_fail,
 711 #endif
 712         },
 713         [IORING_OP_SENDMSG_ZC] = {
 714                 .name                   = "SENDMSG_ZC",
 715 #if defined(CONFIG_NET)
 716                 .cleanup                = io_send_zc_cleanup,
 717                 .fail                   = io_sendrecv_fail,
 718 #endif
 719         },
 720         [IORING_OP_READ_MULTISHOT] = {
 721                 .name                   = "READ_MULTISHOT",
 722                 .cleanup                = io_readv_writev_cleanup,
 723         },
 724         [IORING_OP_WAITID] = {
 725                 .name                   = "WAITID",
 726         },
 727         [IORING_OP_FUTEX_WAIT] = {
 728                 .name                   = "FUTEX_WAIT",
 729         },
 730         [IORING_OP_FUTEX_WAKE] = {
 731                 .name                   = "FUTEX_WAKE",
 732         },
 733         [IORING_OP_FUTEX_WAITV] = {
 734                 .name                   = "FUTEX_WAITV",
 735         },
 736         [IORING_OP_FIXED_FD_INSTALL] = {
 737                 .name                   = "FIXED_FD_INSTALL",
 738         },
 739         [IORING_OP_FTRUNCATE] = {
 740                 .name                   = "FTRUNCATE",
 741         },
 742         [IORING_OP_BIND] = {
 743                 .name                   = "BIND",
 744         },
 745         [IORING_OP_LISTEN] = {
 746                 .name                   = "LISTEN",
 747         },
 748 };
 749
 750 const char *io_uring_get_opcode(u8 opcode)
 751 {
 752         if (opcode < IORING_OP_LAST)
 753                 return io_cold_defs[opcode].name;
 754         return "INVALID";
 755 }
 756
 757 bool io_uring_op_supported(u8 opcode)
 758 {
 759         if (opcode < IORING_OP_LAST &&
 760             io_issue_defs[opcode].prep != io_eopnotsupp_prep)
 761                 return true;
 762         return false;
 763 }
 764
 765 void __init io_uring_optable_init(void)
 766 {
 767         int i;
 768
 769         BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
 770         BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
 771
 772         for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
 773                 BUG_ON(!io_issue_defs[i].prep);
 774                 if (io_issue_defs[i].prep != io_eopnotsupp_prep)
 775                         BUG_ON(!io_issue_defs[i].issue);
 776                 WARN_ON_ONCE(!io_cold_defs[i].name);
 777         }
 778 }