Merge branch 'for-6.10/io_uring' into net-accept-more

* for-6.10/io_uring: (97 commits)
  io_uring: support to inject result for NOP
  io_uring: fail NOP if non-zero op flags is passed in
  io_uring/net: add IORING_ACCEPT_POLL_FIRST flag
  io_uring/net: add IORING_ACCEPT_DONTWAIT flag
  io_uring/filetable: don't unnecessarily clear/reset bitmap
  io_uring/io-wq: Use set_bit() and test_bit() at worker->flags
  io_uring/msg_ring: cleanup posting to IOPOLL vs !IOPOLL ring
  io_uring: Require zeroed sqe->len on provided-buffers send
  io_uring/notif: disable LAZY_WAKE for linked notifs
  io_uring/net: fix sendzc lazy wake polling
  io_uring/msg_ring: reuse ctx->submitter_task read using READ_ONCE instead of re-reading it
  io_uring/rw: reinstate thread check for retries
  io_uring/notif: implement notification stacking
  io_uring/notif: simplify io_notif_flush()
  net: add callback for setting a ubuf_info to skb
  net: extend ubuf_info callback to ops structure
  io_uring/net: support bundles for recv
  io_uring/net: support bundles for send
  io_uring/kbuf: add helpers for getting/peeking multiple buffers
  io_uring/net: add provided buffer support for IORING_OP_SEND
  ...

diff --combined drivers/net/tun.c
index 92da8c03d960c9beca5b425a1e3d366f37fa21fc,b7401d990680453919f83f3fdec0ad1c3c90af35..9254bca2813dc19f75a395fa0d6a26658ecd6cb9
--- 1/drivers/net/tun.c
--- 2/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@@ -1906,7 -1906,7 +1906,7 @@@ static ssize_t tun_get_user(struct tun_
                skb_zcopy_init(skb, msg_control);
        } else if (msg_control) {
                struct ubuf_info *uarg = msg_control;
-               uarg->callback(NULL, uarg, false);
+               uarg->ops->complete(NULL, uarg, false);
        }
  
        skb_reset_network_header(skb);
@@@ -2125,16 -2125,14 +2125,16 @@@ static ssize_t tun_put_user(struct tun_
                                            tun_is_little_endian(tun), true,
                                            vlan_hlen)) {
                        struct skb_shared_info *sinfo = skb_shinfo(skb);
 -                      pr_err("unexpected GSO type: "
 -                             "0x%x, gso_size %d, hdr_len %d\n",
 -                             sinfo->gso_type, tun16_to_cpu(tun, gso.gso_size),
 -                             tun16_to_cpu(tun, gso.hdr_len));
 -                      print_hex_dump(KERN_ERR, "tun: ",
 -                                     DUMP_PREFIX_NONE,
 -                                     16, 1, skb->head,
 -                                     min((int)tun16_to_cpu(tun, gso.hdr_len), 64), true);
 +
 +                      if (net_ratelimit()) {
 +                              netdev_err(tun->dev, "unexpected GSO type: 0x%x, gso_size %d, hdr_len %d\n",
 +                                         sinfo->gso_type, tun16_to_cpu(tun, gso.gso_size),
 +                                         tun16_to_cpu(tun, gso.hdr_len));
 +                              print_hex_dump(KERN_ERR, "tun: ",
 +                                             DUMP_PREFIX_NONE,
 +                                             16, 1, skb->head,
 +                                             min((int)tun16_to_cpu(tun, gso.hdr_len), 64), true);
 +                      }
                        WARN_ON_ONCE(1);
                        return -EINVAL;
                }

diff --combined include/linux/skbuff.h
index 4ff48eda3f642dac705307958c2883f3cdb07ab5,ced69f37977f375dd6d10cced97c74602db67038..1cc72c37019067b78af7671981e9fd73f3d6ae7d
--- 1/include/linux/skbuff.h
--- 2/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@@ -527,6 -527,13 +527,13 @@@ enum 
  #define SKBFL_ALL_ZEROCOPY    (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY | \
                                 SKBFL_DONT_ORPHAN | SKBFL_MANAGED_FRAG_REFS)
  
+ struct ubuf_info_ops {
+       void (*complete)(struct sk_buff *, struct ubuf_info *,
+                        bool zerocopy_success);
+       /* has to be compatible with skb_zcopy_set() */
+       int (*link_skb)(struct sk_buff *skb, struct ubuf_info *uarg);
+ };
+ 
  /*
   * The callback notifies userspace to release buffers when skb DMA is done in
   * lower device, the skb last reference should be 0 when calling this.
@@@ -536,8 -543,7 +543,7 @@@
   * The desc field is used to track userspace buffer index.
   */
  struct ubuf_info {
-       void (*callback)(struct sk_buff *, struct ubuf_info *,
-                        bool zerocopy_success);
+       const struct ubuf_info_ops *ops;
        refcount_t refcnt;
        u8 flags;
  };
@@@ -1662,14 -1668,13 +1668,13 @@@ static inline void skb_set_end_offset(s
  }
  #endif
  
+ extern const struct ubuf_info_ops msg_zerocopy_ubuf_ops;
+ 
  struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
                                       struct ubuf_info *uarg);
  
  void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref);
  
- void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
-                          bool success);
- 
  int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,
                            struct sk_buff *skb, struct iov_iter *from,
                            size_t length);
@@@ -1757,13 -1762,13 +1762,13 @@@ static inline void *skb_zcopy_get_nouar
  static inline void net_zcopy_put(struct ubuf_info *uarg)
  {
        if (uarg)
-               uarg->callback(NULL, uarg, true);
+               uarg->ops->complete(NULL, uarg, true);
  }
  
  static inline void net_zcopy_put_abort(struct ubuf_info *uarg, bool have_uref)
  {
        if (uarg) {
-               if (uarg->callback == msg_zerocopy_callback)
+               if (uarg->ops == &msg_zerocopy_ubuf_ops)
                        msg_zerocopy_put_abort(uarg, have_uref);
                else if (have_uref)
                        net_zcopy_put(uarg);
@@@ -1777,7 -1782,7 +1782,7 @@@ static inline void skb_zcopy_clear(stru
  
        if (uarg) {
                if (!skb_zcopy_is_nouarg(skb))
-                       uarg->callback(skb, uarg, zerocopy_success);
+                       uarg->ops->complete(skb, uarg, zerocopy_success);
  
                skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY;
        }
@@@ -3031,21 -3036,6 +3036,21 @@@ static inline void skb_mac_header_rebui
        }
  }
  
 +/* Move the full mac header up to current network_header.
 + * Leaves skb->data pointing at offset skb->mac_len into the mac_header.
 + * Must be provided the complete mac header length.
 + */
 +static inline void skb_mac_header_rebuild_full(struct sk_buff *skb, u32 full_mac_len)
 +{
 +      if (skb_mac_header_was_set(skb)) {
 +              const unsigned char *old_mac = skb_mac_header(skb);
 +
 +              skb_set_mac_header(skb, -full_mac_len);
 +              memmove(skb_mac_header(skb), old_mac, full_mac_len);
 +              __skb_push(skb, full_mac_len - skb->mac_len);
 +      }
 +}
 +
  static inline int skb_checksum_start_offset(const struct sk_buff *skb)
  {
        return skb->csum_start - skb_headroom(skb);

diff --combined net/core/skbuff.c
index 4096e679f61c76041223b894e3ee4f9a8a051000,4a0c10685cd2607f1ec1e96d9fb032ae96f04235..1169fdbdb2c3a4b3369affe2aa83f1a294319bd9
--- 1/net/core/skbuff.c
--- 2/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@@ -1708,7 -1708,7 +1708,7 @@@ static struct ubuf_info *msg_zerocopy_a
                return NULL;
        }
  
-       uarg->ubuf.callback = msg_zerocopy_callback;
+       uarg->ubuf.ops = &msg_zerocopy_ubuf_ops;
        uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
        uarg->len = 1;
        uarg->bytelen = size;
@@@ -1734,7 -1734,7 +1734,7 @@@ struct ubuf_info *msg_zerocopy_realloc(
                u32 bytelen, next;
  
                /* there might be non MSG_ZEROCOPY users */
-               if (uarg->callback != msg_zerocopy_callback)
+               if (uarg->ops != &msg_zerocopy_ubuf_ops)
                        return NULL;
  
                /* realloc only when socket is locked (TCP, UDP cork),
@@@ -1845,8 -1845,8 +1845,8 @@@ release
        sock_put(sk);
  }
  
- void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
-                          bool success)
+ static void msg_zerocopy_complete(struct sk_buff *skb, struct ubuf_info *uarg,
+                                 bool success)
  {
        struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg);
  
@@@ -1855,7 -1855,6 +1855,6 @@@
        if (refcount_dec_and_test(&uarg->refcnt))
                __msg_zerocopy_callback(uarg_zc);
  }
- EXPORT_SYMBOL_GPL(msg_zerocopy_callback);
  
  void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
  {
@@@ -1865,10 -1864,15 +1864,15 @@@
        uarg_to_msgzc(uarg)->len--;
  
        if (have_uref)
-               msg_zerocopy_callback(NULL, uarg, true);
+               msg_zerocopy_complete(NULL, uarg, true);
  }
  EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort);
  
+ const struct ubuf_info_ops msg_zerocopy_ubuf_ops = {
+       .complete = msg_zerocopy_complete,
+ };
+ EXPORT_SYMBOL_GPL(msg_zerocopy_ubuf_ops);
+ 
  int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
                             struct msghdr *msg, int len,
                             struct ubuf_info *uarg)
@@@ -1876,11 -1880,18 +1880,18 @@@
        struct ubuf_info *orig_uarg = skb_zcopy(skb);
        int err, orig_len = skb->len;
  
-       /* An skb can only point to one uarg. This edge case happens when
-        * TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
-        */
-       if (orig_uarg && uarg != orig_uarg)
-               return -EEXIST;
+       if (uarg->ops->link_skb) {
+               err = uarg->ops->link_skb(skb, uarg);
+               if (err)
+                       return err;
+       } else {
+               /* An skb can only point to one uarg. This edge case happens
+                * when TCP appends to an skb, but zerocopy_realloc triggered
+                * a new alloc.
+                */
+               if (orig_uarg && uarg != orig_uarg)
+                       return -EEXIST;
+       }
  
        err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len);
        if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
@@@ -1894,7 -1905,8 +1905,8 @@@
                return err;
        }
  
-       skb_zcopy_set(skb, uarg, NULL);
+       if (!uarg->ops->link_skb)
+               skb_zcopy_set(skb, uarg, NULL);
        return skb->len - orig_len;
  }
  EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
@@@ -2123,17 -2135,11 +2135,17 @@@ static inline int skb_alloc_rx_flag(con
  
  struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
  {
 -      int headerlen = skb_headroom(skb);
 -      unsigned int size = skb_end_offset(skb) + skb->data_len;
 -      struct sk_buff *n = __alloc_skb(size, gfp_mask,
 -                                      skb_alloc_rx_flag(skb), NUMA_NO_NODE);
 +      struct sk_buff *n;
 +      unsigned int size;
 +      int headerlen;
 +
 +      if (WARN_ON_ONCE(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST))
 +              return NULL;
  
 +      headerlen = skb_headroom(skb);
 +      size = skb_end_offset(skb) + skb->data_len;
 +      n = __alloc_skb(size, gfp_mask,
 +                      skb_alloc_rx_flag(skb), NUMA_NO_NODE);
        if (!n)
                return NULL;
  
@@@ -2461,17 -2467,12 +2473,17 @@@ struct sk_buff *skb_copy_expand(const s
        /*
         *      Allocate the copy buffer
         */
 -      struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
 -                                      gfp_mask, skb_alloc_rx_flag(skb),
 -                                      NUMA_NO_NODE);
 -      int oldheadroom = skb_headroom(skb);
        int head_copy_len, head_copy_off;
 +      struct sk_buff *n;
 +      int oldheadroom;
 +
 +      if (WARN_ON_ONCE(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST))
 +              return NULL;
  
 +      oldheadroom = skb_headroom(skb);
 +      n = __alloc_skb(newheadroom + skb->len + newtailroom,
 +                      gfp_mask, skb_alloc_rx_flag(skb),
 +                      NUMA_NO_NODE);
        if (!n)
                return NULL;