include/net/gro.h

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2
   3 #ifndef _NET_IPV6_GRO_H
   4 #define _NET_IPV6_GRO_H
   5
   6 #include <linux/indirect_call_wrapper.h>
   7 #include <linux/ip.h>
   8 #include <linux/ipv6.h>
   9 #include <net/ip6_checksum.h>
  10 #include <linux/skbuff.h>
  11 #include <net/udp.h>
  12
  13 struct napi_gro_cb {
  14         /* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */
  15         void    *frag0;
  16
  17         /* Length of frag0. */
  18         unsigned int frag0_len;
  19
  20         /* This indicates where we are processing relative to skb->data. */
  21         int     data_offset;
  22
  23         /* This is non-zero if the packet cannot be merged with the new skb. */
  24         u16     flush;
  25
  26         /* Save the IP ID here and check when we get to the transport layer */
  27         u16     flush_id;
  28
  29         /* Number of segments aggregated. */
  30         u16     count;
  31
  32         /* Start offset for remote checksum offload */
  33         u16     gro_remcsum_start;
  34
  35         /* jiffies when first packet was created/queued */
  36         unsigned long age;
  37
  38         /* Used in ipv6_gro_receive() and foo-over-udp */
  39         u16     proto;
  40
  41         /* This is non-zero if the packet may be of the same flow. */
  42         u8      same_flow:1;
  43
  44         /* Used in tunnel GRO receive */
  45         u8      encap_mark:1;
  46
  47         /* GRO checksum is valid */
  48         u8      csum_valid:1;
  49
  50         /* Number of checksums via CHECKSUM_UNNECESSARY */
  51         u8      csum_cnt:3;
  52
  53         /* Free the skb? */
  54         u8      free:2;
  55 #define NAPI_GRO_FREE             1
  56 #define NAPI_GRO_FREE_STOLEN_HEAD 2
  57
  58         /* Used in foo-over-udp, set in udp[46]_gro_receive */
  59         u8      is_ipv6:1;
  60
  61         /* Used in GRE, set in fou/gue_gro_receive */
  62         u8      is_fou:1;
  63
  64         /* Used to determine if flush_id can be ignored */
  65         u8      is_atomic:1;
  66
  67         /* Number of gro_receive callbacks this packet already went through */
  68         u8 recursion_counter:4;
  69
  70         /* GRO is done by frag_list pointer chaining. */
  71         u8      is_flist:1;
  72
  73         /* used to support CHECKSUM_COMPLETE for tunneling protocols */
  74         __wsum  csum;
  75
  76         /* used in skb_gro_receive() slow path */
  77         struct sk_buff *last;
  78 };
  79
  80 #define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
  81
  82 #define GRO_RECURSION_LIMIT 15
  83 static inline int gro_recursion_inc_test(struct sk_buff *skb)
  84 {
  85         return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT;
  86 }
  87
  88 typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *);
  89 static inline struct sk_buff *call_gro_receive(gro_receive_t cb,
  90                                                struct list_head *head,
  91                                                struct sk_buff *skb)
  92 {
  93         if (unlikely(gro_recursion_inc_test(skb))) {
  94                 NAPI_GRO_CB(skb)->flush |= 1;
  95                 return NULL;
  96         }
  97
  98         return cb(head, skb);
  99 }
 100
 101 typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *,
 102                                             struct sk_buff *);
 103 static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb,
 104                                                   struct sock *sk,
 105                                                   struct list_head *head,
 106                                                   struct sk_buff *skb)
 107 {
 108         if (unlikely(gro_recursion_inc_test(skb))) {
 109                 NAPI_GRO_CB(skb)->flush |= 1;
 110                 return NULL;
 111         }
 112
 113         return cb(sk, head, skb);
 114 }
 115
 116 static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
 117 {
 118         return NAPI_GRO_CB(skb)->data_offset;
 119 }
 120
 121 static inline unsigned int skb_gro_len(const struct sk_buff *skb)
 122 {
 123         return skb->len - NAPI_GRO_CB(skb)->data_offset;
 124 }
 125
 126 static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
 127 {
 128         NAPI_GRO_CB(skb)->data_offset += len;
 129 }
 130
 131 static inline void *skb_gro_header_fast(struct sk_buff *skb,
 132                                         unsigned int offset)
 133 {
 134         return NAPI_GRO_CB(skb)->frag0 + offset;
 135 }
 136
 137 static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen)
 138 {
 139         return NAPI_GRO_CB(skb)->frag0_len < hlen;
 140 }
 141
 142 static inline void skb_gro_frag0_invalidate(struct sk_buff *skb)
 143 {
 144         NAPI_GRO_CB(skb)->frag0 = NULL;
 145         NAPI_GRO_CB(skb)->frag0_len = 0;
 146 }
 147
 148 static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
 149                                         unsigned int offset)
 150 {
 151         if (!pskb_may_pull(skb, hlen))
 152                 return NULL;
 153
 154         skb_gro_frag0_invalidate(skb);
 155         return skb->data + offset;
 156 }
 157
 158 static inline void *skb_gro_network_header(struct sk_buff *skb)
 159 {
 160         return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) +
 161                skb_network_offset(skb);
 162 }
 163
 164 static inline __wsum inet_gro_compute_pseudo(struct sk_buff *skb, int proto)
 165 {
 166         const struct iphdr *iph = skb_gro_network_header(skb);
 167
 168         return csum_tcpudp_nofold(iph->saddr, iph->daddr,
 169                                   skb_gro_len(skb), proto, 0);
 170 }
 171
 172 static inline void skb_gro_postpull_rcsum(struct sk_buff *skb,
 173                                         const void *start, unsigned int len)
 174 {
 175         if (NAPI_GRO_CB(skb)->csum_valid)
 176                 NAPI_GRO_CB(skb)->csum = wsum_negate(csum_partial(start, len,
 177                                                 wsum_negate(NAPI_GRO_CB(skb)->csum)));
 178 }
 179
 180 /* GRO checksum functions. These are logical equivalents of the normal
 181  * checksum functions (in skbuff.h) except that they operate on the GRO
 182  * offsets and fields in sk_buff.
 183  */
 184
 185 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb);
 186
 187 static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb)
 188 {
 189         return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb));
 190 }
 191
 192 static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb,
 193                                                       bool zero_okay,
 194                                                       __sum16 check)
 195 {
 196         return ((skb->ip_summed != CHECKSUM_PARTIAL ||
 197                 skb_checksum_start_offset(skb) <
 198                  skb_gro_offset(skb)) &&
 199                 !skb_at_gro_remcsum_start(skb) &&
 200                 NAPI_GRO_CB(skb)->csum_cnt == 0 &&
 201                 (!zero_okay || check));
 202 }
 203
 204 static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb,
 205                                                            __wsum psum)
 206 {
 207         if (NAPI_GRO_CB(skb)->csum_valid &&
 208             !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum)))
 209                 return 0;
 210
 211         NAPI_GRO_CB(skb)->csum = psum;
 212
 213         return __skb_gro_checksum_complete(skb);
 214 }
 215
 216 static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb)
 217 {
 218         if (NAPI_GRO_CB(skb)->csum_cnt > 0) {
 219                 /* Consume a checksum from CHECKSUM_UNNECESSARY */
 220                 NAPI_GRO_CB(skb)->csum_cnt--;
 221         } else {
 222                 /* Update skb for CHECKSUM_UNNECESSARY and csum_level when we
 223                  * verified a new top level checksum or an encapsulated one
 224                  * during GRO. This saves work if we fallback to normal path.
 225                  */
 226                 __skb_incr_checksum_unnecessary(skb);
 227         }
 228 }
 229
 230 #define __skb_gro_checksum_validate(skb, proto, zero_okay, check,       \
 231                                     compute_pseudo)                     \
 232 ({                                                                      \
 233         __sum16 __ret = 0;                                              \
 234         if (__skb_gro_checksum_validate_needed(skb, zero_okay, check))  \
 235                 __ret = __skb_gro_checksum_validate_complete(skb,       \
 236                                 compute_pseudo(skb, proto));            \
 237         if (!__ret)                                                     \
 238                 skb_gro_incr_csum_unnecessary(skb);                     \
 239         __ret;                                                          \
 240 })
 241
 242 #define skb_gro_checksum_validate(skb, proto, compute_pseudo)           \
 243         __skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo)
 244
 245 #define skb_gro_checksum_validate_zero_check(skb, proto, check,         \
 246                                              compute_pseudo)            \
 247         __skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo)
 248
 249 #define skb_gro_checksum_simple_validate(skb)                           \
 250         __skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo)
 251
 252 static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb)
 253 {
 254         return (NAPI_GRO_CB(skb)->csum_cnt == 0 &&
 255                 !NAPI_GRO_CB(skb)->csum_valid);
 256 }
 257
 258 static inline void __skb_gro_checksum_convert(struct sk_buff *skb,
 259                                               __wsum pseudo)
 260 {
 261         NAPI_GRO_CB(skb)->csum = ~pseudo;
 262         NAPI_GRO_CB(skb)->csum_valid = 1;
 263 }
 264
 265 #define skb_gro_checksum_try_convert(skb, proto, compute_pseudo)        \
 266 do {                                                                    \
 267         if (__skb_gro_checksum_convert_check(skb))                      \
 268                 __skb_gro_checksum_convert(skb,                         \
 269                                            compute_pseudo(skb, proto)); \
 270 } while (0)
 271
 272 struct gro_remcsum {
 273         int offset;
 274         __wsum delta;
 275 };
 276
 277 static inline void skb_gro_remcsum_init(struct gro_remcsum *grc)
 278 {
 279         grc->offset = 0;
 280         grc->delta = 0;
 281 }
 282
 283 static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr,
 284                                             unsigned int off, size_t hdrlen,
 285                                             int start, int offset,
 286                                             struct gro_remcsum *grc,
 287                                             bool nopartial)
 288 {
 289         __wsum delta;
 290         size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start);
 291
 292         BUG_ON(!NAPI_GRO_CB(skb)->csum_valid);
 293
 294         if (!nopartial) {
 295                 NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start;
 296                 return ptr;
 297         }
 298
 299         ptr = skb_gro_header_fast(skb, off);
 300         if (skb_gro_header_hard(skb, off + plen)) {
 301                 ptr = skb_gro_header_slow(skb, off + plen, off);
 302                 if (!ptr)
 303                         return NULL;
 304         }
 305
 306         delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum,
 307                                start, offset);
 308
 309         /* Adjust skb->csum since we changed the packet */
 310         NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta);
 311
 312         grc->offset = off + hdrlen + offset;
 313         grc->delta = delta;
 314
 315         return ptr;
 316 }
 317
 318 static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
 319                                            struct gro_remcsum *grc)
 320 {
 321         void *ptr;
 322         size_t plen = grc->offset + sizeof(u16);
 323
 324         if (!grc->delta)
 325                 return;
 326
 327         ptr = skb_gro_header_fast(skb, grc->offset);
 328         if (skb_gro_header_hard(skb, grc->offset + sizeof(u16))) {
 329                 ptr = skb_gro_header_slow(skb, plen, grc->offset);
 330                 if (!ptr)
 331                         return;
 332         }
 333
 334         remcsum_unadjust((__sum16 *)ptr, grc->delta);
 335 }
 336
 337 #ifdef CONFIG_XFRM_OFFLOAD
 338 static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
 339 {
 340         if (PTR_ERR(pp) != -EINPROGRESS)
 341                 NAPI_GRO_CB(skb)->flush |= flush;
 342 }
 343 static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb,
 344                                                struct sk_buff *pp,
 345                                                int flush,
 346                                                struct gro_remcsum *grc)
 347 {
 348         if (PTR_ERR(pp) != -EINPROGRESS) {
 349                 NAPI_GRO_CB(skb)->flush |= flush;
 350                 skb_gro_remcsum_cleanup(skb, grc);
 351                 skb->remcsum_offload = 0;
 352         }
 353 }
 354 #else
 355 static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
 356 {
 357         NAPI_GRO_CB(skb)->flush |= flush;
 358 }
 359 static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb,
 360                                                struct sk_buff *pp,
 361                                                int flush,
 362                                                struct gro_remcsum *grc)
 363 {
 364         NAPI_GRO_CB(skb)->flush |= flush;
 365         skb_gro_remcsum_cleanup(skb, grc);
 366         skb->remcsum_offload = 0;
 367 }
 368 #endif
 369
 370 INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *,
 371                                                            struct sk_buff *));
 372 INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int));
 373 INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *,
 374                                                            struct sk_buff *));
 375 INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int));
 376
 377 INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp4_gro_receive(struct list_head *,
 378                                                            struct sk_buff *));
 379 INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff *, int));
 380
 381 INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp6_gro_receive(struct list_head *,
 382                                                            struct sk_buff *));
 383 INDIRECT_CALLABLE_DECLARE(int udp6_gro_complete(struct sk_buff *, int));
 384
 385 #define indirect_call_gro_receive_inet(cb, f2, f1, head, skb)   \
 386 ({                                                              \
 387         unlikely(gro_recursion_inc_test(skb)) ?                 \
 388                 NAPI_GRO_CB(skb)->flush |= 1, NULL :            \
 389                 INDIRECT_CALL_INET(cb, f2, f1, head, skb);      \
 390 })
 391
 392 struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
 393                                 struct udphdr *uh, struct sock *sk);
 394 int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup);
 395
 396 static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb)
 397 {
 398         struct udphdr *uh;
 399         unsigned int hlen, off;
 400
 401         off  = skb_gro_offset(skb);
 402         hlen = off + sizeof(*uh);
 403         uh   = skb_gro_header_fast(skb, off);
 404         if (skb_gro_header_hard(skb, hlen))
 405                 uh = skb_gro_header_slow(skb, hlen, off);
 406
 407         return uh;
 408 }
 409
 410 static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto)
 411 {
 412         const struct ipv6hdr *iph = skb_gro_network_header(skb);
 413
 414         return ~csum_unfold(csum_ipv6_magic(&iph->saddr, &iph->daddr,
 415                                             skb_gro_len(skb), proto, 0));
 416 }
 417
 418 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb);
 419
 420 /* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
 421 static inline void gro_normal_list(struct napi_struct *napi)
 422 {
 423         if (!napi->rx_count)
 424                 return;
 425         netif_receive_skb_list_internal(&napi->rx_list);
 426         INIT_LIST_HEAD(&napi->rx_list);
 427         napi->rx_count = 0;
 428 }
 429
 430 /* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded,
 431  * pass the whole batch up to the stack.
 432  */
 433 static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs)
 434 {
 435         list_add_tail(&skb->list, &napi->rx_list);
 436         napi->rx_count += segs;
 437         if (napi->rx_count >= gro_normal_batch)
 438                 gro_normal_list(napi);
 439 }
 440
 441
 442 #endif /* _NET_IPV6_GRO_H */