Linux 6.14-rc3

[linux.git] / net / xfrm / xfrm_iptfs.c

// SPDX-License-Identifier: GPL-2.0
/* xfrm_iptfs: IPTFS encapsulation support
 *
 * April 21 2022, Christian Hopps <[email protected]>
 *
 * Copyright (c) 2022, LabN Consulting, L.L.C.
 *
 */

#include <linux/kernel.h>
#include <linux/icmpv6.h>
#include <linux/skbuff_ref.h>
#include <net/gro.h>
#include <net/icmp.h>
#include <net/ip6_route.h>
#include <net/inet_ecn.h>
#include <net/xfrm.h>

#include <crypto/aead.h>

#include "xfrm_inout.h"
#include "trace_iptfs.h"

/* IPTFS encap (header) values. */
#define IPTFS_SUBTYPE_BASIC 0
#define IPTFS_SUBTYPE_CC 1

/* ----------------------------------------------- */
/* IP-TFS default SA values (tunnel egress/dir-in) */
/* ----------------------------------------------- */

/**
 * define IPTFS_DEFAULT_DROP_TIME_USECS - default drop time
 *
 * The default IPTFS drop time in microseconds. The drop time is the amount of
 * time before a missing out-of-order IPTFS tunnel packet is considered lost.
 * See also the reorder window.
 *
 * Default 1s.
 */
#define IPTFS_DEFAULT_DROP_TIME_USECS 1000000

/**
 * define IPTFS_DEFAULT_REORDER_WINDOW - default reorder window size
 *
 * The default IPTFS reorder window size. The reorder window size dictates the
 * maximum number of IPTFS tunnel packets in a sequence that may arrive out of
 * order.
 *
 * Default 3. (tcp folks suggested)
 */
#define IPTFS_DEFAULT_REORDER_WINDOW 3

/* ------------------------------------------------ */
/* IPTFS default SA values (tunnel ingress/dir-out) */
/* ------------------------------------------------ */

/**
 * define IPTFS_DEFAULT_INIT_DELAY_USECS - default initial output delay
 *
 * The initial output delay is the amount of time prior to servicing the output
 * queue after queueing the first packet on said queue. This applies anytime the
 * output queue was previously empty.
 *
 * Default 0.
 */
#define IPTFS_DEFAULT_INIT_DELAY_USECS 0

/**
 * define IPTFS_DEFAULT_MAX_QUEUE_SIZE - default max output queue size.
 *
 * The default IPTFS max output queue size in octets. The output queue is where
 * received packets destined for output over an IPTFS tunnel are stored prior to
 * being output in aggregated/fragmented form over the IPTFS tunnel.
 *
 * Default 1M.
 */
#define IPTFS_DEFAULT_MAX_QUEUE_SIZE (1024 * 10240)

/* Assumed: skb->head is cache aligned.
 *
 * L2 Header resv: Arrange for cacheline to start at skb->data - 16 to keep the
 * to-be-pushed L2 header in the same cacheline as resulting `skb->data` (i.e.,
 * the L3 header). If cacheline size is > 64 then skb->data + pushed L2 will all
 * be in a single cacheline if we simply reserve 64 bytes.
 *
 * L3 Header resv: For L3+L2 headers (i.e., skb->data points at the IPTFS payload)
 * we want `skb->data` to be cacheline aligned and all pushed L2L3 headers will
 * be in their own cacheline[s]. 128 works for cachelins up to 128 bytes, for
 * any larger cacheline sizes the pushed headers will simply share the cacheline
 * with the start of the IPTFS payload (skb->data).
 */
#define XFRM_IPTFS_MIN_L3HEADROOM 128
#define XFRM_IPTFS_MIN_L2HEADROOM (L1_CACHE_BYTES > 64 ? 64 : 64 + 16)

/* Min to try to share outer iptfs skb data vs copying into new skb */
#define IPTFS_PKT_SHARE_MIN 129

#define NSECS_IN_USEC 1000

#define IPTFS_HRTIMER_MODE HRTIMER_MODE_REL_SOFT

/**
 * struct xfrm_iptfs_config - configuration for the IPTFS tunnel.
 * @pkt_size: size of the outer IP packet. 0 to use interface and MTU discovery,
 *      otherwise the user specified value.
 * @max_queue_size: The maximum number of octets allowed to be queued to be sent
 *      over the IPTFS SA. The queue size is measured as the size of all the
 *      packets enqueued.
 * @reorder_win_size: the number slots in the reorder window, thus the number of
 *      packets that may arrive out of order.
 * @dont_frag: true to inhibit fragmenting across IPTFS outer packets.
 */
struct xfrm_iptfs_config {
        u32 pkt_size;       /* outer_packet_size or 0 */
        u32 max_queue_size; /* octets */
        u16 reorder_win_size;
        u8 dont_frag : 1;
};

struct skb_wseq {
        struct sk_buff *skb;
        u64 drop_time;
};

/**
 * struct xfrm_iptfs_data - mode specific xfrm state.
 * @cfg: IPTFS tunnel config.
 * @x: owning SA (xfrm_state).
 * @queue: queued user packets to send.
 * @queue_size: number of octets on queue (sum of packet sizes).
 * @ecn_queue_size: octets above with ECN mark.
 * @init_delay_ns: nanoseconds to wait to send initial IPTFS packet.
 * @iptfs_timer: output timer.
 * @iptfs_settime: time the output timer was set.
 * @payload_mtu: max payload size.
 * @w_seq_set: true after first seq received.
 * @w_wantseq: waiting for this seq number as next to process (in order).
 * @w_saved: the saved buf array (reorder window).
 * @w_savedlen: the saved len (not size).
 * @drop_lock: lock to protect reorder queue.
 * @drop_timer: timer for considering next packet lost.
 * @drop_time_ns: timer intervan in nanoseconds.
 * @ra_newskb: new pkt being reassembled.
 * @ra_wantseq: expected next sequence for reassembly.
 * @ra_runt: last pkt bytes from very end of last skb.
 * @ra_runtlen: size of ra_runt.
 */
struct xfrm_iptfs_data {
        struct xfrm_iptfs_config cfg;

        /* Ingress User Input */
        struct xfrm_state *x;      /* owning state */
        struct sk_buff_head queue; /* output queue */

        u32 queue_size;             /* octets */
        u32 ecn_queue_size;         /* octets above which ECN mark */
        u64 init_delay_ns;          /* nanoseconds */
        struct hrtimer iptfs_timer; /* output timer */
        time64_t iptfs_settime;     /* time timer was set */
        u32 payload_mtu;            /* max payload size */

        /* Tunnel input reordering */
        bool w_seq_set;           /* true after first seq received */
        u64 w_wantseq;            /* expected next sequence */
        struct skb_wseq *w_saved; /* the saved buf array */
        u32 w_savedlen;           /* the saved len (not size) */
        spinlock_t drop_lock;
        struct hrtimer drop_timer;
        u64 drop_time_ns;

        /* Tunnel input reassembly */
        struct sk_buff *ra_newskb; /* new pkt being reassembled */
        u64 ra_wantseq;            /* expected next sequence */
        u8 ra_runt[6];             /* last pkt bytes from last skb */
        u8 ra_runtlen;             /* count of ra_runt */
};

static u32 __iptfs_get_inner_mtu(struct xfrm_state *x, int outer_mtu);
static enum hrtimer_restart iptfs_delay_timer(struct hrtimer *me);
static enum hrtimer_restart iptfs_drop_timer(struct hrtimer *me);

/* ================= */
/* Utility Functions */
/* ================= */

#ifdef TRACEPOINTS_ENABLED
static u32 __trace_ip_proto(struct iphdr *iph)
{
        if (iph->version == 4)
                return iph->protocol;
        return ((struct ipv6hdr *)iph)->nexthdr;
}

static u32 __trace_ip_proto_seq(struct iphdr *iph)
{
        void *nexthdr;
        u32 protocol = 0;

        if (iph->version == 4) {
                nexthdr = (void *)(iph + 1);
                protocol = iph->protocol;
        } else if (iph->version == 6) {
                nexthdr = (void *)(((struct ipv6hdr *)(iph)) + 1);
                protocol = ((struct ipv6hdr *)(iph))->nexthdr;
        }
        switch (protocol) {
        case IPPROTO_ICMP:
                return ntohs(((struct icmphdr *)nexthdr)->un.echo.sequence);
        case IPPROTO_ICMPV6:
                return ntohs(((struct icmp6hdr *)nexthdr)->icmp6_sequence);
        case IPPROTO_TCP:
                return ntohl(((struct tcphdr *)nexthdr)->seq);
        case IPPROTO_UDP:
                return ntohs(((struct udphdr *)nexthdr)->source);
        default:
                return 0;
        }
}
#endif /*TRACEPOINTS_ENABLED*/

static u64 __esp_seq(struct sk_buff *skb)
{
        u64 seq = ntohl(XFRM_SKB_CB(skb)->seq.input.low);

        return seq | (u64)ntohl(XFRM_SKB_CB(skb)->seq.input.hi) << 32;
}

/* ======================= */
/* IPTFS SK_BUFF Functions */
/* ======================= */

/**
 * iptfs_alloc_skb() - Allocate a new `skb`.
 * @tpl: the skb to copy required meta-data from.
 * @len: the linear length of the head data, zero is fine.
 * @l3resv: true if skb reserve needs to support pushing L3 headers
 *
 * A new `skb` is allocated and required meta-data is copied from `tpl`, the
 * head data is sized to `len` + reserved space set according to the @l3resv
 * boolean.
 *
 * When @l3resv is false, resv is XFRM_IPTFS_MIN_L2HEADROOM which arranges for
 * `skb->data - 16`  which is a good guess for good cache alignment (placing the
 * to be pushed L2 header at the start of a cacheline.
 *
 * Otherwise, @l3resv is true and resv is set to the correct reserved space for
 * dst->dev plus the calculated L3 overhead for the xfrm dst or
 * XFRM_IPTFS_MIN_L3HEADROOM whichever is larger. This is then cache aligned so
 * that all the headers will commonly fall in a cacheline when possible.
 *
 * l3resv=true is used on tunnel ingress (tx), because we need to reserve for
 * the new IPTFS packet (i.e., L2+L3 headers). On tunnel egress (rx) the data
 * being copied into the skb includes the user L3 headers already so we only
 * need to reserve for L2.
 *
 * Return: the new skb or NULL.
 */
static struct sk_buff *iptfs_alloc_skb(struct sk_buff *tpl, u32 len, bool l3resv)
{
        struct sk_buff *skb;
        u32 resv;

        if (!l3resv) {
                resv = XFRM_IPTFS_MIN_L2HEADROOM;
        } else {
                struct dst_entry *dst = skb_dst(tpl);

                resv = LL_RESERVED_SPACE(dst->dev) + dst->header_len;
                resv = max(resv, XFRM_IPTFS_MIN_L3HEADROOM);
                resv = L1_CACHE_ALIGN(resv);
        }

        skb = alloc_skb(len + resv, GFP_ATOMIC | __GFP_NOWARN);
        if (!skb)
                return NULL;

        skb_reserve(skb, resv);

        if (!l3resv) {
                /* xfrm_input resume needs dev and xfrm ext from tunnel pkt */
                skb->dev = tpl->dev;
                __skb_ext_copy(skb, tpl);
        }

        /* dropped by xfrm_input, used by xfrm_output */
        skb_dst_copy(skb, tpl);

        return skb;
}

/**
 * iptfs_skb_head_to_frag() - initialize a skb_frag_t based on skb head data
 * @skb: skb with the head data
 * @frag: frag to initialize
 */
static void iptfs_skb_head_to_frag(const struct sk_buff *skb, skb_frag_t *frag)
{
        struct page *page = virt_to_head_page(skb->data);
        unsigned char *addr = (unsigned char *)page_address(page);

        skb_frag_fill_page_desc(frag, page, skb->data - addr, skb_headlen(skb));
}

/**
 * struct iptfs_skb_frag_walk - use to track a walk through fragments
 * @fragi: current fragment index
 * @past: length of data in fragments before @fragi
 * @total: length of data in all fragments
 * @nr_frags: number of fragments present in array
 * @initial_offset: the value passed in to skb_prepare_frag_walk()
 * @frags: the page fragments inc. room for head page
 * @pp_recycle: copy of skb->pp_recycle
 */
struct iptfs_skb_frag_walk {
        u32 fragi;
        u32 past;
        u32 total;
        u32 nr_frags;
        u32 initial_offset;
        skb_frag_t frags[MAX_SKB_FRAGS + 1];
        bool pp_recycle;
};

/**
 * iptfs_skb_prepare_frag_walk() - initialize a frag walk over an skb.
 * @skb: the skb to walk.
 * @initial_offset: start the walk @initial_offset into the skb.
 * @walk: the walk to initialize
 *
 * Future calls to skb_add_frags() will expect the @offset value to be at
 * least @initial_offset large.
 */
static void iptfs_skb_prepare_frag_walk(struct sk_buff *skb, u32 initial_offset,
                                        struct iptfs_skb_frag_walk *walk)
{
        struct skb_shared_info *shinfo = skb_shinfo(skb);
        skb_frag_t *frag, *from;
        u32 i;

        walk->initial_offset = initial_offset;
        walk->fragi = 0;
        walk->past = 0;
        walk->total = 0;
        walk->nr_frags = 0;
        walk->pp_recycle = skb->pp_recycle;

        if (skb->head_frag) {
                if (initial_offset >= skb_headlen(skb)) {
                        initial_offset -= skb_headlen(skb);
                } else {
                        frag = &walk->frags[walk->nr_frags++];
                        iptfs_skb_head_to_frag(skb, frag);
                        frag->offset += initial_offset;
                        frag->len -= initial_offset;
                        walk->total += frag->len;
                        initial_offset = 0;
                }
        } else {
                initial_offset -= skb_headlen(skb);
        }

        for (i = 0; i < shinfo->nr_frags; i++) {
                from = &shinfo->frags[i];
                if (initial_offset >= from->len) {
                        initial_offset -= from->len;
                        continue;
                }
                frag = &walk->frags[walk->nr_frags++];
                *frag = *from;
                if (initial_offset) {
                        frag->offset += initial_offset;
                        frag->len -= initial_offset;
                        initial_offset = 0;
                }
                walk->total += frag->len;
        }
}

static u32 iptfs_skb_reset_frag_walk(struct iptfs_skb_frag_walk *walk,
                                     u32 offset)
{
        /* Adjust offset to refer to internal walk values */
        offset -= walk->initial_offset;

        /* Get to the correct fragment for offset */
        while (offset < walk->past) {
                walk->past -= walk->frags[--walk->fragi].len;
                if (offset >= walk->past)
                        break;
        }
        while (offset >= walk->past + walk->frags[walk->fragi].len)
                walk->past += walk->frags[walk->fragi++].len;

        /* offset now relative to this current frag */
        offset -= walk->past;
        return offset;
}

/**
 * iptfs_skb_can_add_frags() - check if ok to add frags from walk to skb
 * @skb: skb to check for adding frags to
 * @walk: the walk that will be used as source for frags.
 * @offset: offset from beginning of original skb to start from.
 * @len: amount of data to add frag references to in @skb.
 *
 * Return: true if ok to add frags.
 */
static bool iptfs_skb_can_add_frags(const struct sk_buff *skb,
                                    struct iptfs_skb_frag_walk *walk,
                                    u32 offset, u32 len)
{
        struct skb_shared_info *shinfo = skb_shinfo(skb);
        u32 fragi, nr_frags, fraglen;

        if (skb_has_frag_list(skb) || skb->pp_recycle != walk->pp_recycle)
                return false;

        /* Make offset relative to current frag after setting that */
        offset = iptfs_skb_reset_frag_walk(walk, offset);

        /* Verify we have array space for the fragments we need to add */
        fragi = walk->fragi;
        nr_frags = shinfo->nr_frags;
        while (len && fragi < walk->nr_frags) {
                skb_frag_t *frag = &walk->frags[fragi];

                fraglen = frag->len;
                if (offset) {
                        fraglen -= offset;
                        offset = 0;
                }
                if (++nr_frags > MAX_SKB_FRAGS)
                        return false;
                if (len <= fraglen)
                        return true;
                len -= fraglen;
                fragi++;
        }
        /* We may not copy all @len but what we have will fit. */
        return true;
}

/**
 * iptfs_skb_add_frags() - add a range of fragment references into an skb
 * @skb: skb to add references into
 * @walk: the walk to add referenced fragments from.
 * @offset: offset from beginning of original skb to start from.
 * @len: amount of data to add frag references to in @skb.
 *
 * iptfs_skb_can_add_frags() should be called before this function to verify
 * that the destination @skb is compatible with the walk and has space in the
 * array for the to be added frag references.
 *
 * Return: The number of bytes not added to @skb b/c we reached the end of the
 * walk before adding all of @len.
 */
static int iptfs_skb_add_frags(struct sk_buff *skb,
                               struct iptfs_skb_frag_walk *walk, u32 offset,
                               u32 len)
{
        struct skb_shared_info *shinfo = skb_shinfo(skb);
        u32 fraglen;

        if (!walk->nr_frags || offset >= walk->total + walk->initial_offset)
                return len;

        /* make offset relative to current frag after setting that */
        offset = iptfs_skb_reset_frag_walk(walk, offset);

        while (len && walk->fragi < walk->nr_frags) {
                skb_frag_t *frag = &walk->frags[walk->fragi];
                skb_frag_t *tofrag = &shinfo->frags[shinfo->nr_frags];

                *tofrag = *frag;
                if (offset) {
                        tofrag->offset += offset;
                        tofrag->len -= offset;
                        offset = 0;
                }
                __skb_frag_ref(tofrag);
                shinfo->nr_frags++;

                /* see if we are done */
                fraglen = tofrag->len;
                if (len < fraglen) {
                        tofrag->len = len;
                        skb->len += len;
                        skb->data_len += len;
                        return 0;
                }
                /* advance to next source fragment */
                len -= fraglen;                 /* careful, use dst bv_len */
                skb->len += fraglen;            /* careful, "   "    "     */
                skb->data_len += fraglen;       /* careful, "   "    "     */
                walk->past += frag->len;        /* careful, use src bv_len */
                walk->fragi++;
        }
        return len;
}

/* ================================== */
/* IPTFS Trace Event Definitions      */
/* ================================== */

#define CREATE_TRACE_POINTS
#include "trace_iptfs.h"

/* ================================== */
/* IPTFS Receiving (egress) Functions */
/* ================================== */

/**
 * iptfs_pskb_add_frags() - Create and add frags into a new sk_buff.
 * @tpl: template to create new skb from.
 * @walk: The source for fragments to add.
 * @off: The offset into @walk to add frags from, also used with @st and
 *       @copy_len.
 * @len: The length of data to add covering frags from @walk into @skb.
 *       This must be <= @skblen.
 * @st: The sequence state to copy from into the new head skb.
 * @copy_len: Copy @copy_len bytes from @st at offset @off into the new skb
 *            linear space.
 *
 * Create a new sk_buff `skb` using the template @tpl. Copy @copy_len bytes from
 * @st into the new skb linear space, and then add shared fragments from the
 * frag walk for the remaining @len of data (i.e., @len - @copy_len bytes).
 *
 * Return: The newly allocated sk_buff `skb` or NULL if an error occurs.
 */
static struct sk_buff *
iptfs_pskb_add_frags(struct sk_buff *tpl, struct iptfs_skb_frag_walk *walk,
                     u32 off, u32 len, struct skb_seq_state *st, u32 copy_len)
{
        struct sk_buff *skb;

        skb = iptfs_alloc_skb(tpl, copy_len, false);
        if (!skb)
                return NULL;

        /* this should not normally be happening */
        if (!iptfs_skb_can_add_frags(skb, walk, off + copy_len,
                                     len - copy_len)) {
                kfree_skb(skb);
                return NULL;
        }

        if (copy_len &&
            skb_copy_seq_read(st, off, skb_put(skb, copy_len), copy_len)) {
                XFRM_INC_STATS(dev_net(st->root_skb->dev),
                               LINUX_MIB_XFRMINERROR);
                kfree_skb(skb);
                return NULL;
        }

        iptfs_skb_add_frags(skb, walk, off + copy_len, len - copy_len);
        return skb;
}

/**
 * iptfs_pskb_extract_seq() - Create and load data into a new sk_buff.
 * @skblen: the total data size for `skb`.
 * @st: The source for the rest of the data to copy into `skb`.
 * @off: The offset into @st to copy data from.
 * @len: The length of data to copy from @st into `skb`. This must be <=
 *       @skblen.
 *
 * Create a new sk_buff `skb` with @skblen of packet data space. If non-zero,
 * copy @rlen bytes of @runt into `skb`. Then using seq functions copy @len
 * bytes from @st into `skb` starting from @off.
 *
 * It is an error for @len to be greater than the amount of data left in @st.
 *
 * Return: The newly allocated sk_buff `skb` or NULL if an error occurs.
 */
static struct sk_buff *
iptfs_pskb_extract_seq(u32 skblen, struct skb_seq_state *st, u32 off, int len)
{
        struct sk_buff *skb = iptfs_alloc_skb(st->root_skb, skblen, false);

        if (!skb)
                return NULL;
        if (skb_copy_seq_read(st, off, skb_put(skb, len), len)) {
                XFRM_INC_STATS(dev_net(st->root_skb->dev), LINUX_MIB_XFRMINERROR);
                kfree_skb(skb);
                return NULL;
        }
        return skb;
}

/**
 * iptfs_input_save_runt() - save data in xtfs runt space.
 * @xtfs: xtfs state
 * @seq: the current sequence
 * @buf: packet data
 * @len: length of packet data
 *
 * Save the small (`len`) start of a fragmented packet in `buf` in the xtfs data
 * runt space.
 */
static void iptfs_input_save_runt(struct xfrm_iptfs_data *xtfs, u64 seq,
                                  u8 *buf, int len)
{
        memcpy(xtfs->ra_runt, buf, len);

        xtfs->ra_runtlen = len;
        xtfs->ra_wantseq = seq + 1;
}

/**
 * __iptfs_iphlen() - return the v4/v6 header length using packet data.
 * @data: pointer at octet with version nibble
 *
 * The version data has been checked to be valid (i.e., either 4 or 6).
 *
 * Return: the IP header size based on the IP version.
 */
static u32 __iptfs_iphlen(u8 *data)
{
        struct iphdr *iph = (struct iphdr *)data;

        if (iph->version == 0x4)
                return sizeof(*iph);
        return sizeof(struct ipv6hdr);
}

/**
 * __iptfs_iplen() - return the v4/v6 length using packet data.
 * @data: pointer to ip (v4/v6) packet header
 *
 * Grab the IPv4 or IPv6 length value in the start of the inner packet header
 * pointed to by `data`. Assumes data len is enough for the length field only.
 *
 * The version data has been checked to be valid (i.e., either 4 or 6).
 *
 * Return: the length value.
 */
static u32 __iptfs_iplen(u8 *data)
{
        struct iphdr *iph = (struct iphdr *)data;

        if (iph->version == 0x4)
                return ntohs(iph->tot_len);
        return ntohs(((struct ipv6hdr *)iph)->payload_len) +
                sizeof(struct ipv6hdr);
}

/**
 * iptfs_complete_inner_skb() - finish preparing the inner packet for gro recv.
 * @x: xfrm state
 * @skb: the inner packet
 *
 * Finish the standard xfrm processing on the inner packet prior to sending back
 * through gro_cells_receive. We do this separately b/c we are building a list
 * of packets in the hopes that one day a list will be taken by
 * xfrm_input.
 */
static void iptfs_complete_inner_skb(struct xfrm_state *x, struct sk_buff *skb)
{
        skb_reset_network_header(skb);

        /* The packet is going back through gro_cells_receive no need to
         * set this.
         */
        skb_reset_transport_header(skb);

        /* Packet already has checksum value set. */
        skb->ip_summed = CHECKSUM_NONE;

        /* Our skb will contain the header data copied when this outer packet
         * which contained the start of this inner packet. This is true
         * when we allocate a new skb as well as when we reuse the existing skb.
         */
        if (ip_hdr(skb)->version == 0x4) {
                struct iphdr *iph = ip_hdr(skb);

                if (x->props.flags & XFRM_STATE_DECAP_DSCP)
                        ipv4_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, iph);
                if (!(x->props.flags & XFRM_STATE_NOECN))
                        if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))
                                IP_ECN_set_ce(iph);

                skb->protocol = htons(ETH_P_IP);
        } else {
                struct ipv6hdr *iph = ipv6_hdr(skb);

                if (x->props.flags & XFRM_STATE_DECAP_DSCP)
                        ipv6_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, iph);
                if (!(x->props.flags & XFRM_STATE_NOECN))
                        if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))
                                IP6_ECN_set_ce(skb, iph);

                skb->protocol = htons(ETH_P_IPV6);
        }
}

static void __iptfs_reassem_done(struct xfrm_iptfs_data *xtfs, bool free)
{
        assert_spin_locked(&xtfs->drop_lock);

        /* We don't care if it works locking takes care of things */
        hrtimer_try_to_cancel(&xtfs->drop_timer);
        if (free)
                kfree_skb(xtfs->ra_newskb);
        xtfs->ra_newskb = NULL;
}

/**
 * iptfs_reassem_abort() - In-progress packet is aborted free the state.
 * @xtfs: xtfs state
 */
static void iptfs_reassem_abort(struct xfrm_iptfs_data *xtfs)
{
        __iptfs_reassem_done(xtfs, true);
}

/**
 * iptfs_reassem_done() - In-progress packet is complete, clear the state.
 * @xtfs: xtfs state
 */
static void iptfs_reassem_done(struct xfrm_iptfs_data *xtfs)
{
        __iptfs_reassem_done(xtfs, false);
}

/**
 * iptfs_reassem_cont() - Continue the reassembly of an inner packets.
 * @xtfs: xtfs state
 * @seq: sequence of current packet
 * @st: seq read stat for current packet
 * @skb: current packet
 * @data: offset into sequential packet data
 * @blkoff: packet blkoff value
 * @list: list of skbs to enqueue completed packet on
 *
 * Process an IPTFS payload that has a non-zero `blkoff` or when we are
 * expecting the continuation b/c we have a runt or in-progress packet.
 *
 * Return: the new data offset to continue processing from.
 */
static u32 iptfs_reassem_cont(struct xfrm_iptfs_data *xtfs, u64 seq,
                              struct skb_seq_state *st, struct sk_buff *skb,
                              u32 data, u32 blkoff, struct list_head *list)
{
        struct iptfs_skb_frag_walk _fragwalk;
        struct iptfs_skb_frag_walk *fragwalk = NULL;
        struct sk_buff *newskb = xtfs->ra_newskb;
        u32 remaining = skb->len - data;
        u32 runtlen = xtfs->ra_runtlen;
        u32 copylen, fraglen, ipremain, iphlen, iphremain, rrem;

        /* Handle packet fragment we aren't expecting */
        if (!runtlen && !xtfs->ra_newskb)
                return data + min(blkoff, remaining);

        /* Important to remember that input to this function is an ordered
         * packet stream (unless the user disabled the reorder window). Thus if
         * we are waiting for, and expecting the next packet so we can continue
         * assembly, a newer sequence number indicates older ones are not coming
         * (or if they do should be ignored). Technically we can receive older
         * ones when the reorder window is disabled; however, the user should
         * have disabled fragmentation in this case, and regardless we don't
         * deal with it.
         *
         * blkoff could be zero if the stream is messed up (or it's an all pad
         * insertion) be careful to handle that case in each of the below
         */

        /* Too old case: This can happen when the reorder window is disabled so
         * ordering isn't actually guaranteed.
         */
        if (seq < xtfs->ra_wantseq)
                return data + remaining;

        /* Too new case: We missed what we wanted cleanup. */
        if (seq > xtfs->ra_wantseq) {
                XFRM_INC_STATS(xs_net(xtfs->x), LINUX_MIB_XFRMINIPTFSERROR);
                goto abandon;
        }

        if (blkoff == 0) {
                if ((*skb->data & 0xF0) != 0) {
                        XFRM_INC_STATS(xs_net(xtfs->x),
                                       LINUX_MIB_XFRMINIPTFSERROR);
                        goto abandon;
                }
                /* Handle all pad case, advance expected sequence number.
                 * (RFC 9347 S2.2.3)
                 */
                xtfs->ra_wantseq++;
                /* will end parsing */
                return data + remaining;
        }

        if (runtlen) {
                /* Regardless of what happens we're done with the runt */
                xtfs->ra_runtlen = 0;

                /* The start of this inner packet was at the very end of the last
                 * iptfs payload which didn't include enough for the ip header
                 * length field. We must have *at least* that now.
                 */
                rrem = sizeof(xtfs->ra_runt) - runtlen;
                if (remaining < rrem || blkoff < rrem) {
                        XFRM_INC_STATS(xs_net(xtfs->x),
                                       LINUX_MIB_XFRMINIPTFSERROR);
                        goto abandon;
                }

                /* fill in the runt data */
                if (skb_copy_seq_read(st, data, &xtfs->ra_runt[runtlen],
                                      rrem)) {
                        XFRM_INC_STATS(xs_net(xtfs->x),
                                       LINUX_MIB_XFRMINBUFFERERROR);
                        goto abandon;
                }

                /* We have enough data to get the ip length value now,
                 * allocate an in progress skb
                 */
                ipremain = __iptfs_iplen(xtfs->ra_runt);
                if (ipremain < sizeof(xtfs->ra_runt)) {
                        /* length has to be at least runtsize large */
                        XFRM_INC_STATS(xs_net(xtfs->x),
                                       LINUX_MIB_XFRMINIPTFSERROR);
                        goto abandon;
                }

                /* For the runt case we don't attempt sharing currently. NOTE:
                 * Currently, this IPTFS implementation will not create runts.
                 */

                newskb = iptfs_alloc_skb(skb, ipremain, false);
                if (!newskb) {
                        XFRM_INC_STATS(xs_net(xtfs->x), LINUX_MIB_XFRMINERROR);
                        goto abandon;
                }
                xtfs->ra_newskb = newskb;

                /* Copy the runt data into the buffer, but leave data
                 * pointers the same as normal non-runt case. The extra `rrem`
                 * recopied bytes are basically cacheline free. Allows using
                 * same logic below to complete.
                 */
                memcpy(skb_put(newskb, runtlen), xtfs->ra_runt,
                       sizeof(xtfs->ra_runt));
        }

        /* Continue reassembling the packet */
        ipremain = __iptfs_iplen(newskb->data);
        iphlen = __iptfs_iphlen(newskb->data);

        ipremain -= newskb->len;
        if (blkoff < ipremain) {
                /* Corrupt data, we don't have enough to complete the packet */
                XFRM_INC_STATS(xs_net(xtfs->x), LINUX_MIB_XFRMINIPTFSERROR);
                goto abandon;
        }

        /* We want the IP header in linear space */
        if (newskb->len < iphlen) {
                iphremain = iphlen - newskb->len;
                if (blkoff < iphremain) {
                        XFRM_INC_STATS(xs_net(xtfs->x),
                                       LINUX_MIB_XFRMINIPTFSERROR);
                        goto abandon;
                }
                fraglen = min(blkoff, remaining);
                copylen = min(fraglen, iphremain);
                if (skb_copy_seq_read(st, data, skb_put(newskb, copylen),
                                      copylen)) {
                        XFRM_INC_STATS(xs_net(xtfs->x),
                                       LINUX_MIB_XFRMINBUFFERERROR);
                        goto abandon;
                }
                /* this is a silly condition that might occur anyway */
                if (copylen < iphremain) {
                        xtfs->ra_wantseq++;
                        return data + fraglen;
                }
                /* update data and things derived from it */
                data += copylen;
                blkoff -= copylen;
                remaining -= copylen;
                ipremain -= copylen;
        }

        fraglen = min(blkoff, remaining);
        copylen = min(fraglen, ipremain);

        /* If we may have the opportunity to share prepare a fragwalk. */
        if (!skb_has_frag_list(skb) && !skb_has_frag_list(newskb) &&
            (skb->head_frag || skb->len == skb->data_len) &&
            skb->pp_recycle == newskb->pp_recycle) {
                fragwalk = &_fragwalk;
                iptfs_skb_prepare_frag_walk(skb, data, fragwalk);
        }

        /* Try share then copy. */
        if (fragwalk &&
            iptfs_skb_can_add_frags(newskb, fragwalk, data, copylen)) {
                iptfs_skb_add_frags(newskb, fragwalk, data, copylen);
        } else {
                /* copy fragment data into newskb */
                if (skb_copy_seq_read(st, data, skb_put(newskb, copylen),
                                      copylen)) {
                        XFRM_INC_STATS(xs_net(xtfs->x),
                                       LINUX_MIB_XFRMINBUFFERERROR);
                        goto abandon;
                }
        }

        if (copylen < ipremain) {
                xtfs->ra_wantseq++;
        } else {
                /* We are done with packet reassembly! */
                iptfs_reassem_done(xtfs);
                iptfs_complete_inner_skb(xtfs->x, newskb);
                list_add_tail(&newskb->list, list);
        }

        /* will continue on to new data block or end */
        return data + fraglen;

abandon:
        if (xtfs->ra_newskb) {
                iptfs_reassem_abort(xtfs);
        } else {
                xtfs->ra_runtlen = 0;
                xtfs->ra_wantseq = 0;
        }
        /* skip past fragment, maybe to end */
        return data + min(blkoff, remaining);
}

static bool __input_process_payload(struct xfrm_state *x, u32 data,
                                    struct skb_seq_state *skbseq,
                                    struct list_head *sublist)
{
        u8 hbytes[sizeof(struct ipv6hdr)];
        struct iptfs_skb_frag_walk _fragwalk;
        struct iptfs_skb_frag_walk *fragwalk = NULL;
        struct sk_buff *defer, *first_skb, *next, *skb;
        const unsigned char *old_mac;
        struct xfrm_iptfs_data *xtfs;
        struct iphdr *iph;
        struct net *net;
        u32 first_iplen, iphlen, iplen, remaining, tail;
        u32 capturelen;
        u64 seq;

        xtfs = x->mode_data;
        net = xs_net(x);
        skb = skbseq->root_skb;
        first_skb = NULL;
        defer = NULL;

        seq = __esp_seq(skb);

        /* Save the old mac header if set */
        old_mac = skb_mac_header_was_set(skb) ? skb_mac_header(skb) : NULL;

        /* New packets */

        tail = skb->len;
        while (data < tail) {
                __be16 protocol = 0;

                /* Gather information on the next data block.
                 * `data` points to the start of the data block.
                 */
                remaining = tail - data;

                /* try and copy enough bytes to read length from ipv4/ipv6 */
                iphlen = min_t(u32, remaining, 6);
                if (skb_copy_seq_read(skbseq, data, hbytes, iphlen)) {
                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
                        goto done;
                }

                iph = (struct iphdr *)hbytes;
                if (iph->version == 0x4) {
                        /* must have at least tot_len field present */
                        if (remaining < 4) {
                                /* save the bytes we have, advance data and exit */
                                iptfs_input_save_runt(xtfs, seq, hbytes,
                                                      remaining);
                                data += remaining;
                                break;
                        }

                        iplen = be16_to_cpu(iph->tot_len);
                        iphlen = iph->ihl << 2;
                        protocol = cpu_to_be16(ETH_P_IP);
                        XFRM_MODE_SKB_CB(skbseq->root_skb)->tos = iph->tos;
                } else if (iph->version == 0x6) {
                        /* must have at least payload_len field present */
                        if (remaining < 6) {
                                /* save the bytes we have, advance data and exit */
                                iptfs_input_save_runt(xtfs, seq, hbytes,
                                                      remaining);
                                data += remaining;
                                break;
                        }

                        iplen = be16_to_cpu(((struct ipv6hdr *)hbytes)->payload_len);
                        iplen += sizeof(struct ipv6hdr);
                        iphlen = sizeof(struct ipv6hdr);
                        protocol = cpu_to_be16(ETH_P_IPV6);
                        XFRM_MODE_SKB_CB(skbseq->root_skb)->tos =
                                ipv6_get_dsfield((struct ipv6hdr *)iph);
                } else if (iph->version == 0x0) {
                        /* pad */
                        data = tail;
                        break;
                } else {
                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
                        goto done;
                }

                if (unlikely(skbseq->stepped_offset)) {
                        /* We need to reset our seq read, it can't backup at
                         * this point.
                         */
                        struct sk_buff *save = skbseq->root_skb;

                        skb_abort_seq_read(skbseq);
                        skb_prepare_seq_read(save, data, tail, skbseq);
                }

                if (first_skb) {
                        skb = NULL;
                } else {
                        first_skb = skb;
                        first_iplen = iplen;
                        fragwalk = NULL;

                        /* We are going to skip over `data` bytes to reach the
                         * start of the IP header of `iphlen` len for `iplen`
                         * inner packet.
                         */

                        if (skb_has_frag_list(skb)) {
                                defer = skb;
                                skb = NULL;
                        } else if (data + iphlen <= skb_headlen(skb) &&
                                   /* make sure our header is 32-bit aligned? */
                                   /* ((uintptr_t)(skb->data + data) & 0x3) == 0 && */
                                   skb_tailroom(skb) + tail - data >= iplen) {
                                /* Reuse the received skb.
                                 *
                                 * We have enough headlen to pull past any
                                 * initial fragment data, leaving at least the
                                 * IP header in the linear buffer space.
                                 *
                                 * For linear buffer space we only require that
                                 * linear buffer space is large enough to
                                 * eventually hold the entire reassembled
                                 * packet (by including tailroom in the check).
                                 *
                                 * For non-linear tailroom is 0 and so we only
                                 * re-use if the entire packet is present
                                 * already.
                                 *
                                 * NOTE: there are many more options for
                                 * sharing, KISS for now. Also, this can produce
                                 * skb's with the IP header unaligned to 32
                                 * bits. If that ends up being a problem then a
                                 * check should be added to the conditional
                                 * above that the header lies on a 32-bit
                                 * boundary as well.
                                 */
                                skb_pull(skb, data);

                                /* our range just changed */
                                data = 0;
                                tail = skb->len;
                                remaining = skb->len;

                                skb->protocol = protocol;
                                skb_mac_header_rebuild(skb);
                                if (skb->mac_len)
                                        eth_hdr(skb)->h_proto = skb->protocol;

                                /* all pointers could be changed now reset walk */
                                skb_abort_seq_read(skbseq);
                                skb_prepare_seq_read(skb, data, tail, skbseq);
                        } else if (skb->head_frag &&
                                   /* We have the IP header right now */
                                   remaining >= iphlen) {
                                fragwalk = &_fragwalk;
                                iptfs_skb_prepare_frag_walk(skb, data, fragwalk);
                                defer = skb;
                                skb = NULL;
                        } else {
                                /* We couldn't reuse the input skb so allocate a
                                 * new one.
                                 */
                                defer = skb;
                                skb = NULL;
                        }

                        /* Don't trim `first_skb` until the end as we are
                         * walking that data now.
                         */
                }

                capturelen = min(iplen, remaining);
                if (!skb) {
                        if (!fragwalk ||
                            /* Large enough to be worth sharing */
                            iplen < IPTFS_PKT_SHARE_MIN ||
                            /* Have IP header + some data to share. */
                            capturelen <= iphlen ||
                            /* Try creating skb and adding frags */
                            !(skb = iptfs_pskb_add_frags(first_skb, fragwalk,
                                                         data, capturelen,
                                                         skbseq, iphlen))) {
                                skb = iptfs_pskb_extract_seq(iplen, skbseq, data, capturelen);
                        }
                        if (!skb) {
                                /* skip to next packet or done */
                                data += capturelen;
                                continue;
                        }

                        skb->protocol = protocol;
                        if (old_mac) {
                                /* rebuild the mac header */
                                skb_set_mac_header(skb, -first_skb->mac_len);
                                memcpy(skb_mac_header(skb), old_mac, first_skb->mac_len);
                                eth_hdr(skb)->h_proto = skb->protocol;
                        }
                }

                data += capturelen;

                if (skb->len < iplen) {
                        /* Start reassembly */
                        spin_lock(&xtfs->drop_lock);

                        xtfs->ra_newskb = skb;
                        xtfs->ra_wantseq = seq + 1;
                        if (!hrtimer_is_queued(&xtfs->drop_timer)) {
                                /* softirq blocked lest the timer fire and interrupt us */
                                hrtimer_start(&xtfs->drop_timer,
                                              xtfs->drop_time_ns,
                                              IPTFS_HRTIMER_MODE);
                        }

                        spin_unlock(&xtfs->drop_lock);

                        break;
                }

                iptfs_complete_inner_skb(x, skb);
                list_add_tail(&skb->list, sublist);
        }

        if (data != tail)
                /* this should not happen from the above code */
                XFRM_INC_STATS(net, LINUX_MIB_XFRMINIPTFSERROR);

        if (first_skb && first_iplen && !defer && first_skb != xtfs->ra_newskb) {
                /* first_skb is queued b/c !defer and not partial */
                if (pskb_trim(first_skb, first_iplen)) {
                        /* error trimming */
                        list_del(&first_skb->list);
                        defer = first_skb;
                }
                first_skb->ip_summed = CHECKSUM_NONE;
        }

        /* Send the packets! */
        list_for_each_entry_safe(skb, next, sublist, list) {
                skb_list_del_init(skb);
                if (xfrm_input(skb, 0, 0, -2))
                        kfree_skb(skb);
        }
done:
        skb = skbseq->root_skb;
        skb_abort_seq_read(skbseq);

        if (defer) {
                consume_skb(defer);
        } else if (!first_skb) {
                /* skb is the original passed in skb, but we didn't get far
                 * enough to process it as the first_skb, if we had it would
                 * either be save in ra_newskb, trimmed and sent on as an skb or
                 * placed in defer to be freed.
                 */
                kfree_skb(skb);
        }
        return true;
}

/**
 * iptfs_input_ordered() - handle next in order IPTFS payload.
 * @x: xfrm state
 * @skb: current packet
 *
 * Process the IPTFS payload in `skb` and consume it afterwards.
 */
static void iptfs_input_ordered(struct xfrm_state *x, struct sk_buff *skb)
{
        struct ip_iptfs_cc_hdr iptcch;
        struct skb_seq_state skbseq;
        struct list_head sublist; /* rename this it's just a list */
        struct xfrm_iptfs_data *xtfs;
        struct ip_iptfs_hdr *ipth;
        struct net *net;
        u32 blkoff, data, remaining;
        bool consumed = false;
        u64 seq;

        xtfs = x->mode_data;
        net = xs_net(x);

        seq = __esp_seq(skb);

        /* Large enough to hold both types of header */
        ipth = (struct ip_iptfs_hdr *)&iptcch;

        skb_prepare_seq_read(skb, 0, skb->len, &skbseq);

        /* Get the IPTFS header and validate it */

        if (skb_copy_seq_read(&skbseq, 0, ipth, sizeof(*ipth))) {
                XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
                goto done;
        }
        data = sizeof(*ipth);

        trace_iptfs_egress_recv(skb, xtfs, be16_to_cpu(ipth->block_offset));

        /* Set data past the basic header */
        if (ipth->subtype == IPTFS_SUBTYPE_CC) {
                /* Copy the rest of the CC header */
                remaining = sizeof(iptcch) - sizeof(*ipth);
                if (skb_copy_seq_read(&skbseq, data, ipth + 1, remaining)) {
                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
                        goto done;
                }
                data += remaining;
        } else if (ipth->subtype != IPTFS_SUBTYPE_BASIC) {
                XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
                goto done;
        }

        if (ipth->flags != 0) {
                XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
                goto done;
        }

        INIT_LIST_HEAD(&sublist);

        /* Handle fragment at start of payload, and/or waiting reassembly. */

        blkoff = ntohs(ipth->block_offset);
        /* check before locking i.e., maybe */
        if (blkoff || xtfs->ra_runtlen || xtfs->ra_newskb) {
                spin_lock(&xtfs->drop_lock);

                /* check again after lock */
                if (blkoff || xtfs->ra_runtlen || xtfs->ra_newskb) {
                        data = iptfs_reassem_cont(xtfs, seq, &skbseq, skb, data,
                                                  blkoff, &sublist);
                }

                spin_unlock(&xtfs->drop_lock);
        }

        /* New packets */
        consumed = __input_process_payload(x, data, &skbseq, &sublist);
done:
        if (!consumed) {
                skb = skbseq.root_skb;
                skb_abort_seq_read(&skbseq);
                kfree_skb(skb);
        }
}

/* ------------------------------- */
/* Input (Egress) Re-ordering Code */
/* ------------------------------- */

static void __vec_shift(struct xfrm_iptfs_data *xtfs, u32 shift)
{
        u32 savedlen = xtfs->w_savedlen;

        if (shift > savedlen)
                shift = savedlen;
        if (shift != savedlen)
                memcpy(xtfs->w_saved, xtfs->w_saved + shift,
                       (savedlen - shift) * sizeof(*xtfs->w_saved));
        memset(xtfs->w_saved + savedlen - shift, 0,
               shift * sizeof(*xtfs->w_saved));
        xtfs->w_savedlen -= shift;
}

static void __reorder_past(struct xfrm_iptfs_data *xtfs, struct sk_buff *inskb,
                           struct list_head *freelist)
{
        list_add_tail(&inskb->list, freelist);
}

static u32 __reorder_drop(struct xfrm_iptfs_data *xtfs, struct list_head *list)

{
        struct skb_wseq *s, *se;
        const u32 savedlen = xtfs->w_savedlen;
        time64_t now = ktime_get_raw_fast_ns();
        u32 count = 0;
        u32 scount = 0;

        if (xtfs->w_saved[0].drop_time > now)
                goto set_timer;

        ++xtfs->w_wantseq;

        /* Keep flushing packets until we reach a drop time greater than now. */
        s = xtfs->w_saved;
        se = s + savedlen;
        do {
                /* Walking past empty slots until we reach a packet */
                for (; s < se && !s->skb; s++) {
                        if (s->drop_time > now)
                                goto outerdone;
                }
                /* Sending packets until we hit another empty slot. */
                for (; s < se && s->skb; scount++, s++)
                        list_add_tail(&s->skb->list, list);
        } while (s < se);
outerdone:

        count = s - xtfs->w_saved;
        if (count) {
                xtfs->w_wantseq += count;

                /* Shift handled slots plus final empty slot into slot 0. */
                __vec_shift(xtfs, count);
        }

        if (xtfs->w_savedlen) {
set_timer:
                /* Drifting is OK */
                hrtimer_start(&xtfs->drop_timer,
                              xtfs->w_saved[0].drop_time - now,
                              IPTFS_HRTIMER_MODE);
        }
        return scount;
}

static void __reorder_this(struct xfrm_iptfs_data *xtfs, struct sk_buff *inskb,
                           struct list_head *list)
{
        struct skb_wseq *s, *se;
        const u32 savedlen = xtfs->w_savedlen;
        u32 count = 0;

        /* Got what we wanted. */
        list_add_tail(&inskb->list, list);
        ++xtfs->w_wantseq;
        if (!savedlen)
                return;

        /* Flush remaining consecutive packets. */

        /* Keep sending until we hit another missed pkt. */
        for (s = xtfs->w_saved, se = s + savedlen; s < se && s->skb; s++)
                list_add_tail(&s->skb->list, list);
        count = s - xtfs->w_saved;
        if (count)
                xtfs->w_wantseq += count;

        /* Shift handled slots plus final empty slot into slot 0. */
        __vec_shift(xtfs, count + 1);
}

/* Set the slot's drop time and all the empty slots below it until reaching a
 * filled slot which will already be set.
 */
static void iptfs_set_window_drop_times(struct xfrm_iptfs_data *xtfs, int index)
{
        const u32 savedlen = xtfs->w_savedlen;
        struct skb_wseq *s = xtfs->w_saved;
        time64_t drop_time;

        assert_spin_locked(&xtfs->drop_lock);

        if (savedlen > index + 1) {
                /* we are below another, our drop time and the timer are already set */
                return;
        }
        /* we are the most future so get a new drop time. */
        drop_time = ktime_get_raw_fast_ns();
        drop_time += xtfs->drop_time_ns;

        /* Walk back through the array setting drop times as we go */
        s[index].drop_time = drop_time;
        while (index-- > 0 && !s[index].skb)
                s[index].drop_time = drop_time;

        /* If we walked all the way back, schedule the drop timer if needed */
        if (index == -1 && !hrtimer_is_queued(&xtfs->drop_timer))
                hrtimer_start(&xtfs->drop_timer, xtfs->drop_time_ns,
                              IPTFS_HRTIMER_MODE);
}

static void __reorder_future_fits(struct xfrm_iptfs_data *xtfs,
                                  struct sk_buff *inskb,
                                  struct list_head *freelist)
{
        const u64 inseq = __esp_seq(inskb);
        const u64 wantseq = xtfs->w_wantseq;
        const u64 distance = inseq - wantseq;
        const u32 savedlen = xtfs->w_savedlen;
        const u32 index = distance - 1;

        /* Handle future sequence number received which fits in the window.
         *
         * We know we don't have the seq we want so we won't be able to flush
         * anything.
         */

        /* slot count is 4, saved size is 3 savedlen is 2
         *
         * "window boundary" is based on the fixed window size
         * distance is also slot number
         * index is an array index (i.e., - 1 of slot)
         * : : - implicit NULL after array len
         *
         *          +--------- used length (savedlen == 2)
         *          |   +----- array size (nslots - 1 == 3)
         *          |   |   + window boundary (nslots == 4)
         *          V   V | V
         *                |
         *  0   1   2   3 |   slot number
         * ---  0   1   2 |   array index
         *     [-] [b] : :|   array
         *
         * "2" "3" "4" *5*|   seq numbers
         *
         * We receive seq number 5
         * distance == 3 [inseq(5) - w_wantseq(2)]
         * index == 2 [distance(6) - 1]
         */

        if (xtfs->w_saved[index].skb) {
                /* a dup of a future */
                list_add_tail(&inskb->list, freelist);
                return;
        }

        xtfs->w_saved[index].skb = inskb;
        xtfs->w_savedlen = max(savedlen, index + 1);
        iptfs_set_window_drop_times(xtfs, index);
}

static void __reorder_future_shifts(struct xfrm_iptfs_data *xtfs,
                                    struct sk_buff *inskb,
                                    struct list_head *list)
{
        const u32 nslots = xtfs->cfg.reorder_win_size + 1;
        const u64 inseq = __esp_seq(inskb);
        u32 savedlen = xtfs->w_savedlen;
        u64 wantseq = xtfs->w_wantseq;
        struct skb_wseq *wnext;
        struct sk_buff *slot0;
        u32 beyond, shifting, slot;
        u64 distance;

        /* Handle future sequence number received.
         *
         * IMPORTANT: we are at least advancing w_wantseq (i.e., wantseq) by 1
         * b/c we are beyond the window boundary.
         *
         * We know we don't have the wantseq so that counts as a drop.
         */

        /* example: slot count is 4, array size is 3 savedlen is 2, slot 0 is
         * the missing sequence number.
         *
         * the final slot at savedlen (index savedlen - 1) is always occupied.
         *
         * beyond is "beyond array size" not savedlen.
         *
         *          +--------- array length (savedlen == 2)
         *          |   +----- array size (nslots - 1 == 3)
         *          |   | +- window boundary (nslots == 4)
         *          V   V |
         *                |
         *  0   1   2   3 |   slot number
         * ---  0   1   2 |   array index
         *     [b] [c] : :|   array
         *                |
         * "2" "3" "4" "5"|*6*  seq numbers
         *
         * We receive seq number 6
         * distance == 4 [inseq(6) - w_wantseq(2)]
         * newslot == distance
         * index == 3 [distance(4) - 1]
         * beyond == 1 [newslot(4) - lastslot((nslots(4) - 1))]
         * shifting == 1 [min(savedlen(2), beyond(1)]
         * slot0_skb == [b], and should match w_wantseq
         *
         *                +--- window boundary (nslots == 4)
         *  0   1   2   3 | 4   slot number
         * ---  0   1   2 | 3   array index
         *     [b] : : : :|     array
         * "2" "3" "4" "5" *6*  seq numbers
         *
         * We receive seq number 6
         * distance == 4 [inseq(6) - w_wantseq(2)]
         * newslot == distance
         * index == 3 [distance(4) - 1]
         * beyond == 1 [newslot(4) - lastslot((nslots(4) - 1))]
         * shifting == 1 [min(savedlen(1), beyond(1)]
         * slot0_skb == [b] and should match w_wantseq
         *
         *                +-- window boundary (nslots == 4)
         *  0   1   2   3 | 4   5   6   slot number
         * ---  0   1   2 | 3   4   5   array index
         *     [-] [c] : :|             array
         * "2" "3" "4" "5" "6" "7" *8*  seq numbers
         *
         * savedlen = 2, beyond = 3
         * iter 1: slot0 == NULL, missed++, lastdrop = 2 (2+1-1), slot0 = [-]
         * iter 2: slot0 == NULL, missed++, lastdrop = 3 (2+2-1), slot0 = [c]
         * 2 < 3, extra = 1 (3-2), missed += extra, lastdrop = 4 (2+2+1-1)
         *
         * We receive seq number 8
         * distance == 6 [inseq(8) - w_wantseq(2)]
         * newslot == distance
         * index == 5 [distance(6) - 1]
         * beyond == 3 [newslot(6) - lastslot((nslots(4) - 1))]
         * shifting == 2 [min(savedlen(2), beyond(3)]
         *
         * slot0_skb == NULL changed from [b] when "savedlen < beyond" is true.
         */

        /* Now send any packets that are being shifted out of saved, and account
         * for missing packets that are exiting the window as we shift it.
         */

        distance = inseq - wantseq;
        beyond = distance - (nslots - 1);

        /* If savedlen > beyond we are shifting some, else all. */
        shifting = min(savedlen, beyond);

        /* slot0 is the buf that just shifted out and into slot0 */
        slot0 = NULL;
        wnext = xtfs->w_saved;
        for (slot = 1; slot <= shifting; slot++, wnext++) {
                /* handle what was in slot0 before we occupy it */
                if (slot0)
                        list_add_tail(&slot0->list, list);
                slot0 = wnext->skb;
                wnext->skb = NULL;
        }

        /* slot0 is now either NULL (in which case it's what we now are waiting
         * for, or a buf in which case we need to handle it like we received it;
         * however, we may be advancing past that buffer as well..
         */

        /* Handle case where we need to shift more than we had saved, slot0 will
         * be NULL iff savedlen is 0, otherwise slot0 will always be
         * non-NULL b/c we shifted the final element, which is always set if
         * there is any saved, into slot0.
         */
        if (savedlen < beyond) {
                if (savedlen != 0)
                        list_add_tail(&slot0->list, list);
                slot0 = NULL;
                /* slot0 has had an empty slot pushed into it */
        }

        /* Remove the entries */
        __vec_shift(xtfs, beyond);

        /* Advance want seq */
        xtfs->w_wantseq += beyond;

        /* Process drops here when implementing congestion control */

        /* We've shifted. plug the packet in at the end. */
        xtfs->w_savedlen = nslots - 1;
        xtfs->w_saved[xtfs->w_savedlen - 1].skb = inskb;
        iptfs_set_window_drop_times(xtfs, xtfs->w_savedlen - 1);

        /* if we don't have a slot0 then we must wait for it */
        if (!slot0)
                return;

        /* If slot0, seq must match new want seq */

        /* slot0 is valid, treat like we received expected. */
        __reorder_this(xtfs, slot0, list);
}

/* Receive a new packet into the reorder window. Return a list of ordered
 * packets from the window.
 */
static void iptfs_input_reorder(struct xfrm_iptfs_data *xtfs,
                                struct sk_buff *inskb, struct list_head *list,
                                struct list_head *freelist)
{
        const u32 nslots = xtfs->cfg.reorder_win_size + 1;
        u64 inseq = __esp_seq(inskb);
        u64 wantseq;

        assert_spin_locked(&xtfs->drop_lock);

        if (unlikely(!xtfs->w_seq_set)) {
                xtfs->w_seq_set = true;
                xtfs->w_wantseq = inseq;
        }
        wantseq = xtfs->w_wantseq;

        if (likely(inseq == wantseq))
                __reorder_this(xtfs, inskb, list);
        else if (inseq < wantseq)
                __reorder_past(xtfs, inskb, freelist);
        else if ((inseq - wantseq) < nslots)
                __reorder_future_fits(xtfs, inskb, freelist);
        else
                __reorder_future_shifts(xtfs, inskb, list);
}

/**
 * iptfs_drop_timer() - Handle drop timer expiry.
 * @me: the timer
 *
 * This is similar to our input function.
 *
 * The drop timer is set when we start an in progress reassembly, and also when
 * we save a future packet in the window saved array.
 *
 * NOTE packets in the save window are always newer WRT drop times as
 * they get further in the future. i.e. for:
 *
 *    if slots (S0, S1, ... Sn) and `Dn` is the drop time for slot `Sn`,
 *    then D(n-1) <= D(n).
 *
 * So, regardless of why the timer is firing we can always discard any inprogress
 * fragment; either it's the reassembly timer, or slot 0 is going to be
 * dropped as S0 must have the most recent drop time, and slot 0 holds the
 * continuation fragment of the in progress packet.
 *
 * Returns HRTIMER_NORESTART.
 */
static enum hrtimer_restart iptfs_drop_timer(struct hrtimer *me)
{
        struct sk_buff *skb, *next;
        struct list_head list;
        struct xfrm_iptfs_data *xtfs;
        struct xfrm_state *x;
        u32 count;

        xtfs = container_of(me, typeof(*xtfs), drop_timer);
        x = xtfs->x;

        INIT_LIST_HEAD(&list);

        spin_lock(&xtfs->drop_lock);

        /* Drop any in progress packet */
        skb = xtfs->ra_newskb;
        xtfs->ra_newskb = NULL;

        /* Now drop as many packets as we should from the reordering window
         * saved array
         */
        count = xtfs->w_savedlen ? __reorder_drop(xtfs, &list) : 0;

        spin_unlock(&xtfs->drop_lock);

        if (skb)
                kfree_skb_reason(skb, SKB_DROP_REASON_FRAG_REASM_TIMEOUT);

        if (count) {
                list_for_each_entry_safe(skb, next, &list, list) {
                        skb_list_del_init(skb);
                        iptfs_input_ordered(x, skb);
                }
        }

        return HRTIMER_NORESTART;
}

/**
 * iptfs_input() - handle receipt of iptfs payload
 * @x: xfrm state
 * @skb: the packet
 *
 * We have an IPTFS payload order it if needed, then process newly in order
 * packets.
 *
 * Return: -EINPROGRESS to inform xfrm_input to stop processing the skb.
 */
static int iptfs_input(struct xfrm_state *x, struct sk_buff *skb)
{
        struct list_head freelist, list;
        struct xfrm_iptfs_data *xtfs = x->mode_data;
        struct sk_buff *next;

        /* Fast path for no reorder window. */
        if (xtfs->cfg.reorder_win_size == 0) {
                iptfs_input_ordered(x, skb);
                goto done;
        }

        /* Fetch list of in-order packets from the reordering window as well as
         * a list of buffers we need to now free.
         */
        INIT_LIST_HEAD(&list);
        INIT_LIST_HEAD(&freelist);

        spin_lock(&xtfs->drop_lock);
        iptfs_input_reorder(xtfs, skb, &list, &freelist);
        spin_unlock(&xtfs->drop_lock);

        list_for_each_entry_safe(skb, next, &list, list) {
                skb_list_del_init(skb);
                iptfs_input_ordered(x, skb);
        }

        list_for_each_entry_safe(skb, next, &freelist, list) {
                skb_list_del_init(skb);
                kfree_skb(skb);
        }
done:
        /* We always have dealt with the input SKB, either we are re-using it,
         * or we have freed it. Return EINPROGRESS so that xfrm_input stops
         * processing it.
         */
        return -EINPROGRESS;
}

/* ================================= */
/* IPTFS Sending (ingress) Functions */
/* ================================= */

/* ------------------------- */
/* Enqueue to send functions */
/* ------------------------- */

/**
 * iptfs_enqueue() - enqueue packet if ok to send.
 * @xtfs: xtfs state
 * @skb: the packet
 *
 * Return: true if packet enqueued.
 */
static bool iptfs_enqueue(struct xfrm_iptfs_data *xtfs, struct sk_buff *skb)
{
        u64 newsz = xtfs->queue_size + skb->len;
        struct iphdr *iph;

        assert_spin_locked(&xtfs->x->lock);

        if (newsz > xtfs->cfg.max_queue_size)
                return false;

        /* Set ECN CE if we are above our ECN queue threshold */
        if (newsz > xtfs->ecn_queue_size) {
                iph = ip_hdr(skb);
                if (iph->version == 4)
                        IP_ECN_set_ce(iph);
                else if (iph->version == 6)
                        IP6_ECN_set_ce(skb, ipv6_hdr(skb));
        }

        __skb_queue_tail(&xtfs->queue, skb);
        xtfs->queue_size += skb->len;
        return true;
}

static int iptfs_get_cur_pmtu(struct xfrm_state *x, struct xfrm_iptfs_data *xtfs,
                              struct sk_buff *skb)
{
        struct xfrm_dst *xdst = (struct xfrm_dst *)skb_dst(skb);
        u32 payload_mtu = xtfs->payload_mtu;
        u32 pmtu = __iptfs_get_inner_mtu(x, xdst->child_mtu_cached);

        if (payload_mtu && payload_mtu < pmtu)
                pmtu = payload_mtu;

        return pmtu;
}

static int iptfs_is_too_big(struct sock *sk, struct sk_buff *skb, u32 pmtu)
{
        if (skb->len <= pmtu)
                return 0;

        /* We only send ICMP too big if the user has configured us as
         * dont-fragment.
         */
        if (skb->dev)
                XFRM_INC_STATS(dev_net(skb->dev), LINUX_MIB_XFRMOUTERROR);

        if (sk)
                xfrm_local_error(skb, pmtu);
        else if (ip_hdr(skb)->version == 4)
                icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, htonl(pmtu));
        else
                icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, pmtu);

        return 1;
}

/* IPv4/IPv6 packet ingress to IPTFS tunnel, arrange to send in IPTFS payload
 * (i.e., aggregating or fragmenting as appropriate).
 * This is set in dst->output for an SA.
 */
static int iptfs_output_collect(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        struct dst_entry *dst = skb_dst(skb);
        struct xfrm_state *x = dst->xfrm;
        struct xfrm_iptfs_data *xtfs = x->mode_data;
        struct sk_buff *segs, *nskb;
        u32 pmtu = 0;
        bool ok = true;
        bool was_gso;

        /* We have hooked into dst_entry->output which means we have skipped the
         * protocol specific netfilter (see xfrm4_output, xfrm6_output).
         * when our timer runs we will end up calling xfrm_output directly on
         * the encapsulated traffic.
         *
         * For both cases this is the NF_INET_POST_ROUTING hook which allows
         * changing the skb->dst entry which then may not be xfrm based anymore
         * in which case a REROUTED flag is set. and dst_output is called.
         *
         * For IPv6 we are also skipping fragmentation handling for local
         * sockets, which may or may not be good depending on our tunnel DF
         * setting. Normally with fragmentation supported we want to skip this
         * fragmentation.
         */

        if (xtfs->cfg.dont_frag)
                pmtu = iptfs_get_cur_pmtu(x, xtfs, skb);

        /* Break apart GSO skbs. If the queue is nearing full then we want the
         * accounting and queuing to be based on the individual packets not on the
         * aggregate GSO buffer.
         */
        was_gso = skb_is_gso(skb);
        if (!was_gso) {
                segs = skb;
        } else {
                segs = skb_gso_segment(skb, 0);
                if (IS_ERR_OR_NULL(segs)) {
                        XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
                        kfree_skb(skb);
                        if (IS_ERR(segs))
                                return PTR_ERR(segs);
                        return -EINVAL;
                }
                consume_skb(skb);
                skb = NULL;
        }

        /* We can be running on multiple cores and from the network softirq or
         * from user context depending on where the packet is coming from.
         */
        spin_lock_bh(&x->lock);

        skb_list_walk_safe(segs, skb, nskb) {
                skb_mark_not_on_list(skb);

                /* Once we drop due to no queue space we continue to drop the
                 * rest of the packets from that GRO.
                 */
                if (!ok) {
nospace:
                        trace_iptfs_no_queue_space(skb, xtfs, pmtu, was_gso);
                        XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOQSPACE);
                        kfree_skb_reason(skb, SKB_DROP_REASON_FULL_RING);
                        continue;
                }

                /* If the user indicated no iptfs fragmenting check before
                 * enqueue.
                 */
                if (xtfs->cfg.dont_frag && iptfs_is_too_big(sk, skb, pmtu)) {
                        trace_iptfs_too_big(skb, xtfs, pmtu, was_gso);
                        kfree_skb_reason(skb, SKB_DROP_REASON_PKT_TOO_BIG);
                        continue;
                }

                /* Enqueue to send in tunnel */
                ok = iptfs_enqueue(xtfs, skb);
                if (!ok)
                        goto nospace;

                trace_iptfs_enqueue(skb, xtfs, pmtu, was_gso);
        }

        /* Start a delay timer if we don't have one yet */
        if (!hrtimer_is_queued(&xtfs->iptfs_timer)) {
                hrtimer_start(&xtfs->iptfs_timer, xtfs->init_delay_ns, IPTFS_HRTIMER_MODE);
                xtfs->iptfs_settime = ktime_get_raw_fast_ns();
                trace_iptfs_timer_start(xtfs, xtfs->init_delay_ns);
        }

        spin_unlock_bh(&x->lock);
        return 0;
}

/* -------------------------- */
/* Dequeue and send functions */
/* -------------------------- */

static void iptfs_output_prepare_skb(struct sk_buff *skb, u32 blkoff)
{
        struct ip_iptfs_hdr *h;
        size_t hsz = sizeof(*h);

        /* now reset values to be pointing at the rest of the packets */
        h = skb_push(skb, hsz);
        memset(h, 0, hsz);
        if (blkoff)
                h->block_offset = htons(blkoff);

        /* network_header current points at the inner IP packet
         * move it to the iptfs header
         */
        skb->transport_header = skb->network_header;
        skb->network_header -= hsz;

        IPCB(skb)->flags |= IPSKB_XFRM_TUNNEL_SIZE;
}

/**
 * iptfs_copy_create_frag() - create an inner fragment skb.
 * @st: The source packet data.
 * @offset: offset in @st of the new fragment data.
 * @copy_len: the amount of data to copy from @st.
 *
 * Create a new skb holding a single IPTFS inner packet fragment. @copy_len must
 * not be greater than the max fragment size.
 *
 * Return: the new fragment skb or an ERR_PTR().
 */
static struct sk_buff *iptfs_copy_create_frag(struct skb_seq_state *st, u32 offset, u32 copy_len)
{
        struct sk_buff *src = st->root_skb;
        struct sk_buff *skb;
        int err;

        skb = iptfs_alloc_skb(src, copy_len, true);
        if (!skb)
                return ERR_PTR(-ENOMEM);

        /* Now copy `copy_len` data from src */
        err = skb_copy_seq_read(st, offset, skb_put(skb, copy_len), copy_len);
        if (err) {
                kfree_skb(skb);
                return ERR_PTR(err);
        }

        return skb;
}

/**
 * iptfs_copy_create_frags() - create and send N-1 fragments of a larger skb.
 * @skbp: the source packet skb (IN), skb holding the last fragment in
 *        the fragment stream (OUT).
 * @xtfs: IPTFS SA state.
 * @mtu: the max IPTFS fragment size.
 *
 * This function is responsible for fragmenting a larger inner packet into a
 * sequence of IPTFS payload packets. The last fragment is returned rather than
 * being sent so that the caller can append more inner packets (aggregation) if
 * there is room.
 *
 * Return: 0 on success or a negative error code on failure
 */
static int iptfs_copy_create_frags(struct sk_buff **skbp, struct xfrm_iptfs_data *xtfs, u32 mtu)
{
        struct skb_seq_state skbseq;
        struct list_head sublist;
        struct sk_buff *skb = *skbp;
        struct sk_buff *nskb = *skbp;
        u32 copy_len, offset;
        u32 to_copy = skb->len - mtu;
        u32 blkoff = 0;
        int err = 0;

        INIT_LIST_HEAD(&sublist);

        skb_prepare_seq_read(skb, 0, skb->len, &skbseq);

        /* A trimmed `skb` will be sent as the first fragment, later. */
        offset = mtu;
        to_copy = skb->len - offset;
        while (to_copy) {
                /* Send all but last fragment to allow agg. append */
                trace_iptfs_first_fragmenting(nskb, mtu, to_copy, NULL);
                list_add_tail(&nskb->list, &sublist);

                /* FUTURE: if the packet has an odd/non-aligning length we could
                 * send less data in the penultimate fragment so that the last
                 * fragment then ends on an aligned boundary.
                 */
                copy_len = min(to_copy, mtu);
                nskb = iptfs_copy_create_frag(&skbseq, offset, copy_len);
                if (IS_ERR(nskb)) {
                        XFRM_INC_STATS(xs_net(xtfs->x), LINUX_MIB_XFRMOUTERROR);
                        skb_abort_seq_read(&skbseq);
                        err = PTR_ERR(nskb);
                        nskb = NULL;
                        break;
                }
                iptfs_output_prepare_skb(nskb, to_copy);
                offset += copy_len;
                to_copy -= copy_len;
                blkoff = to_copy;
        }
        skb_abort_seq_read(&skbseq);

        /* return last fragment that will be unsent (or NULL) */
        *skbp = nskb;
        if (nskb)
                trace_iptfs_first_final_fragment(nskb, mtu, blkoff, NULL);

        /* trim the original skb to MTU */
        if (!err)
                err = pskb_trim(skb, mtu);

        if (err) {
                /* Free all frags. Don't bother sending a partial packet we will
                 * never complete.
                 */
                kfree_skb(nskb);
                list_for_each_entry_safe(skb, nskb, &sublist, list) {
                        skb_list_del_init(skb);
                        kfree_skb(skb);
                }
                return err;
        }

        /* prepare the initial fragment with an iptfs header */
        iptfs_output_prepare_skb(skb, 0);

        /* Send all but last fragment, if we fail to send a fragment then free
         * the rest -- no point in sending a packet that can't be reassembled.
         */
        list_for_each_entry_safe(skb, nskb, &sublist, list) {
                skb_list_del_init(skb);
                if (!err)
                        err = xfrm_output(NULL, skb);
                else
                        kfree_skb(skb);
        }
        if (err)
                kfree_skb(*skbp);
        return err;
}

/**
 * iptfs_first_skb() - handle the first dequeued inner packet for output
 * @skbp: the source packet skb (IN), skb holding the last fragment in
 *        the fragment stream (OUT).
 * @xtfs: IPTFS SA state.
 * @mtu: the max IPTFS fragment size.
 *
 * This function is responsible for fragmenting a larger inner packet into a
 * sequence of IPTFS payload packets.
 *
 * The last fragment is returned rather than being sent so that the caller can
 * append more inner packets (aggregation) if there is room.
 *
 * Return: 0 on success or a negative error code on failure
 */
static int iptfs_first_skb(struct sk_buff **skbp, struct xfrm_iptfs_data *xtfs, u32 mtu)
{
        struct sk_buff *skb = *skbp;
        int err;

        /* Classic ESP skips the don't fragment ICMP error if DF is clear on
         * the inner packet or ignore_df is set. Otherwise it will send an ICMP
         * or local error if the inner packet won't fit it's MTU.
         *
         * With IPTFS we do not care about the inner packet DF bit. If the
         * tunnel is configured to "don't fragment" we error back if things
         * don't fit in our max packet size. Otherwise we iptfs-fragment as
         * normal.
         */

        /* The opportunity for HW offload has ended */
        if (skb->ip_summed == CHECKSUM_PARTIAL) {
                err = skb_checksum_help(skb);
                if (err)
                        return err;
        }

        /* We've split gso up before queuing */

        trace_iptfs_first_dequeue(skb, mtu, 0, ip_hdr(skb));

        /* Consider the buffer Tx'd and no longer owned */
        skb_orphan(skb);

        /* Simple case -- it fits. `mtu` accounted for all the overhead
         * including the basic IPTFS header.
         */
        if (skb->len <= mtu) {
                iptfs_output_prepare_skb(skb, 0);
                return 0;
        }

        return iptfs_copy_create_frags(skbp, xtfs, mtu);
}

static struct sk_buff **iptfs_rehome_fraglist(struct sk_buff **nextp, struct sk_buff *child)
{
        u32 fllen = 0;

        /* It might be possible to account for a frag list in addition to page
         * fragment if it's a valid state to be in. The page fragments size
         * should be kept as data_len so only the frag_list size is removed,
         * this must be done above as well.
         */
        *nextp = skb_shinfo(child)->frag_list;
        while (*nextp) {
                fllen += (*nextp)->len;
                nextp = &(*nextp)->next;
        }
        skb_frag_list_init(child);
        child->len -= fllen;
        child->data_len -= fllen;

        return nextp;
}

static void iptfs_consume_frags(struct sk_buff *to, struct sk_buff *from)
{
        struct skb_shared_info *fromi = skb_shinfo(from);
        struct skb_shared_info *toi = skb_shinfo(to);
        unsigned int new_truesize;

        /* If we have data in a head page, grab it */
        if (!skb_headlen(from)) {
                new_truesize = SKB_TRUESIZE(skb_end_offset(from));
        } else {
                iptfs_skb_head_to_frag(from, &toi->frags[toi->nr_frags]);
                skb_frag_ref(to, toi->nr_frags++);
                new_truesize = SKB_DATA_ALIGN(sizeof(struct sk_buff));
        }

        /* Move any other page fragments rather than copy */
        memcpy(&toi->frags[toi->nr_frags], fromi->frags,
               sizeof(fromi->frags[0]) * fromi->nr_frags);
        toi->nr_frags += fromi->nr_frags;
        fromi->nr_frags = 0;
        from->data_len = 0;
        from->len = 0;
        to->truesize += from->truesize - new_truesize;
        from->truesize = new_truesize;

        /* We are done with this SKB */
        consume_skb(from);
}

static void iptfs_output_queued(struct xfrm_state *x, struct sk_buff_head *list)
{
        struct xfrm_iptfs_data *xtfs = x->mode_data;
        struct sk_buff *skb, *skb2, **nextp;
        struct skb_shared_info *shi, *shi2;

        /* If we are fragmenting due to a large inner packet we will output all
         * the outer IPTFS packets required to contain the fragments of the
         * single large inner packet. These outer packets need to be sent
         * consecutively (ESP seq-wise). Since this output function is always
         * running from a timer we do not need a lock to provide this guarantee.
         * We will output our packets consecutively before the timer is allowed
         * to run again on some other CPU.
         */

        while ((skb = __skb_dequeue(list))) {
                u32 mtu = iptfs_get_cur_pmtu(x, xtfs, skb);
                bool share_ok = true;
                int remaining;

                /* protocol comes to us cleared sometimes */
                skb->protocol = x->outer_mode.family == AF_INET ? htons(ETH_P_IP) :
                                                                  htons(ETH_P_IPV6);

                if (skb->len > mtu && xtfs->cfg.dont_frag) {
                        /* We handle this case before enqueueing so we are only
                         * here b/c MTU changed after we enqueued before we
                         * dequeued, just drop these.
                         */
                        XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTERROR);

                        trace_iptfs_first_toobig(skb, mtu, 0, ip_hdr(skb));
                        kfree_skb_reason(skb, SKB_DROP_REASON_PKT_TOO_BIG);
                        continue;
                }

                /* Convert first inner packet into an outer IPTFS packet,
                 * dealing with any fragmentation into multiple outer packets
                 * if necessary.
                 */
                if (iptfs_first_skb(&skb, xtfs, mtu))
                        continue;

                /* If fragmentation was required the returned skb is the last
                 * IPTFS fragment in the chain, and it's IPTFS header blkoff has
                 * been set just past the end of the fragment data.
                 *
                 * In either case the space remaining to send more inner packet
                 * data is `mtu` - (skb->len - sizeof iptfs header). This is b/c
                 * the `mtu` value has the basic IPTFS header len accounted for,
                 * and we added that header to the skb so it is a part of
                 * skb->len, thus we subtract it from the skb length.
                 */
                remaining = mtu - (skb->len - sizeof(struct ip_iptfs_hdr));

                /* Re-home (un-nest) nested fragment lists. We need to do this
                 * b/c we will simply be appending any following aggregated
                 * inner packets using the frag list.
                 */
                shi = skb_shinfo(skb);
                nextp = &shi->frag_list;
                while (*nextp) {
                        if (skb_has_frag_list(*nextp))
                                nextp = iptfs_rehome_fraglist(&(*nextp)->next, *nextp);
                        else
                                nextp = &(*nextp)->next;
                }

                if (shi->frag_list || skb_cloned(skb) || skb_shared(skb))
                        share_ok = false;

                /* See if we have enough space to simply append.
                 *
                 * NOTE: Maybe do not append if we will be mis-aligned,
                 * SW-based endpoints will probably have to copy in this
                 * case.
                 */
                while ((skb2 = skb_peek(list))) {
                        trace_iptfs_ingress_nth_peek(skb2, remaining);
                        if (skb2->len > remaining)
                                break;

                        __skb_unlink(skb2, list);

                        /* Consider the buffer Tx'd and no longer owned */
                        skb_orphan(skb);

                        /* If we don't have a cksum in the packet we need to add
                         * one before encapsulation.
                         */
                        if (skb2->ip_summed == CHECKSUM_PARTIAL) {
                                if (skb_checksum_help(skb2)) {
                                        XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTERROR);
                                        kfree_skb(skb2);
                                        continue;
                                }
                        }

                        /* skb->pp_recycle is passed to __skb_flag_unref for all
                         * frag pages so we can only share pages with skb's who
                         * match ourselves.
                         */
                        shi2 = skb_shinfo(skb2);
                        if (share_ok &&
                            (shi2->frag_list ||
                             (!skb2->head_frag && skb_headlen(skb)) ||
                             skb->pp_recycle != skb2->pp_recycle ||
                             skb_zcopy(skb2) ||
                             (shi->nr_frags + shi2->nr_frags + 1 > MAX_SKB_FRAGS)))
                                share_ok = false;

                        /* Do accounting */
                        skb->data_len += skb2->len;
                        skb->len += skb2->len;
                        remaining -= skb2->len;

                        trace_iptfs_ingress_nth_add(skb2, share_ok);

                        if (share_ok) {
                                iptfs_consume_frags(skb, skb2);
                        } else {
                                /* Append to the frag_list */
                                *nextp = skb2;
                                nextp = &skb2->next;
                                if (skb_has_frag_list(skb2))
                                        nextp = iptfs_rehome_fraglist(nextp,
                                                                      skb2);
                                skb->truesize += skb2->truesize;
                        }
                }

                xfrm_output(NULL, skb);
        }
}

static enum hrtimer_restart iptfs_delay_timer(struct hrtimer *me)
{
        struct sk_buff_head list;
        struct xfrm_iptfs_data *xtfs;
        struct xfrm_state *x;
        time64_t settime;

        xtfs = container_of(me, typeof(*xtfs), iptfs_timer);
        x = xtfs->x;

        /* Process all the queued packets
         *
         * softirq execution order: timer > tasklet > hrtimer
         *
         * Network rx will have run before us giving one last chance to queue
         * ingress packets for us to process and transmit.
         */

        spin_lock(&x->lock);
        __skb_queue_head_init(&list);
        skb_queue_splice_init(&xtfs->queue, &list);
        xtfs->queue_size = 0;
        settime = xtfs->iptfs_settime;
        spin_unlock(&x->lock);

        /* After the above unlock, packets can begin queuing again, and the
         * timer can be set again, from another CPU either in softirq or user
         * context (not from this one since we are running at softirq level
         * already).
         */

        trace_iptfs_timer_expire(xtfs, (unsigned long long)(ktime_get_raw_fast_ns() - settime));

        iptfs_output_queued(x, &list);

        return HRTIMER_NORESTART;
}

/**
 * iptfs_encap_add_ipv4() - add outer encaps
 * @x: xfrm state
 * @skb: the packet
 *
 * This was originally taken from xfrm4_tunnel_encap_add. The reason for the
 * copy is that IP-TFS/AGGFRAG can have different functionality for how to set
 * the TOS/DSCP bits. Sets the protocol to a different value and doesn't do
 * anything with inner headers as they aren't pointing into a normal IP
 * singleton inner packet.
 *
 * Return: 0 on success or a negative error code on failure
 */
static int iptfs_encap_add_ipv4(struct xfrm_state *x, struct sk_buff *skb)
{
        struct dst_entry *dst = skb_dst(skb);
        struct iphdr *top_iph;

        skb_reset_inner_network_header(skb);
        skb_reset_inner_transport_header(skb);

        skb_set_network_header(skb, -(x->props.header_len - x->props.enc_hdr_len));
        skb->mac_header = skb->network_header + offsetof(struct iphdr, protocol);
        skb->transport_header = skb->network_header + sizeof(*top_iph);

        top_iph = ip_hdr(skb);
        top_iph->ihl = 5;
        top_iph->version = 4;
        top_iph->protocol = IPPROTO_AGGFRAG;

        /* As we have 0, fractional, 1 or N inner packets there's no obviously
         * correct DSCP mapping to inherit. ECN should be cleared per RFC9347
         * 3.1.
         */
        top_iph->tos = 0;

        top_iph->frag_off = htons(IP_DF);
        top_iph->ttl = ip4_dst_hoplimit(xfrm_dst_child(dst));
        top_iph->saddr = x->props.saddr.a4;
        top_iph->daddr = x->id.daddr.a4;
        ip_select_ident(dev_net(dst->dev), skb, NULL);

        return 0;
}

#if IS_ENABLED(CONFIG_IPV6)
/**
 * iptfs_encap_add_ipv6() - add outer encaps
 * @x: xfrm state
 * @skb: the packet
 *
 * This was originally taken from xfrm6_tunnel_encap_add. The reason for the
 * copy is that IP-TFS/AGGFRAG can have different functionality for how to set
 * the flow label and TOS/DSCP bits. It also sets the protocol to a different
 * value and doesn't do anything with inner headers as they aren't pointing into
 * a normal IP singleton inner packet.
 *
 * Return: 0 on success or a negative error code on failure
 */
static int iptfs_encap_add_ipv6(struct xfrm_state *x, struct sk_buff *skb)
{
        struct dst_entry *dst = skb_dst(skb);
        struct ipv6hdr *top_iph;
        int dsfield;

        skb_reset_inner_network_header(skb);
        skb_reset_inner_transport_header(skb);

        skb_set_network_header(skb, -x->props.header_len + x->props.enc_hdr_len);
        skb->mac_header = skb->network_header + offsetof(struct ipv6hdr, nexthdr);
        skb->transport_header = skb->network_header + sizeof(*top_iph);

        top_iph = ipv6_hdr(skb);
        top_iph->version = 6;
        top_iph->priority = 0;
        memset(top_iph->flow_lbl, 0, sizeof(top_iph->flow_lbl));
        top_iph->nexthdr = IPPROTO_AGGFRAG;

        /* As we have 0, fractional, 1 or N inner packets there's no obviously
         * correct DSCP mapping to inherit. ECN should be cleared per RFC9347
         * 3.1.
         */
        dsfield = 0;
        ipv6_change_dsfield(top_iph, 0, dsfield);

        top_iph->hop_limit = ip6_dst_hoplimit(xfrm_dst_child(dst));
        top_iph->saddr = *(struct in6_addr *)&x->props.saddr;
        top_iph->daddr = *(struct in6_addr *)&x->id.daddr;

        return 0;
}
#endif

/**
 * iptfs_prepare_output() -  prepare the skb for output
 * @x: xfrm state
 * @skb: the packet
 *
 * Return: Error value, if 0 then skb values should be as follows:
 *    - transport_header should point at ESP header
 *    - network_header should point at Outer IP header
 *    - mac_header should point at protocol/nexthdr of the outer IP
 */
static int iptfs_prepare_output(struct xfrm_state *x, struct sk_buff *skb)
{
        if (x->outer_mode.family == AF_INET)
                return iptfs_encap_add_ipv4(x, skb);
        if (x->outer_mode.family == AF_INET6) {
#if IS_ENABLED(CONFIG_IPV6)
                return iptfs_encap_add_ipv6(x, skb);
#else
                return -EAFNOSUPPORT;
#endif
        }
        return -EOPNOTSUPP;
}

/* ========================== */
/* State Management Functions */
/* ========================== */

/**
 * __iptfs_get_inner_mtu() - return inner MTU with no fragmentation.
 * @x: xfrm state.
 * @outer_mtu: the outer mtu
 *
 * Return: Correct MTU taking in to account the encap overhead.
 */
static u32 __iptfs_get_inner_mtu(struct xfrm_state *x, int outer_mtu)
{
        struct crypto_aead *aead;
        u32 blksize;

        aead = x->data;
        blksize = ALIGN(crypto_aead_blocksize(aead), 4);
        return ((outer_mtu - x->props.header_len - crypto_aead_authsize(aead)) &
                ~(blksize - 1)) - 2;
}

/**
 * iptfs_get_inner_mtu() - return the inner MTU for an IPTFS xfrm.
 * @x: xfrm state.
 * @outer_mtu: Outer MTU for the encapsulated packet.
 *
 * Return: Correct MTU taking in to account the encap overhead.
 */
static u32 iptfs_get_inner_mtu(struct xfrm_state *x, int outer_mtu)
{
        struct xfrm_iptfs_data *xtfs = x->mode_data;

        /* If not dont-frag we have no MTU */
        if (!xtfs->cfg.dont_frag)
                return x->outer_mode.family == AF_INET ? IP_MAX_MTU : IP6_MAX_MTU;
        return __iptfs_get_inner_mtu(x, outer_mtu);
}

/**
 * iptfs_user_init() - initialize the SA with IPTFS options from netlink.
 * @net: the net data
 * @x: xfrm state
 * @attrs: netlink attributes
 * @extack: extack return data
 *
 * Return: 0 on success or a negative error code on failure
 */
static int iptfs_user_init(struct net *net, struct xfrm_state *x,
                           struct nlattr **attrs,
                           struct netlink_ext_ack *extack)
{
        struct xfrm_iptfs_data *xtfs = x->mode_data;
        struct xfrm_iptfs_config *xc;
        u64 q;

        xc = &xtfs->cfg;
        xc->max_queue_size = IPTFS_DEFAULT_MAX_QUEUE_SIZE;
        xc->reorder_win_size = IPTFS_DEFAULT_REORDER_WINDOW;
        xtfs->drop_time_ns = IPTFS_DEFAULT_DROP_TIME_USECS * NSECS_IN_USEC;
        xtfs->init_delay_ns = IPTFS_DEFAULT_INIT_DELAY_USECS * NSECS_IN_USEC;

        if (attrs[XFRMA_IPTFS_DONT_FRAG])
                xc->dont_frag = true;
        if (attrs[XFRMA_IPTFS_REORDER_WINDOW])
                xc->reorder_win_size =
                        nla_get_u16(attrs[XFRMA_IPTFS_REORDER_WINDOW]);
        /* saved array is for saving 1..N seq nums from wantseq */
        if (xc->reorder_win_size) {
                xtfs->w_saved = kcalloc(xc->reorder_win_size,
                                        sizeof(*xtfs->w_saved), GFP_KERNEL);
                if (!xtfs->w_saved) {
                        NL_SET_ERR_MSG(extack, "Cannot alloc reorder window");
                        return -ENOMEM;
                }
        }
        if (attrs[XFRMA_IPTFS_PKT_SIZE]) {
                xc->pkt_size = nla_get_u32(attrs[XFRMA_IPTFS_PKT_SIZE]);
                if (!xc->pkt_size) {
                        xtfs->payload_mtu = 0;
                } else if (xc->pkt_size > x->props.header_len) {
                        xtfs->payload_mtu = xc->pkt_size - x->props.header_len;
                } else {
                        NL_SET_ERR_MSG(extack,
                                       "Packet size must be 0 or greater than IPTFS/ESP header length");
                        return -EINVAL;
                }
        }
        if (attrs[XFRMA_IPTFS_MAX_QSIZE])
                xc->max_queue_size = nla_get_u32(attrs[XFRMA_IPTFS_MAX_QSIZE]);
        if (attrs[XFRMA_IPTFS_DROP_TIME])
                xtfs->drop_time_ns =
                        (u64)nla_get_u32(attrs[XFRMA_IPTFS_DROP_TIME]) *
                        NSECS_IN_USEC;
        if (attrs[XFRMA_IPTFS_INIT_DELAY])
                xtfs->init_delay_ns =
                        (u64)nla_get_u32(attrs[XFRMA_IPTFS_INIT_DELAY]) * NSECS_IN_USEC;

        q = (u64)xc->max_queue_size * 95;
        do_div(q, 100);
        xtfs->ecn_queue_size = (u32)q;

        return 0;
}

static unsigned int iptfs_sa_len(const struct xfrm_state *x)
{
        struct xfrm_iptfs_data *xtfs = x->mode_data;
        struct xfrm_iptfs_config *xc = &xtfs->cfg;
        unsigned int l = 0;

        if (x->dir == XFRM_SA_DIR_IN) {
                l += nla_total_size(sizeof(u32)); /* drop time usec */
                l += nla_total_size(sizeof(xc->reorder_win_size));
        } else {
                if (xc->dont_frag)
                        l += nla_total_size(0);   /* dont-frag flag */
                l += nla_total_size(sizeof(u32)); /* init delay usec */
                l += nla_total_size(sizeof(xc->max_queue_size));
                l += nla_total_size(sizeof(xc->pkt_size));
        }

        return l;
}

static int iptfs_copy_to_user(struct xfrm_state *x, struct sk_buff *skb)
{
        struct xfrm_iptfs_data *xtfs = x->mode_data;
        struct xfrm_iptfs_config *xc = &xtfs->cfg;
        int ret = 0;
        u64 q;

        if (x->dir == XFRM_SA_DIR_IN) {
                q = xtfs->drop_time_ns;
                do_div(q, NSECS_IN_USEC);
                ret = nla_put_u32(skb, XFRMA_IPTFS_DROP_TIME, q);
                if (ret)
                        return ret;

                ret = nla_put_u16(skb, XFRMA_IPTFS_REORDER_WINDOW,
                                  xc->reorder_win_size);
        } else {
                if (xc->dont_frag) {
                        ret = nla_put_flag(skb, XFRMA_IPTFS_DONT_FRAG);
                        if (ret)
                                return ret;
                }

                q = xtfs->init_delay_ns;
                do_div(q, NSECS_IN_USEC);
                ret = nla_put_u32(skb, XFRMA_IPTFS_INIT_DELAY, q);
                if (ret)
                        return ret;

                ret = nla_put_u32(skb, XFRMA_IPTFS_MAX_QSIZE, xc->max_queue_size);
                if (ret)
                        return ret;

                ret = nla_put_u32(skb, XFRMA_IPTFS_PKT_SIZE, xc->pkt_size);
        }

        return ret;
}

static void __iptfs_init_state(struct xfrm_state *x,
                               struct xfrm_iptfs_data *xtfs)
{
        __skb_queue_head_init(&xtfs->queue);
        hrtimer_init(&xtfs->iptfs_timer, CLOCK_MONOTONIC, IPTFS_HRTIMER_MODE);
        xtfs->iptfs_timer.function = iptfs_delay_timer;

        spin_lock_init(&xtfs->drop_lock);
        hrtimer_init(&xtfs->drop_timer, CLOCK_MONOTONIC, IPTFS_HRTIMER_MODE);
        xtfs->drop_timer.function = iptfs_drop_timer;

        /* Modify type (esp) adjustment values */

        if (x->props.family == AF_INET)
                x->props.header_len += sizeof(struct iphdr) + sizeof(struct ip_iptfs_hdr);
        else if (x->props.family == AF_INET6)
                x->props.header_len += sizeof(struct ipv6hdr) + sizeof(struct ip_iptfs_hdr);
        x->props.enc_hdr_len = sizeof(struct ip_iptfs_hdr);

        /* Always keep a module reference when x->mode_data is set */
        __module_get(x->mode_cbs->owner);

        x->mode_data = xtfs;
        xtfs->x = x;
}

static int iptfs_clone_state(struct xfrm_state *x, struct xfrm_state *orig)
{
        struct xfrm_iptfs_data *xtfs;

        xtfs = kmemdup(orig->mode_data, sizeof(*xtfs), GFP_KERNEL);
        if (!xtfs)
                return -ENOMEM;

        x->mode_data = xtfs;
        xtfs->x = x;

        xtfs->ra_newskb = NULL;
        if (xtfs->cfg.reorder_win_size) {
                xtfs->w_saved = kcalloc(xtfs->cfg.reorder_win_size,
                                        sizeof(*xtfs->w_saved), GFP_KERNEL);
                if (!xtfs->w_saved) {
                        kfree_sensitive(xtfs);
                        return -ENOMEM;
                }
        }

        return 0;
}

static int iptfs_init_state(struct xfrm_state *x)
{
        struct xfrm_iptfs_data *xtfs;

        if (x->mode_data) {
                /* We have arrived here from xfrm_state_clone() */
                xtfs = x->mode_data;
        } else {
                xtfs = kzalloc(sizeof(*xtfs), GFP_KERNEL);
                if (!xtfs)
                        return -ENOMEM;
        }

        __iptfs_init_state(x, xtfs);

        return 0;
}

static void iptfs_destroy_state(struct xfrm_state *x)
{
        struct xfrm_iptfs_data *xtfs = x->mode_data;
        struct sk_buff_head list;
        struct skb_wseq *s, *se;
        struct sk_buff *skb;

        if (!xtfs)
                return;

        spin_lock_bh(&xtfs->x->lock);
        hrtimer_cancel(&xtfs->iptfs_timer);
        __skb_queue_head_init(&list);
        skb_queue_splice_init(&xtfs->queue, &list);
        spin_unlock_bh(&xtfs->x->lock);

        while ((skb = __skb_dequeue(&list)))
                kfree_skb(skb);

        spin_lock_bh(&xtfs->drop_lock);
        hrtimer_cancel(&xtfs->drop_timer);
        spin_unlock_bh(&xtfs->drop_lock);

        if (xtfs->ra_newskb)
                kfree_skb(xtfs->ra_newskb);

        for (s = xtfs->w_saved, se = s + xtfs->w_savedlen; s < se; s++) {
                if (s->skb)
                        kfree_skb(s->skb);
        }

        kfree_sensitive(xtfs->w_saved);
        kfree_sensitive(xtfs);

        module_put(x->mode_cbs->owner);
}

static const struct xfrm_mode_cbs iptfs_mode_cbs = {
        .owner = THIS_MODULE,
        .init_state = iptfs_init_state,
        .clone_state = iptfs_clone_state,
        .destroy_state = iptfs_destroy_state,
        .user_init = iptfs_user_init,
        .copy_to_user = iptfs_copy_to_user,
        .sa_len = iptfs_sa_len,
        .get_inner_mtu = iptfs_get_inner_mtu,
        .input = iptfs_input,
        .output = iptfs_output_collect,
        .prepare_output = iptfs_prepare_output,
};

static int __init xfrm_iptfs_init(void)
{
        int err;

        pr_info("xfrm_iptfs: IPsec IP-TFS tunnel mode module\n");

        err = xfrm_register_mode_cbs(XFRM_MODE_IPTFS, &iptfs_mode_cbs);
        if (err < 0)
                pr_info("%s: can't register IP-TFS\n", __func__);

        return err;
}

static void __exit xfrm_iptfs_fini(void)
{
        xfrm_unregister_mode_cbs(XFRM_MODE_IPTFS);
}

module_init(xfrm_iptfs_init);
module_exit(xfrm_iptfs_fini);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("IP-TFS support for xfrm ipsec tunnels");