[linux.git] / net / ipv4 / gre_offload.c

/*
 *	IPV4 GSO/GRO offload support
 *	Linux INET implementation
 *
 *	This program is free software; you can redistribute it and/or
 *	modify it under the terms of the GNU General Public License
 *	as published by the Free Software Foundation; either version
 *	2 of the License, or (at your option) any later version.
 *
 *	GRE GSO support
 */

#include <linux/skbuff.h>
#include <linux/init.h>
#include <net/protocol.h>
#include <net/gre.h>

static struct sk_buff *gre_gso_segment(struct sk_buff *skb,
				       netdev_features_t features)
{
	struct sk_buff *segs = ERR_PTR(-EINVAL);
	netdev_features_t enc_features;
	int ghl;
	struct gre_base_hdr *greh;
	u16 mac_offset = skb->mac_header;
	int mac_len = skb->mac_len;
	__be16 protocol = skb->protocol;
	int tnl_hlen;
	bool csum;

	if (unlikely(skb_shinfo(skb)->gso_type &
				~(SKB_GSO_TCPV4 |
				  SKB_GSO_TCPV6 |
				  SKB_GSO_UDP |
				  SKB_GSO_DODGY |
				  SKB_GSO_TCP_ECN |
				  SKB_GSO_GRE |
				  SKB_GSO_GRE_CSUM |
				  SKB_GSO_IPIP |
				  SKB_GSO_SIT)))
		goto out;

	if (!skb->encapsulation)
		goto out;

	if (unlikely(!pskb_may_pull(skb, sizeof(*greh))))
		goto out;

	greh = (struct gre_base_hdr *)skb_transport_header(skb);

	ghl = skb_inner_mac_header(skb) - skb_transport_header(skb);
	if (unlikely(ghl < sizeof(*greh)))
		goto out;

	csum = !!(greh->flags & GRE_CSUM);
	if (csum)
		skb->encap_hdr_csum = 1;

	/* setup inner skb. */
	skb->protocol = greh->protocol;
	skb->encapsulation = 0;

	if (unlikely(!pskb_may_pull(skb, ghl)))
		goto out;

	__skb_pull(skb, ghl);
	skb_reset_mac_header(skb);
	skb_set_network_header(skb, skb_inner_network_offset(skb));
	skb->mac_len = skb_inner_network_offset(skb);

	/* segment inner packet. */
	enc_features = skb->dev->hw_enc_features & features;
	segs = skb_mac_gso_segment(skb, enc_features);
	if (IS_ERR_OR_NULL(segs)) {
		skb_gso_error_unwind(skb, protocol, ghl, mac_offset, mac_len);
		goto out;
	}

	skb = segs;
	tnl_hlen = skb_tnl_header_len(skb);
	do {
		__skb_push(skb, ghl);
		if (csum) {
			__be32 *pcsum;

			if (skb_has_shared_frag(skb)) {
				int err;

				err = __skb_linearize(skb);
				if (err) {
					kfree_skb_list(segs);
					segs = ERR_PTR(err);
					goto out;
				}
			}

			skb_reset_transport_header(skb);

			greh = (struct gre_base_hdr *)
			    skb_transport_header(skb);
			pcsum = (__be32 *)(greh + 1);
			*pcsum = 0;
			*(__sum16 *)pcsum = gso_make_checksum(skb, 0);
		}
		__skb_push(skb, tnl_hlen - ghl);

		skb_reset_inner_headers(skb);
		skb->encapsulation = 1;

		skb_reset_mac_header(skb);
		skb_set_network_header(skb, mac_len);
		skb->mac_len = mac_len;
		skb->protocol = protocol;
	} while ((skb = skb->next));
out:
	return segs;
}

static struct sk_buff **gre_gro_receive(struct sk_buff **head,
					struct sk_buff *skb)
{
	struct sk_buff **pp = NULL;
	struct sk_buff *p;
	const struct gre_base_hdr *greh;
	unsigned int hlen, grehlen;
	unsigned int off;
	int flush = 1;
	struct packet_offload *ptype;
	__be16 type;

	off = skb_gro_offset(skb);
	hlen = off + sizeof(*greh);
	greh = skb_gro_header_fast(skb, off);
	if (skb_gro_header_hard(skb, hlen)) {
		greh = skb_gro_header_slow(skb, hlen, off);
		if (unlikely(!greh))
			goto out;
	}

	/* Only support version 0 and K (key), C (csum) flags. Note that
	 * although the support for the S (seq#) flag can be added easily
	 * for GRO, this is problematic for GSO hence can not be enabled
	 * here because a GRO pkt may end up in the forwarding path, thus
	 * requiring GSO support to break it up correctly.
	 */
	if ((greh->flags & ~(GRE_KEY|GRE_CSUM)) != 0)
		goto out;

	type = greh->protocol;

	rcu_read_lock();
	ptype = gro_find_receive_by_type(type);
	if (!ptype)
		goto out_unlock;

	grehlen = GRE_HEADER_SECTION;

	if (greh->flags & GRE_KEY)
		grehlen += GRE_HEADER_SECTION;

	if (greh->flags & GRE_CSUM)
		grehlen += GRE_HEADER_SECTION;

	hlen = off + grehlen;
	if (skb_gro_header_hard(skb, hlen)) {
		greh = skb_gro_header_slow(skb, hlen, off);
		if (unlikely(!greh))
			goto out_unlock;
	}

	/* Don't bother verifying checksum if we're going to flush anyway. */
	if ((greh->flags & GRE_CSUM) && !NAPI_GRO_CB(skb)->flush) {
		if (skb_gro_checksum_simple_validate(skb))
			goto out_unlock;

		skb_gro_checksum_try_convert(skb, IPPROTO_GRE, 0,
					     null_compute_pseudo);
	}

	flush = 0;

	for (p = *head; p; p = p->next) {
		const struct gre_base_hdr *greh2;

		if (!NAPI_GRO_CB(p)->same_flow)
			continue;

		/* The following checks are needed to ensure only pkts
		 * from the same tunnel are considered for aggregation.
		 * The criteria for "the same tunnel" includes:
		 * 1) same version (we only support version 0 here)
		 * 2) same protocol (we only support ETH_P_IP for now)
		 * 3) same set of flags
		 * 4) same key if the key field is present.
		 */
		greh2 = (struct gre_base_hdr *)(p->data + off);

		if (greh2->flags != greh->flags ||
		    greh2->protocol != greh->protocol) {
			NAPI_GRO_CB(p)->same_flow = 0;
			continue;
		}
		if (greh->flags & GRE_KEY) {
			/* compare keys */
			if (*(__be32 *)(greh2+1) != *(__be32 *)(greh+1)) {
				NAPI_GRO_CB(p)->same_flow = 0;
				continue;
			}
		}
	}

	skb_gro_pull(skb, grehlen);

	/* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/
	skb_gro_postpull_rcsum(skb, greh, grehlen);

	pp = ptype->callbacks.gro_receive(head, skb);

out_unlock:
	rcu_read_unlock();
out:
	NAPI_GRO_CB(skb)->flush |= flush;

	return pp;
}

static int gre_gro_complete(struct sk_buff *skb, int nhoff)
{
	struct gre_base_hdr *greh = (struct gre_base_hdr *)(skb->data + nhoff);
	struct packet_offload *ptype;
	unsigned int grehlen = sizeof(*greh);
	int err = -ENOENT;
	__be16 type;

	skb->encapsulation = 1;
	skb_shinfo(skb)->gso_type = SKB_GSO_GRE;

	type = greh->protocol;
	if (greh->flags & GRE_KEY)
		grehlen += GRE_HEADER_SECTION;

	if (greh->flags & GRE_CSUM)
		grehlen += GRE_HEADER_SECTION;

	rcu_read_lock();
	ptype = gro_find_complete_by_type(type);
	if (ptype)
		err = ptype->callbacks.gro_complete(skb, nhoff + grehlen);

	rcu_read_unlock();

	skb_set_inner_mac_header(skb, nhoff + grehlen);

	return err;
}

static const struct net_offload gre_offload = {
	.callbacks = {
		.gso_segment = gre_gso_segment,
		.gro_receive = gre_gro_receive,
		.gro_complete = gre_gro_complete,
	},
};

static int __init gre_offload_init(void)
{
	return inet_add_offload(&gre_offload, IPPROTO_GRE);
}
device_initcall(gre_offload_init);
Commit	Line	Data
c50cd357 DB	1	/*
	2	* IPV4 GSO/GRO offload support
	3	* Linux INET implementation
	4	*
	5	* This program is free software; you can redistribute it and/or
	6	* modify it under the terms of the GNU General Public License
	7	* as published by the Free Software Foundation; either version
	8	* 2 of the License, or (at your option) any later version.
	9	*
	10	* GRE GSO support
	11	*/
	12
	13	#include <linux/skbuff.h>
cf172283	14	#include <linux/init.h>
c50cd357 DB	15	#include <net/protocol.h>
	16	#include <net/gre.h>
	17
c50cd357 DB	18	static struct sk_buff gre_gso_segment(struct sk_buff skb,
	19	netdev_features_t features)
	20	{
	21	struct sk_buff *segs = ERR_PTR(-EINVAL);
	22	netdev_features_t enc_features;
b884b1a4	23	int ghl;
c50cd357	24	struct gre_base_hdr *greh;
7a7ffbab	25	u16 mac_offset = skb->mac_header;
c50cd357 DB	26	int mac_len = skb->mac_len;
	27	__be16 protocol = skb->protocol;
	28	int tnl_hlen;
	29	bool csum;
	30
	31	if (unlikely(skb_shinfo(skb)->gso_type &
	32	~(SKB_GSO_TCPV4 \|
	33	SKB_GSO_TCPV6 \|
	34	SKB_GSO_UDP \|
	35	SKB_GSO_DODGY \|
	36	SKB_GSO_TCP_ECN \|
cb32f511	37	SKB_GSO_GRE \|
4749c09c	38	SKB_GSO_GRE_CSUM \|
7e3b6e74 ED	39	SKB_GSO_IPIP \|
7e3b6e74 ED	40	SKB_GSO_SIT)))
c50cd357 DB	41	goto out;
c50cd357 DB	42
53e50398 TH	43	if (!skb->encapsulation)
	44	goto out;
	45
c50cd357 DB	46	if (unlikely(!pskb_may_pull(skb, sizeof(*greh))))
	47	goto out;
	48
	49	greh = (struct gre_base_hdr *)skb_transport_header(skb);
	50
14051f04	51	ghl = skb_inner_mac_header(skb) - skb_transport_header(skb);
b884b1a4 NC	52	if (unlikely(ghl < sizeof(*greh)))
	53	goto out;
	54
	55	csum = !!(greh->flags & GRE_CSUM);
4749c09c TH	56	if (csum)
4749c09c TH	57	skb->encap_hdr_csum = 1;
c50cd357 DB	58
	59	/* setup inner skb. */
	60	skb->protocol = greh->protocol;
	61	skb->encapsulation = 0;
	62
b4e3cef7 LR	63	if (unlikely(!pskb_may_pull(skb, ghl)))
	64	goto out;
	65
c50cd357 DB	66	__skb_pull(skb, ghl);
	67	skb_reset_mac_header(skb);
	68	skb_set_network_header(skb, skb_inner_network_offset(skb));
	69	skb->mac_len = skb_inner_network_offset(skb);
	70
	71	/* segment inner packet. */
1e16aa3d	72	enc_features = skb->dev->hw_enc_features & features;
c50cd357	73	segs = skb_mac_gso_segment(skb, enc_features);
5a8dbf03	74	if (IS_ERR_OR_NULL(segs)) {
7a7ffbab	75	skb_gso_error_unwind(skb, protocol, ghl, mac_offset, mac_len);
c50cd357	76	goto out;
7a7ffbab	77	}
c50cd357 DB	78
	79	skb = segs;
	80	tnl_hlen = skb_tnl_header_len(skb);
	81	do {
	82	__skb_push(skb, ghl);
	83	if (csum) {
	84	__be32 *pcsum;
	85
	86	if (skb_has_shared_frag(skb)) {
	87	int err;
	88
	89	err = __skb_linearize(skb);
	90	if (err) {
0c1072ae	91	kfree_skb_list(segs);
c50cd357 DB	92	segs = ERR_PTR(err);
	93	goto out;
	94	}
	95	}
	96
4749c09c TH	97	skb_reset_transport_header(skb);
	98
	99	greh = (struct gre_base_hdr *)
	100	skb_transport_header(skb);
c50cd357 DB	101	pcsum = (__be32 *)(greh + 1);
c50cd357 DB	102	*pcsum = 0;
4749c09c	103	(__sum16 )pcsum = gso_make_checksum(skb, 0);
c50cd357 DB	104	}
	105	__skb_push(skb, tnl_hlen - ghl);
	106
cdbaa0bb AD	107	skb_reset_inner_headers(skb);
	108	skb->encapsulation = 1;
	109
c50cd357 DB	110	skb_reset_mac_header(skb);
	111	skb_set_network_header(skb, mac_len);
	112	skb->mac_len = mac_len;
	113	skb->protocol = protocol;
	114	} while ((skb = skb->next));
	115	out:
	116	return segs;
	117	}
	118
bf5a755f JC	119	static struct sk_buff gre_gro_receive(struct sk_buff head,
	120	struct sk_buff *skb)
	121	{
	122	struct sk_buff **pp = NULL;
	123	struct sk_buff *p;
	124	const struct gre_base_hdr *greh;
	125	unsigned int hlen, grehlen;
	126	unsigned int off;
	127	int flush = 1;
	128	struct packet_offload *ptype;
	129	__be16 type;
	130
	131	off = skb_gro_offset(skb);
	132	hlen = off + sizeof(*greh);
	133	greh = skb_gro_header_fast(skb, off);
	134	if (skb_gro_header_hard(skb, hlen)) {
	135	greh = skb_gro_header_slow(skb, hlen, off);
	136	if (unlikely(!greh))
	137	goto out;
	138	}
	139
	140	/* Only support version 0 and K (key), C (csum) flags. Note that
	141	* although the support for the S (seq#) flag can be added easily
	142	* for GRO, this is problematic for GSO hence can not be enabled
	143	* here because a GRO pkt may end up in the forwarding path, thus
	144	* requiring GSO support to break it up correctly.
	145	*/
	146	if ((greh->flags & ~(GRE_KEY\|GRE_CSUM)) != 0)
	147	goto out;
	148
	149	type = greh->protocol;
	150
	151	rcu_read_lock();
	152	ptype = gro_find_receive_by_type(type);
51456b29	153	if (!ptype)
bf5a755f JC	154	goto out_unlock;
	155
	156	grehlen = GRE_HEADER_SECTION;
	157
	158	if (greh->flags & GRE_KEY)
	159	grehlen += GRE_HEADER_SECTION;
	160
	161	if (greh->flags & GRE_CSUM)
	162	grehlen += GRE_HEADER_SECTION;
	163
	164	hlen = off + grehlen;
	165	if (skb_gro_header_hard(skb, hlen)) {
	166	greh = skb_gro_header_slow(skb, hlen, off);
	167	if (unlikely(!greh))
	168	goto out_unlock;
	169	}
758f75d1 TH	170
758f75d1 TH	171	/* Don't bother verifying checksum if we're going to flush anyway. */
884d338c TH	172	if ((greh->flags & GRE_CSUM) && !NAPI_GRO_CB(skb)->flush) {
884d338c TH	173	if (skb_gro_checksum_simple_validate(skb))
bf5a755f	174	goto out_unlock;
758f75d1	175
884d338c TH	176	skb_gro_checksum_try_convert(skb, IPPROTO_GRE, 0,
	177	null_compute_pseudo);
	178	}
	179
bf5a755f JC	180	flush = 0;
	181
	182	for (p = *head; p; p = p->next) {
	183	const struct gre_base_hdr *greh2;
	184
	185	if (!NAPI_GRO_CB(p)->same_flow)
	186	continue;
	187
	188	/* The following checks are needed to ensure only pkts
	189	* from the same tunnel are considered for aggregation.
	190	* The criteria for "the same tunnel" includes:
	191	* 1) same version (we only support version 0 here)
	192	* 2) same protocol (we only support ETH_P_IP for now)
	193	* 3) same set of flags
	194	* 4) same key if the key field is present.
	195	*/
	196	greh2 = (struct gre_base_hdr *)(p->data + off);
	197
	198	if (greh2->flags != greh->flags \|\|
	199	greh2->protocol != greh->protocol) {
	200	NAPI_GRO_CB(p)->same_flow = 0;
	201	continue;
	202	}
	203	if (greh->flags & GRE_KEY) {
	204	/* compare keys */
	205	if ((__be32 )(greh2+1) != (__be32 )(greh+1)) {
	206	NAPI_GRO_CB(p)->same_flow = 0;
	207	continue;
	208	}
	209	}
	210	}
	211
	212	skb_gro_pull(skb, grehlen);
	213
	214	/* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/
	215	skb_gro_postpull_rcsum(skb, greh, grehlen);
	216
	217	pp = ptype->callbacks.gro_receive(head, skb);
	218
	219	out_unlock:
	220	rcu_read_unlock();
	221	out:
	222	NAPI_GRO_CB(skb)->flush \|= flush;
	223
	224	return pp;
	225	}
	226
d10dbad2	227	static int gre_gro_complete(struct sk_buff *skb, int nhoff)
bf5a755f JC	228	{
	229	struct gre_base_hdr greh = (struct gre_base_hdr )(skb->data + nhoff);
	230	struct packet_offload *ptype;
	231	unsigned int grehlen = sizeof(*greh);
	232	int err = -ENOENT;
	233	__be16 type;
	234
c3caf119 JC	235	skb->encapsulation = 1;
	236	skb_shinfo(skb)->gso_type = SKB_GSO_GRE;
	237
bf5a755f JC	238	type = greh->protocol;
	239	if (greh->flags & GRE_KEY)
	240	grehlen += GRE_HEADER_SECTION;
	241
	242	if (greh->flags & GRE_CSUM)
	243	grehlen += GRE_HEADER_SECTION;
	244
	245	rcu_read_lock();
	246	ptype = gro_find_complete_by_type(type);
00db4124	247	if (ptype)
bf5a755f JC	248	err = ptype->callbacks.gro_complete(skb, nhoff + grehlen);
	249
	250	rcu_read_unlock();
6fb2a756 TH	251
	252	skb_set_inner_mac_header(skb, nhoff + grehlen);
	253
bf5a755f JC	254	return err;
	255	}
	256
c50cd357 DB	257	static const struct net_offload gre_offload = {
c50cd357 DB	258	.callbacks = {
c50cd357	259	.gso_segment = gre_gso_segment,
bf5a755f JC	260	.gro_receive = gre_gro_receive,
bf5a755f JC	261	.gro_complete = gre_gro_complete,
c50cd357 DB	262	},
	263	};
	264
438e38fa	265	static int __init gre_offload_init(void)
c50cd357 DB	266	{
	267	return inet_add_offload(&gre_offload, IPPROTO_GRE);
	268	}
cf172283	269	device_initcall(gre_offload_init);