[linux.git] / include / net / kcm.h

/*
 * Kernel Connection Multiplexor
 *
 * Copyright (c) 2016 Tom Herbert <[email protected]>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2
 * as published by the Free Software Foundation.
 */

#ifndef __NET_KCM_H_
#define __NET_KCM_H_

#include <linux/skbuff.h>
#include <net/sock.h>
#include <uapi/linux/kcm.h>

extern unsigned int kcm_net_id;

#define KCM_STATS_ADD(stat, count) ((stat) += (count))
#define KCM_STATS_INCR(stat) ((stat)++)

struct kcm_psock_stats {
	unsigned long long rx_msgs;
	unsigned long long rx_bytes;
	unsigned long long tx_msgs;
	unsigned long long tx_bytes;
	unsigned int rx_aborts;
	unsigned int rx_mem_fail;
	unsigned int rx_need_more_hdr;
	unsigned int rx_msg_too_big;
	unsigned int rx_msg_timeouts;
	unsigned int rx_bad_hdr_len;
	unsigned long long reserved;
	unsigned long long unreserved;
	unsigned int tx_aborts;
};

struct kcm_mux_stats {
	unsigned long long rx_msgs;
	unsigned long long rx_bytes;
	unsigned long long tx_msgs;
	unsigned long long tx_bytes;
	unsigned int rx_ready_drops;
	unsigned int tx_retries;
	unsigned int psock_attach;
	unsigned int psock_unattach_rsvd;
	unsigned int psock_unattach;
};

struct kcm_stats {
	unsigned long long rx_msgs;
	unsigned long long rx_bytes;
	unsigned long long tx_msgs;
	unsigned long long tx_bytes;
};

struct kcm_tx_msg {
	unsigned int sent;
	unsigned int fragidx;
	unsigned int frag_offset;
	unsigned int msg_flags;
	struct sk_buff *frag_skb;
	struct sk_buff *last_skb;
};

struct kcm_rx_msg {
	int full_len;
	int accum_len;
	int offset;
	int early_eaten;
};

/* Socket structure for KCM client sockets */
struct kcm_sock {
	struct sock sk;
	struct kcm_mux *mux;
	struct list_head kcm_sock_list;
	int index;
	u32 done : 1;
	struct work_struct done_work;

	struct kcm_stats stats;

	/* Transmit */
	struct kcm_psock *tx_psock;
	struct work_struct tx_work;
	struct list_head wait_psock_list;
	struct sk_buff *seq_skb;

	/* Don't use bit fields here, these are set under different locks */
	bool tx_wait;
	bool tx_wait_more;

	/* Receive */
	struct kcm_psock *rx_psock;
	struct list_head wait_rx_list; /* KCMs waiting for receiving */
	bool rx_wait;
	u32 rx_disabled : 1;
};

struct bpf_prog;

/* Structure for an attached lower socket */
struct kcm_psock {
	struct sock *sk;
	struct kcm_mux *mux;
	int index;

	u32 tx_stopped : 1;
	u32 rx_stopped : 1;
	u32 done : 1;
	u32 unattaching : 1;

	void (*save_state_change)(struct sock *sk);
	void (*save_data_ready)(struct sock *sk);
	void (*save_write_space)(struct sock *sk);

	struct list_head psock_list;

	struct kcm_psock_stats stats;

	/* Receive */
	struct sk_buff *rx_skb_head;
	struct sk_buff **rx_skb_nextp;
	struct sk_buff *ready_rx_msg;
	struct list_head psock_ready_list;
	struct work_struct rx_work;
	struct delayed_work rx_delayed_work;
	struct bpf_prog *bpf_prog;
	struct kcm_sock *rx_kcm;
	unsigned long long saved_rx_bytes;
	unsigned long long saved_rx_msgs;
	struct timer_list rx_msg_timer;
	unsigned int rx_need_bytes;

	/* Transmit */
	struct kcm_sock *tx_kcm;
	struct list_head psock_avail_list;
	unsigned long long saved_tx_bytes;
	unsigned long long saved_tx_msgs;
};

/* Per net MUX list */
struct kcm_net {
	struct mutex mutex;
	struct kcm_psock_stats aggregate_psock_stats;
	struct kcm_mux_stats aggregate_mux_stats;
	struct list_head mux_list;
	int count;
};

/* Structure for a MUX */
struct kcm_mux {
	struct list_head kcm_mux_list;
	struct rcu_head rcu;
	struct kcm_net *knet;

	struct list_head kcm_socks;	/* All KCM sockets on MUX */
	int kcm_socks_cnt;		/* Total KCM socket count for MUX */
	struct list_head psocks;	/* List of all psocks on MUX */
	int psocks_cnt;		/* Total attached sockets */

	struct kcm_mux_stats stats;
	struct kcm_psock_stats aggregate_psock_stats;

	/* Receive */
	spinlock_t rx_lock ____cacheline_aligned_in_smp;
	struct list_head kcm_rx_waiters; /* KCMs waiting for receiving */
	struct list_head psocks_ready;	/* List of psocks with a msg ready */
	struct sk_buff_head rx_hold_queue;

	/* Transmit */
	spinlock_t  lock ____cacheline_aligned_in_smp;	/* TX and mux locking */
	struct list_head psocks_avail;	/* List of available psocks */
	struct list_head kcm_tx_waiters; /* KCMs waiting for a TX psock */
};

#ifdef CONFIG_PROC_FS
int kcm_proc_init(void);
void kcm_proc_exit(void);
#else
static inline int kcm_proc_init(void) { return 0; }
static inline void kcm_proc_exit(void) { }
#endif

static inline void aggregate_psock_stats(struct kcm_psock_stats *stats,
					 struct kcm_psock_stats *agg_stats)
{
	/* Save psock statistics in the mux when psock is being unattached. */

#define SAVE_PSOCK_STATS(_stat) (agg_stats->_stat += stats->_stat)
	SAVE_PSOCK_STATS(rx_msgs);
	SAVE_PSOCK_STATS(rx_bytes);
	SAVE_PSOCK_STATS(rx_aborts);
	SAVE_PSOCK_STATS(rx_mem_fail);
	SAVE_PSOCK_STATS(rx_need_more_hdr);
	SAVE_PSOCK_STATS(rx_msg_too_big);
	SAVE_PSOCK_STATS(rx_msg_timeouts);
	SAVE_PSOCK_STATS(rx_bad_hdr_len);
	SAVE_PSOCK_STATS(tx_msgs);
	SAVE_PSOCK_STATS(tx_bytes);
	SAVE_PSOCK_STATS(reserved);
	SAVE_PSOCK_STATS(unreserved);
	SAVE_PSOCK_STATS(tx_aborts);
#undef SAVE_PSOCK_STATS
}

static inline void aggregate_mux_stats(struct kcm_mux_stats *stats,
				       struct kcm_mux_stats *agg_stats)
{
	/* Save psock statistics in the mux when psock is being unattached. */

#define SAVE_MUX_STATS(_stat) (agg_stats->_stat += stats->_stat)
	SAVE_MUX_STATS(rx_msgs);
	SAVE_MUX_STATS(rx_bytes);
	SAVE_MUX_STATS(tx_msgs);
	SAVE_MUX_STATS(tx_bytes);
	SAVE_MUX_STATS(rx_ready_drops);
	SAVE_MUX_STATS(psock_attach);
	SAVE_MUX_STATS(psock_unattach_rsvd);
	SAVE_MUX_STATS(psock_unattach);
#undef SAVE_MUX_STATS
}

#endif /* __NET_KCM_H_ */
Commit	Line	Data
ab7ac4eb TH	1	/*
	2	* Kernel Connection Multiplexor
	3	*
	4	* Copyright (c) 2016 Tom Herbert <[email protected]>
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2
	8	* as published by the Free Software Foundation.
	9	*/
	10
	11	#ifndef __NET_KCM_H_
	12	#define __NET_KCM_H_
	13
	14	#include <linux/skbuff.h>
	15	#include <net/sock.h>
	16	#include <uapi/linux/kcm.h>
	17
	18	extern unsigned int kcm_net_id;
	19
cd6e111b TH	20	#define KCM_STATS_ADD(stat, count) ((stat) += (count))
	21	#define KCM_STATS_INCR(stat) ((stat)++)
	22
	23	struct kcm_psock_stats {
	24	unsigned long long rx_msgs;
	25	unsigned long long rx_bytes;
	26	unsigned long long tx_msgs;
	27	unsigned long long tx_bytes;
	28	unsigned int rx_aborts;
	29	unsigned int rx_mem_fail;
	30	unsigned int rx_need_more_hdr;
7ced95ef	31	unsigned int rx_msg_too_big;
29152a34	32	unsigned int rx_msg_timeouts;
cd6e111b TH	33	unsigned int rx_bad_hdr_len;
	34	unsigned long long reserved;
	35	unsigned long long unreserved;
	36	unsigned int tx_aborts;
	37	};
	38
	39	struct kcm_mux_stats {
	40	unsigned long long rx_msgs;
	41	unsigned long long rx_bytes;
	42	unsigned long long tx_msgs;
	43	unsigned long long tx_bytes;
	44	unsigned int rx_ready_drops;
	45	unsigned int tx_retries;
	46	unsigned int psock_attach;
	47	unsigned int psock_unattach_rsvd;
	48	unsigned int psock_unattach;
	49	};
	50
	51	struct kcm_stats {
	52	unsigned long long rx_msgs;
	53	unsigned long long rx_bytes;
	54	unsigned long long tx_msgs;
	55	unsigned long long tx_bytes;
	56	};
	57
ab7ac4eb TH	58	struct kcm_tx_msg {
	59	unsigned int sent;
	60	unsigned int fragidx;
	61	unsigned int frag_offset;
	62	unsigned int msg_flags;
	63	struct sk_buff *frag_skb;
	64	struct sk_buff *last_skb;
	65	};
	66
	67	struct kcm_rx_msg {
	68	int full_len;
	69	int accum_len;
	70	int offset;
7ced95ef	71	int early_eaten;
ab7ac4eb TH	72	};
	73
	74	/* Socket structure for KCM client sockets */
	75	struct kcm_sock {
	76	struct sock sk;
	77	struct kcm_mux *mux;
	78	struct list_head kcm_sock_list;
	79	int index;
	80	u32 done : 1;
	81	struct work_struct done_work;
	82
cd6e111b TH	83	struct kcm_stats stats;
cd6e111b TH	84
ab7ac4eb TH	85	/* Transmit */
	86	struct kcm_psock *tx_psock;
	87	struct work_struct tx_work;
	88	struct list_head wait_psock_list;
	89	struct sk_buff *seq_skb;
	90
	91	/* Don't use bit fields here, these are set under different locks */
	92	bool tx_wait;
	93	bool tx_wait_more;
	94
	95	/* Receive */
	96	struct kcm_psock *rx_psock;
	97	struct list_head wait_rx_list; /* KCMs waiting for receiving */
	98	bool rx_wait;
	99	u32 rx_disabled : 1;
	100	};
	101
	102	struct bpf_prog;
	103
	104	/* Structure for an attached lower socket */
	105	struct kcm_psock {
	106	struct sock *sk;
	107	struct kcm_mux *mux;
	108	int index;
	109
	110	u32 tx_stopped : 1;
	111	u32 rx_stopped : 1;
	112	u32 done : 1;
	113	u32 unattaching : 1;
	114
	115	void (save_state_change)(struct sock sk);
	116	void (save_data_ready)(struct sock sk);
	117	void (save_write_space)(struct sock sk);
	118
	119	struct list_head psock_list;
	120
cd6e111b TH	121	struct kcm_psock_stats stats;
cd6e111b TH	122
ab7ac4eb TH	123	/* Receive */
	124	struct sk_buff *rx_skb_head;
	125	struct sk_buff **rx_skb_nextp;
	126	struct sk_buff *ready_rx_msg;
	127	struct list_head psock_ready_list;
	128	struct work_struct rx_work;
	129	struct delayed_work rx_delayed_work;
	130	struct bpf_prog *bpf_prog;
	131	struct kcm_sock *rx_kcm;
cd6e111b TH	132	unsigned long long saved_rx_bytes;
cd6e111b TH	133	unsigned long long saved_rx_msgs;
29152a34	134	struct timer_list rx_msg_timer;
7ced95ef	135	unsigned int rx_need_bytes;
ab7ac4eb TH	136
	137	/* Transmit */
	138	struct kcm_sock *tx_kcm;
	139	struct list_head psock_avail_list;
cd6e111b TH	140	unsigned long long saved_tx_bytes;
cd6e111b TH	141	unsigned long long saved_tx_msgs;
ab7ac4eb TH	142	};
	143
	144	/* Per net MUX list */
	145	struct kcm_net {
	146	struct mutex mutex;
cd6e111b TH	147	struct kcm_psock_stats aggregate_psock_stats;
cd6e111b TH	148	struct kcm_mux_stats aggregate_mux_stats;
ab7ac4eb TH	149	struct list_head mux_list;
	150	int count;
	151	};
	152
	153	/* Structure for a MUX */
	154	struct kcm_mux {
	155	struct list_head kcm_mux_list;
	156	struct rcu_head rcu;
	157	struct kcm_net *knet;
	158
	159	struct list_head kcm_socks; /* All KCM sockets on MUX */
	160	int kcm_socks_cnt; /* Total KCM socket count for MUX */
	161	struct list_head psocks; /* List of all psocks on MUX */
	162	int psocks_cnt; /* Total attached sockets */
	163
cd6e111b TH	164	struct kcm_mux_stats stats;
	165	struct kcm_psock_stats aggregate_psock_stats;
	166
ab7ac4eb TH	167	/* Receive */
	168	spinlock_t rx_lock ____cacheline_aligned_in_smp;
	169	struct list_head kcm_rx_waiters; /* KCMs waiting for receiving */
	170	struct list_head psocks_ready; /* List of psocks with a msg ready */
	171	struct sk_buff_head rx_hold_queue;
	172
	173	/* Transmit */
	174	spinlock_t lock ____cacheline_aligned_in_smp; /* TX and mux locking */
	175	struct list_head psocks_avail; /* List of available psocks */
	176	struct list_head kcm_tx_waiters; /* KCMs waiting for a TX psock */
	177	};
	178
cd6e111b TH	179	#ifdef CONFIG_PROC_FS
	180	int kcm_proc_init(void);
	181	void kcm_proc_exit(void);
	182	#else
f720d0ca AB	183	static inline int kcm_proc_init(void) { return 0; }
f720d0ca AB	184	static inline void kcm_proc_exit(void) { }
cd6e111b TH	185	#endif
	186
	187	static inline void aggregate_psock_stats(struct kcm_psock_stats *stats,
	188	struct kcm_psock_stats *agg_stats)
	189	{
	190	/* Save psock statistics in the mux when psock is being unattached. */
	191
	192	#define SAVE_PSOCK_STATS(_stat) (agg_stats->_stat += stats->_stat)
	193	SAVE_PSOCK_STATS(rx_msgs);
	194	SAVE_PSOCK_STATS(rx_bytes);
	195	SAVE_PSOCK_STATS(rx_aborts);
	196	SAVE_PSOCK_STATS(rx_mem_fail);
	197	SAVE_PSOCK_STATS(rx_need_more_hdr);
7ced95ef	198	SAVE_PSOCK_STATS(rx_msg_too_big);
29152a34	199	SAVE_PSOCK_STATS(rx_msg_timeouts);
cd6e111b TH	200	SAVE_PSOCK_STATS(rx_bad_hdr_len);
	201	SAVE_PSOCK_STATS(tx_msgs);
	202	SAVE_PSOCK_STATS(tx_bytes);
	203	SAVE_PSOCK_STATS(reserved);
	204	SAVE_PSOCK_STATS(unreserved);
	205	SAVE_PSOCK_STATS(tx_aborts);
	206	#undef SAVE_PSOCK_STATS
	207	}
	208
	209	static inline void aggregate_mux_stats(struct kcm_mux_stats *stats,
	210	struct kcm_mux_stats *agg_stats)
	211	{
	212	/* Save psock statistics in the mux when psock is being unattached. */
	213
	214	#define SAVE_MUX_STATS(_stat) (agg_stats->_stat += stats->_stat)
	215	SAVE_MUX_STATS(rx_msgs);
	216	SAVE_MUX_STATS(rx_bytes);
	217	SAVE_MUX_STATS(tx_msgs);
	218	SAVE_MUX_STATS(tx_bytes);
	219	SAVE_MUX_STATS(rx_ready_drops);
	220	SAVE_MUX_STATS(psock_attach);
	221	SAVE_MUX_STATS(psock_unattach_rsvd);
	222	SAVE_MUX_STATS(psock_unattach);
	223	#undef SAVE_MUX_STATS
	224	}
	225
ab7ac4eb	226	#endif /* __NET_KCM_H_ */