[linux.git] / include / net / kcm.h

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Kernel Connection Multiplexor
 *
 * Copyright (c) 2016 Tom Herbert <[email protected]>
 */

#ifndef __NET_KCM_H_
#define __NET_KCM_H_

#include <linux/skbuff.h>
#include <net/sock.h>
#include <net/strparser.h>
#include <uapi/linux/kcm.h>

extern unsigned int kcm_net_id;

#define KCM_STATS_ADD(stat, count) ((stat) += (count))
#define KCM_STATS_INCR(stat) ((stat)++)

struct kcm_psock_stats {
	unsigned long long tx_msgs;
	unsigned long long tx_bytes;
	unsigned long long reserved;
	unsigned long long unreserved;
	unsigned int tx_aborts;
};

struct kcm_mux_stats {
	unsigned long long rx_msgs;
	unsigned long long rx_bytes;
	unsigned long long tx_msgs;
	unsigned long long tx_bytes;
	unsigned int rx_ready_drops;
	unsigned int tx_retries;
	unsigned int psock_attach;
	unsigned int psock_unattach_rsvd;
	unsigned int psock_unattach;
};

struct kcm_stats {
	unsigned long long rx_msgs;
	unsigned long long rx_bytes;
	unsigned long long tx_msgs;
	unsigned long long tx_bytes;
};

struct kcm_tx_msg {
	unsigned int sent;
	unsigned int frag_offset;
	unsigned int msg_flags;
	bool started_tx;
	struct sk_buff *frag_skb;
	struct sk_buff *last_skb;
};

/* Socket structure for KCM client sockets */
struct kcm_sock {
	struct sock sk;
	struct kcm_mux *mux;
	struct list_head kcm_sock_list;
	int index;
	u32 done : 1;
	struct work_struct done_work;

	struct kcm_stats stats;

	/* Transmit */
	struct kcm_psock *tx_psock;
	struct work_struct tx_work;
	struct list_head wait_psock_list;
	struct sk_buff *seq_skb;
	struct mutex tx_mutex;
	u32 tx_stopped : 1;

	/* Don't use bit fields here, these are set under different locks */
	bool tx_wait;
	bool tx_wait_more;

	/* Receive */
	struct kcm_psock *rx_psock;
	struct list_head wait_rx_list; /* KCMs waiting for receiving */
	bool rx_wait;
	u32 rx_disabled : 1;
};

struct bpf_prog;

/* Structure for an attached lower socket */
struct kcm_psock {
	struct sock *sk;
	struct strparser strp;
	struct kcm_mux *mux;
	int index;

	u32 tx_stopped : 1;
	u32 done : 1;
	u32 unattaching : 1;

	void (*save_state_change)(struct sock *sk);
	void (*save_data_ready)(struct sock *sk);
	void (*save_write_space)(struct sock *sk);

	struct list_head psock_list;

	struct kcm_psock_stats stats;

	/* Receive */
	struct list_head psock_ready_list;
	struct bpf_prog *bpf_prog;
	struct kcm_sock *rx_kcm;
	unsigned long long saved_rx_bytes;
	unsigned long long saved_rx_msgs;
	struct sk_buff *ready_rx_msg;

	/* Transmit */
	struct kcm_sock *tx_kcm;
	struct list_head psock_avail_list;
	unsigned long long saved_tx_bytes;
	unsigned long long saved_tx_msgs;
};

/* Per net MUX list */
struct kcm_net {
	struct mutex mutex;
	struct kcm_psock_stats aggregate_psock_stats;
	struct kcm_mux_stats aggregate_mux_stats;
	struct strp_aggr_stats aggregate_strp_stats;
	struct list_head mux_list;
	int count;
};

/* Structure for a MUX */
struct kcm_mux {
	struct list_head kcm_mux_list;
	struct rcu_head rcu;
	struct kcm_net *knet;

	struct list_head kcm_socks;	/* All KCM sockets on MUX */
	int kcm_socks_cnt;		/* Total KCM socket count for MUX */
	struct list_head psocks;	/* List of all psocks on MUX */
	int psocks_cnt;		/* Total attached sockets */

	struct kcm_mux_stats stats;
	struct kcm_psock_stats aggregate_psock_stats;
	struct strp_aggr_stats aggregate_strp_stats;

	/* Receive */
	spinlock_t rx_lock ____cacheline_aligned_in_smp;
	struct list_head kcm_rx_waiters; /* KCMs waiting for receiving */
	struct list_head psocks_ready;	/* List of psocks with a msg ready */
	struct sk_buff_head rx_hold_queue;

	/* Transmit */
	spinlock_t  lock ____cacheline_aligned_in_smp;	/* TX and mux locking */
	struct list_head psocks_avail;	/* List of available psocks */
	struct list_head kcm_tx_waiters; /* KCMs waiting for a TX psock */
};

#ifdef CONFIG_PROC_FS
int kcm_proc_init(void);
void kcm_proc_exit(void);
#else
static inline int kcm_proc_init(void) { return 0; }
static inline void kcm_proc_exit(void) { }
#endif

static inline void aggregate_psock_stats(struct kcm_psock_stats *stats,
					 struct kcm_psock_stats *agg_stats)
{
	/* Save psock statistics in the mux when psock is being unattached. */

#define SAVE_PSOCK_STATS(_stat) (agg_stats->_stat += stats->_stat)
	SAVE_PSOCK_STATS(tx_msgs);
	SAVE_PSOCK_STATS(tx_bytes);
	SAVE_PSOCK_STATS(reserved);
	SAVE_PSOCK_STATS(unreserved);
	SAVE_PSOCK_STATS(tx_aborts);
#undef SAVE_PSOCK_STATS
}

static inline void aggregate_mux_stats(struct kcm_mux_stats *stats,
				       struct kcm_mux_stats *agg_stats)
{
	/* Save psock statistics in the mux when psock is being unattached. */

#define SAVE_MUX_STATS(_stat) (agg_stats->_stat += stats->_stat)
	SAVE_MUX_STATS(rx_msgs);
	SAVE_MUX_STATS(rx_bytes);
	SAVE_MUX_STATS(tx_msgs);
	SAVE_MUX_STATS(tx_bytes);
	SAVE_MUX_STATS(rx_ready_drops);
	SAVE_MUX_STATS(psock_attach);
	SAVE_MUX_STATS(psock_unattach_rsvd);
	SAVE_MUX_STATS(psock_unattach);
#undef SAVE_MUX_STATS
}

#endif /* __NET_KCM_H_ */
Commit	Line	Data
d2912cb1	1	/* SPDX-License-Identifier: GPL-2.0-only */
ab7ac4eb TH	2	/*
	3	* Kernel Connection Multiplexor
	4	*
	5	* Copyright (c) 2016 Tom Herbert <[email protected]>
ab7ac4eb TH	6	*/
	7
	8	#ifndef __NET_KCM_H_
	9	#define __NET_KCM_H_
	10
	11	#include <linux/skbuff.h>
	12	#include <net/sock.h>
9b73896a	13	#include <net/strparser.h>
ab7ac4eb TH	14	#include <uapi/linux/kcm.h>
	15
	16	extern unsigned int kcm_net_id;
	17
cd6e111b TH	18	#define KCM_STATS_ADD(stat, count) ((stat) += (count))
	19	#define KCM_STATS_INCR(stat) ((stat)++)
	20
	21	struct kcm_psock_stats {
cd6e111b TH	22	unsigned long long tx_msgs;
cd6e111b TH	23	unsigned long long tx_bytes;
cd6e111b TH	24	unsigned long long reserved;
	25	unsigned long long unreserved;
	26	unsigned int tx_aborts;
	27	};
	28
	29	struct kcm_mux_stats {
	30	unsigned long long rx_msgs;
	31	unsigned long long rx_bytes;
	32	unsigned long long tx_msgs;
	33	unsigned long long tx_bytes;
	34	unsigned int rx_ready_drops;
	35	unsigned int tx_retries;
	36	unsigned int psock_attach;
	37	unsigned int psock_unattach_rsvd;
	38	unsigned int psock_unattach;
	39	};
	40
	41	struct kcm_stats {
	42	unsigned long long rx_msgs;
	43	unsigned long long rx_bytes;
	44	unsigned long long tx_msgs;
	45	unsigned long long tx_bytes;
	46	};
	47
ab7ac4eb TH	48	struct kcm_tx_msg {
ab7ac4eb TH	49	unsigned int sent;
ab7ac4eb TH	50	unsigned int frag_offset;
ab7ac4eb TH	51	unsigned int msg_flags;
c31a25e1	52	bool started_tx;
ab7ac4eb TH	53	struct sk_buff *frag_skb;
	54	struct sk_buff *last_skb;
	55	};
	56
ab7ac4eb TH	57	/* Socket structure for KCM client sockets */
	58	struct kcm_sock {
	59	struct sock sk;
	60	struct kcm_mux *mux;
	61	struct list_head kcm_sock_list;
	62	int index;
	63	u32 done : 1;
	64	struct work_struct done_work;
	65
cd6e111b TH	66	struct kcm_stats stats;
cd6e111b TH	67
ab7ac4eb TH	68	/* Transmit */
	69	struct kcm_psock *tx_psock;
	70	struct work_struct tx_work;
	71	struct list_head wait_psock_list;
	72	struct sk_buff *seq_skb;
807067bf	73	struct mutex tx_mutex;
9b73896a	74	u32 tx_stopped : 1;
ab7ac4eb TH	75
	76	/* Don't use bit fields here, these are set under different locks */
	77	bool tx_wait;
	78	bool tx_wait_more;
	79
	80	/* Receive */
	81	struct kcm_psock *rx_psock;
	82	struct list_head wait_rx_list; /* KCMs waiting for receiving */
	83	bool rx_wait;
	84	u32 rx_disabled : 1;
	85	};
	86
	87	struct bpf_prog;
	88
	89	/* Structure for an attached lower socket */
	90	struct kcm_psock {
	91	struct sock *sk;
9b73896a	92	struct strparser strp;
ab7ac4eb TH	93	struct kcm_mux *mux;
	94	int index;
	95
	96	u32 tx_stopped : 1;
ab7ac4eb TH	97	u32 done : 1;
	98	u32 unattaching : 1;
	99
	100	void (save_state_change)(struct sock sk);
	101	void (save_data_ready)(struct sock sk);
	102	void (save_write_space)(struct sock sk);
	103
	104	struct list_head psock_list;
	105
cd6e111b TH	106	struct kcm_psock_stats stats;
cd6e111b TH	107
ab7ac4eb	108	/* Receive */
ab7ac4eb	109	struct list_head psock_ready_list;
ab7ac4eb TH	110	struct bpf_prog *bpf_prog;
ab7ac4eb TH	111	struct kcm_sock *rx_kcm;
cd6e111b TH	112	unsigned long long saved_rx_bytes;
cd6e111b TH	113	unsigned long long saved_rx_msgs;
9b73896a	114	struct sk_buff *ready_rx_msg;
ab7ac4eb TH	115
	116	/* Transmit */
	117	struct kcm_sock *tx_kcm;
	118	struct list_head psock_avail_list;
cd6e111b TH	119	unsigned long long saved_tx_bytes;
cd6e111b TH	120	unsigned long long saved_tx_msgs;
ab7ac4eb TH	121	};
	122
	123	/* Per net MUX list */
	124	struct kcm_net {
	125	struct mutex mutex;
cd6e111b TH	126	struct kcm_psock_stats aggregate_psock_stats;
cd6e111b TH	127	struct kcm_mux_stats aggregate_mux_stats;
9b73896a	128	struct strp_aggr_stats aggregate_strp_stats;
ab7ac4eb TH	129	struct list_head mux_list;
	130	int count;
	131	};
	132
	133	/* Structure for a MUX */
	134	struct kcm_mux {
	135	struct list_head kcm_mux_list;
	136	struct rcu_head rcu;
	137	struct kcm_net *knet;
	138
	139	struct list_head kcm_socks; /* All KCM sockets on MUX */
	140	int kcm_socks_cnt; /* Total KCM socket count for MUX */
	141	struct list_head psocks; /* List of all psocks on MUX */
	142	int psocks_cnt; /* Total attached sockets */
	143
cd6e111b TH	144	struct kcm_mux_stats stats;
cd6e111b TH	145	struct kcm_psock_stats aggregate_psock_stats;
9b73896a	146	struct strp_aggr_stats aggregate_strp_stats;
cd6e111b	147
ab7ac4eb TH	148	/* Receive */
	149	spinlock_t rx_lock ____cacheline_aligned_in_smp;
	150	struct list_head kcm_rx_waiters; /* KCMs waiting for receiving */
	151	struct list_head psocks_ready; /* List of psocks with a msg ready */
	152	struct sk_buff_head rx_hold_queue;
	153
	154	/* Transmit */
	155	spinlock_t lock ____cacheline_aligned_in_smp; /* TX and mux locking */
	156	struct list_head psocks_avail; /* List of available psocks */
	157	struct list_head kcm_tx_waiters; /* KCMs waiting for a TX psock */
	158	};
	159
cd6e111b TH	160	#ifdef CONFIG_PROC_FS
	161	int kcm_proc_init(void);
	162	void kcm_proc_exit(void);
	163	#else
f720d0ca AB	164	static inline int kcm_proc_init(void) { return 0; }
f720d0ca AB	165	static inline void kcm_proc_exit(void) { }
cd6e111b TH	166	#endif
	167
	168	static inline void aggregate_psock_stats(struct kcm_psock_stats *stats,
	169	struct kcm_psock_stats *agg_stats)
	170	{
	171	/* Save psock statistics in the mux when psock is being unattached. */
	172
	173	#define SAVE_PSOCK_STATS(_stat) (agg_stats->_stat += stats->_stat)
cd6e111b TH	174	SAVE_PSOCK_STATS(tx_msgs);
	175	SAVE_PSOCK_STATS(tx_bytes);
	176	SAVE_PSOCK_STATS(reserved);
	177	SAVE_PSOCK_STATS(unreserved);
	178	SAVE_PSOCK_STATS(tx_aborts);
	179	#undef SAVE_PSOCK_STATS
	180	}
	181
	182	static inline void aggregate_mux_stats(struct kcm_mux_stats *stats,
	183	struct kcm_mux_stats *agg_stats)
	184	{
	185	/* Save psock statistics in the mux when psock is being unattached. */
	186
	187	#define SAVE_MUX_STATS(_stat) (agg_stats->_stat += stats->_stat)
	188	SAVE_MUX_STATS(rx_msgs);
	189	SAVE_MUX_STATS(rx_bytes);
	190	SAVE_MUX_STATS(tx_msgs);
	191	SAVE_MUX_STATS(tx_bytes);
	192	SAVE_MUX_STATS(rx_ready_drops);
	193	SAVE_MUX_STATS(psock_attach);
	194	SAVE_MUX_STATS(psock_unattach_rsvd);
	195	SAVE_MUX_STATS(psock_unattach);
	196	#undef SAVE_MUX_STATS
	197	}
	198
ab7ac4eb	199	#endif /* __NET_KCM_H_ */