[linux.git] / fs / cifs / smbdirect.h

/*
 *   Copyright (C) 2017, Microsoft Corporation.
 *
 *   Author(s): Long Li <[email protected]>
 *
 *   This program is free software;  you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation; either version 2 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY;  without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See
 *   the GNU General Public License for more details.
 */
#ifndef _SMBDIRECT_H
#define _SMBDIRECT_H

#ifdef CONFIG_CIFS_SMB_DIRECT
#define cifs_rdma_enabled(server)	((server)->rdma)

#include "cifsglob.h"
#include <rdma/ib_verbs.h>
#include <rdma/rdma_cm.h>
#include <linux/mempool.h>

extern int rdma_readwrite_threshold;
extern int smbd_max_frmr_depth;
extern int smbd_keep_alive_interval;
extern int smbd_max_receive_size;
extern int smbd_max_fragmented_recv_size;
extern int smbd_max_send_size;
extern int smbd_send_credit_target;
extern int smbd_receive_credit_max;

enum keep_alive_status {
	KEEP_ALIVE_NONE,
	KEEP_ALIVE_PENDING,
	KEEP_ALIVE_SENT,
};

enum smbd_connection_status {
	SMBD_CREATED,
	SMBD_CONNECTING,
	SMBD_CONNECTED,
	SMBD_NEGOTIATE_FAILED,
	SMBD_DISCONNECTING,
	SMBD_DISCONNECTED,
	SMBD_DESTROYED
};

/*
 * The context for the SMBDirect transport
 * Everything related to the transport is here. It has several logical parts
 * 1. RDMA related structures
 * 2. SMBDirect connection parameters
 * 3. Memory registrations
 * 4. Receive and reassembly queues for data receive path
 * 5. mempools for allocating packets
 */
struct smbd_connection {
	enum smbd_connection_status transport_status;

	/* RDMA related */
	struct rdma_cm_id *id;
	struct ib_qp_init_attr qp_attr;
	struct ib_pd *pd;
	struct ib_cq *send_cq, *recv_cq;
	struct ib_device_attr dev_attr;
	int ri_rc;
	struct completion ri_done;
	wait_queue_head_t conn_wait;
	wait_queue_head_t wait_destroy;

	struct completion negotiate_completion;
	bool negotiate_done;

	struct work_struct destroy_work;
	struct work_struct disconnect_work;
	struct work_struct recv_done_work;
	struct work_struct post_send_credits_work;

	spinlock_t lock_new_credits_offered;
	int new_credits_offered;

	/* Connection parameters defined in [MS-SMBD] 3.1.1.1 */
	int receive_credit_max;
	int send_credit_target;
	int max_send_size;
	int max_fragmented_recv_size;
	int max_fragmented_send_size;
	int max_receive_size;
	int keep_alive_interval;
	int max_readwrite_size;
	enum keep_alive_status keep_alive_requested;
	int protocol;
	atomic_t send_credits;
	atomic_t receive_credits;
	int receive_credit_target;
	int fragment_reassembly_remaining;

	/* Memory registrations */
	/* Maximum number of RDMA read/write outstanding on this connection */
	int responder_resources;
	/* Maximum number of SGEs in a RDMA write/read */
	int max_frmr_depth;
	/*
	 * If payload is less than or equal to the threshold,
	 * use RDMA send/recv to send upper layer I/O.
	 * If payload is more than the threshold,
	 * use RDMA read/write through memory registration for I/O.
	 */
	int rdma_readwrite_threshold;
	enum ib_mr_type mr_type;
	struct list_head mr_list;
	spinlock_t mr_list_lock;
	/* The number of available MRs ready for memory registration */
	atomic_t mr_ready_count;
	atomic_t mr_used_count;
	wait_queue_head_t wait_mr;
	struct work_struct mr_recovery_work;
	/* Used by transport to wait until all MRs are returned */
	wait_queue_head_t wait_for_mr_cleanup;

	/* Activity accoutning */
	/* Pending reqeusts issued from upper layer */
	int smbd_send_pending;
	wait_queue_head_t wait_smbd_send_pending;

	int smbd_recv_pending;
	wait_queue_head_t wait_smbd_recv_pending;

	atomic_t send_pending;
	wait_queue_head_t wait_send_pending;
	atomic_t send_payload_pending;
	wait_queue_head_t wait_send_payload_pending;

	/* Receive queue */
	struct list_head receive_queue;
	int count_receive_queue;
	spinlock_t receive_queue_lock;

	struct list_head empty_packet_queue;
	int count_empty_packet_queue;
	spinlock_t empty_packet_queue_lock;

	wait_queue_head_t wait_receive_queues;

	/* Reassembly queue */
	struct list_head reassembly_queue;
	spinlock_t reassembly_queue_lock;
	wait_queue_head_t wait_reassembly_queue;

	/* total data length of reassembly queue */
	int reassembly_data_length;
	int reassembly_queue_length;
	/* the offset to first buffer in reassembly queue */
	int first_entry_offset;

	bool send_immediate;

	wait_queue_head_t wait_send_queue;

	/*
	 * Indicate if we have received a full packet on the connection
	 * This is used to identify the first SMBD packet of a assembled
	 * payload (SMB packet) in reassembly queue so we can return a
	 * RFC1002 length to upper layer to indicate the length of the SMB
	 * packet received
	 */
	bool full_packet_received;

	struct workqueue_struct *workqueue;
	struct delayed_work idle_timer_work;
	struct delayed_work send_immediate_work;

	/* Memory pool for preallocating buffers */
	/* request pool for RDMA send */
	struct kmem_cache *request_cache;
	mempool_t *request_mempool;

	/* response pool for RDMA receive */
	struct kmem_cache *response_cache;
	mempool_t *response_mempool;

	/* for debug purposes */
	unsigned int count_get_receive_buffer;
	unsigned int count_put_receive_buffer;
	unsigned int count_reassembly_queue;
	unsigned int count_enqueue_reassembly_queue;
	unsigned int count_dequeue_reassembly_queue;
	unsigned int count_send_empty;
};

enum smbd_message_type {
	SMBD_NEGOTIATE_RESP,
	SMBD_TRANSFER_DATA,
};

#define SMB_DIRECT_RESPONSE_REQUESTED 0x0001

/* SMBD negotiation request packet [MS-SMBD] 2.2.1 */
struct smbd_negotiate_req {
	__le16 min_version;
	__le16 max_version;
	__le16 reserved;
	__le16 credits_requested;
	__le32 preferred_send_size;
	__le32 max_receive_size;
	__le32 max_fragmented_size;
} __packed;

/* SMBD negotiation response packet [MS-SMBD] 2.2.2 */
struct smbd_negotiate_resp {
	__le16 min_version;
	__le16 max_version;
	__le16 negotiated_version;
	__le16 reserved;
	__le16 credits_requested;
	__le16 credits_granted;
	__le32 status;
	__le32 max_readwrite_size;
	__le32 preferred_send_size;
	__le32 max_receive_size;
	__le32 max_fragmented_size;
} __packed;

/* SMBD data transfer packet with payload [MS-SMBD] 2.2.3 */
struct smbd_data_transfer {
	__le16 credits_requested;
	__le16 credits_granted;
	__le16 flags;
	__le16 reserved;
	__le32 remaining_data_length;
	__le32 data_offset;
	__le32 data_length;
	__le32 padding;
	__u8 buffer[];
} __packed;

/* The packet fields for a registered RDMA buffer */
struct smbd_buffer_descriptor_v1 {
	__le64 offset;
	__le32 token;
	__le32 length;
} __packed;

/* Default maximum number of SGEs in a RDMA send/recv */
#define SMBDIRECT_MAX_SGE	16
/* The context for a SMBD request */
struct smbd_request {
	struct smbd_connection *info;
	struct ib_cqe cqe;

	/* true if this request carries upper layer payload */
	bool has_payload;

	/* the SGE entries for this packet */
	struct ib_sge sge[SMBDIRECT_MAX_SGE];
	int num_sge;

	/* SMBD packet header follows this structure */
	u8 packet[];
};

/* The context for a SMBD response */
struct smbd_response {
	struct smbd_connection *info;
	struct ib_cqe cqe;
	struct ib_sge sge;

	enum smbd_message_type type;

	/* Link to receive queue or reassembly queue */
	struct list_head list;

	/* Indicate if this is the 1st packet of a payload */
	bool first_segment;

	/* SMBD packet header and payload follows this structure */
	u8 packet[];
};

/* Create a SMBDirect session */
struct smbd_connection *smbd_get_connection(
	struct TCP_Server_Info *server, struct sockaddr *dstaddr);

/* Reconnect SMBDirect session */
int smbd_reconnect(struct TCP_Server_Info *server);
/* Destroy SMBDirect session */
void smbd_destroy(struct smbd_connection *info);

/* Interface for carrying upper layer I/O through send/recv */
int smbd_recv(struct smbd_connection *info, struct msghdr *msg);
int smbd_send(struct TCP_Server_Info *server, struct smb_rqst *rqst);

enum mr_state {
	MR_READY,
	MR_REGISTERED,
	MR_INVALIDATED,
	MR_ERROR
};

struct smbd_mr {
	struct smbd_connection	*conn;
	struct list_head	list;
	enum mr_state		state;
	struct ib_mr		*mr;
	struct scatterlist	*sgl;
	int			sgl_count;
	enum dma_data_direction	dir;
	union {
		struct ib_reg_wr	wr;
		struct ib_send_wr	inv_wr;
	};
	struct ib_cqe		cqe;
	bool			need_invalidate;
	struct completion	invalidate_done;
};

/* Interfaces to register and deregister MR for RDMA read/write */
struct smbd_mr *smbd_register_mr(
	struct smbd_connection *info, struct page *pages[], int num_pages,
	int offset, int tailsz, bool writing, bool need_invalidate);
int smbd_deregister_mr(struct smbd_mr *mr);

#else
#define cifs_rdma_enabled(server)	0
struct smbd_connection {};
static inline void *smbd_get_connection(
	struct TCP_Server_Info *server, struct sockaddr *dstaddr) {return NULL;}
static inline int smbd_reconnect(struct TCP_Server_Info *server) {return -1; }
static inline void smbd_destroy(struct smbd_connection *info) {}
static inline int smbd_recv(struct smbd_connection *info, struct msghdr *msg) {return -1; }
static inline int smbd_send(struct TCP_Server_Info *server, struct smb_rqst *rqst) {return -1; }
#endif

#endif
Commit	Line	Data
03bee01d LL	1	/*
	2	* Copyright (C) 2017, Microsoft Corporation.
	3	*
	4	* Author(s): Long Li <[email protected]>
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License as published by
	8	* the Free Software Foundation; either version 2 of the License, or
	9	* (at your option) any later version.
	10	*
	11	* This program is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See
	14	* the GNU General Public License for more details.
	15	*/
	16	#ifndef _SMBDIRECT_H
	17	#define _SMBDIRECT_H
	18
f198186a LL	19	#ifdef CONFIG_CIFS_SMB_DIRECT
	20	#define cifs_rdma_enabled(server) ((server)->rdma)
	21
	22	#include "cifsglob.h"
	23	#include <rdma/ib_verbs.h>
	24	#include <rdma/rdma_cm.h>
	25	#include <linux/mempool.h>
	26
6b314714 SF	27	extern int rdma_readwrite_threshold;
	28	extern int smbd_max_frmr_depth;
	29	extern int smbd_keep_alive_interval;
	30	extern int smbd_max_receive_size;
	31	extern int smbd_max_fragmented_recv_size;
	32	extern int smbd_max_send_size;
	33	extern int smbd_send_credit_target;
	34	extern int smbd_receive_credit_max;
	35
f198186a LL	36	enum keep_alive_status {
	37	KEEP_ALIVE_NONE,
	38	KEEP_ALIVE_PENDING,
	39	KEEP_ALIVE_SENT,
	40	};
	41
	42	enum smbd_connection_status {
	43	SMBD_CREATED,
	44	SMBD_CONNECTING,
	45	SMBD_CONNECTED,
	46	SMBD_NEGOTIATE_FAILED,
	47	SMBD_DISCONNECTING,
	48	SMBD_DISCONNECTED,
	49	SMBD_DESTROYED
	50	};
	51
	52	/*
	53	* The context for the SMBDirect transport
	54	* Everything related to the transport is here. It has several logical parts
	55	* 1. RDMA related structures
	56	* 2. SMBDirect connection parameters
	57	* 3. Memory registrations
	58	* 4. Receive and reassembly queues for data receive path
	59	* 5. mempools for allocating packets
	60	*/
	61	struct smbd_connection {
	62	enum smbd_connection_status transport_status;
	63
	64	/* RDMA related */
	65	struct rdma_cm_id *id;
	66	struct ib_qp_init_attr qp_attr;
	67	struct ib_pd *pd;
	68	struct ib_cq send_cq, recv_cq;
	69	struct ib_device_attr dev_attr;
	70	int ri_rc;
	71	struct completion ri_done;
	72	wait_queue_head_t conn_wait;
	73	wait_queue_head_t wait_destroy;
	74
	75	struct completion negotiate_completion;
	76	bool negotiate_done;
	77
	78	struct work_struct destroy_work;
	79	struct work_struct disconnect_work;
	80	struct work_struct recv_done_work;
	81	struct work_struct post_send_credits_work;
	82
	83	spinlock_t lock_new_credits_offered;
	84	int new_credits_offered;
	85
	86	/* Connection parameters defined in [MS-SMBD] 3.1.1.1 */
	87	int receive_credit_max;
	88	int send_credit_target;
	89	int max_send_size;
	90	int max_fragmented_recv_size;
	91	int max_fragmented_send_size;
	92	int max_receive_size;
	93	int keep_alive_interval;
	94	int max_readwrite_size;
	95	enum keep_alive_status keep_alive_requested;
	96	int protocol;
	97	atomic_t send_credits;
	98	atomic_t receive_credits;
	99	int receive_credit_target;
100	int fragment_reassembly_remaining;
101
c7398583 LL	102	/* Memory registrations */
	103	/* Maximum number of RDMA read/write outstanding on this connection */
	104	int responder_resources;
	105	/* Maximum number of SGEs in a RDMA write/read */
	106	int max_frmr_depth;
	107	/*
	108	* If payload is less than or equal to the threshold,
	109	* use RDMA send/recv to send upper layer I/O.
	110	* If payload is more than the threshold,
	111	* use RDMA read/write through memory registration for I/O.
	112	*/
	113	int rdma_readwrite_threshold;
	114	enum ib_mr_type mr_type;
	115	struct list_head mr_list;
	116	spinlock_t mr_list_lock;
	117	/* The number of available MRs ready for memory registration */
	118	atomic_t mr_ready_count;
	119	atomic_t mr_used_count;
	120	wait_queue_head_t wait_mr;
	121	struct work_struct mr_recovery_work;
	122	/* Used by transport to wait until all MRs are returned */
	123	wait_queue_head_t wait_for_mr_cleanup;
	124
f198186a	125	/* Activity accoutning */
f64b78fd	126	/* Pending reqeusts issued from upper layer */
d649e1bb LL	127	int smbd_send_pending;
	128	wait_queue_head_t wait_smbd_send_pending;
	129
f64b78fd LL	130	int smbd_recv_pending;
f64b78fd LL	131	wait_queue_head_t wait_smbd_recv_pending;
f198186a LL	132
	133	atomic_t send_pending;
	134	wait_queue_head_t wait_send_pending;
	135	atomic_t send_payload_pending;
	136	wait_queue_head_t wait_send_payload_pending;
	137
	138	/* Receive queue */
	139	struct list_head receive_queue;
	140	int count_receive_queue;
	141	spinlock_t receive_queue_lock;
	142
	143	struct list_head empty_packet_queue;
	144	int count_empty_packet_queue;
	145	spinlock_t empty_packet_queue_lock;
	146
	147	wait_queue_head_t wait_receive_queues;
	148
	149	/* Reassembly queue */
	150	struct list_head reassembly_queue;
	151	spinlock_t reassembly_queue_lock;
	152	wait_queue_head_t wait_reassembly_queue;
	153
	154	/* total data length of reassembly queue */
	155	int reassembly_data_length;
	156	int reassembly_queue_length;
	157	/* the offset to first buffer in reassembly queue */
	158	int first_entry_offset;
	159
	160	bool send_immediate;
	161
	162	wait_queue_head_t wait_send_queue;
	163
	164	/*
	165	* Indicate if we have received a full packet on the connection
	166	* This is used to identify the first SMBD packet of a assembled
	167	* payload (SMB packet) in reassembly queue so we can return a
	168	* RFC1002 length to upper layer to indicate the length of the SMB
	169	* packet received
	170	*/
	171	bool full_packet_received;
	172
	173	struct workqueue_struct *workqueue;
	174	struct delayed_work idle_timer_work;
	175	struct delayed_work send_immediate_work;
	176
	177	/* Memory pool for preallocating buffers */
	178	/* request pool for RDMA send */
	179	struct kmem_cache *request_cache;
	180	mempool_t *request_mempool;
	181
	182	/* response pool for RDMA receive */
	183	struct kmem_cache *response_cache;
	184	mempool_t *response_mempool;
	185
	186	/* for debug purposes */
	187	unsigned int count_get_receive_buffer;
	188	unsigned int count_put_receive_buffer;
	189	unsigned int count_reassembly_queue;
	190	unsigned int count_enqueue_reassembly_queue;
	191	unsigned int count_dequeue_reassembly_queue;
	192	unsigned int count_send_empty;
	193	};
	194
	195	enum smbd_message_type {
196	SMBD_NEGOTIATE_RESP,
197	SMBD_TRANSFER_DATA,
198	};
199
200	#define SMB_DIRECT_RESPONSE_REQUESTED 0x0001
201
202	/* SMBD negotiation request packet [MS-SMBD] 2.2.1 */
203	struct smbd_negotiate_req {
204	__le16 min_version;
205	__le16 max_version;
206	__le16 reserved;
207	__le16 credits_requested;
208	__le32 preferred_send_size;
209	__le32 max_receive_size;
210	__le32 max_fragmented_size;
211	} __packed;
212
213	/* SMBD negotiation response packet [MS-SMBD] 2.2.2 */
214	struct smbd_negotiate_resp {
215	__le16 min_version;
216	__le16 max_version;
217	__le16 negotiated_version;
218	__le16 reserved;
219	__le16 credits_requested;
220	__le16 credits_granted;
221	__le32 status;
222	__le32 max_readwrite_size;
223	__le32 preferred_send_size;
224	__le32 max_receive_size;
225	__le32 max_fragmented_size;
226	} __packed;
227
228	/* SMBD data transfer packet with payload [MS-SMBD] 2.2.3 */
229	struct smbd_data_transfer {
230	__le16 credits_requested;
231	__le16 credits_granted;
232	__le16 flags;
233	__le16 reserved;
234	__le32 remaining_data_length;
235	__le32 data_offset;
236	__le32 data_length;
237	__le32 padding;
238	__u8 buffer[];
239	} __packed;
240
241	/* The packet fields for a registered RDMA buffer */
242	struct smbd_buffer_descriptor_v1 {
243	__le64 offset;
244	__le32 token;
245	__le32 length;
246	} __packed;
247
03bee01d LL	248	/* Default maximum number of SGEs in a RDMA send/recv */
03bee01d LL	249	#define SMBDIRECT_MAX_SGE 16
f198186a LL	250	/* The context for a SMBD request */
	251	struct smbd_request {
	252	struct smbd_connection *info;
	253	struct ib_cqe cqe;
	254
	255	/* true if this request carries upper layer payload */
	256	bool has_payload;
	257
	258	/* the SGE entries for this packet */
	259	struct ib_sge sge[SMBDIRECT_MAX_SGE];
	260	int num_sge;
	261
	262	/* SMBD packet header follows this structure */
	263	u8 packet[];
	264	};
	265
	266	/* The context for a SMBD response */
	267	struct smbd_response {
	268	struct smbd_connection *info;
	269	struct ib_cqe cqe;
	270	struct ib_sge sge;
	271
	272	enum smbd_message_type type;
	273
	274	/* Link to receive queue or reassembly queue */
	275	struct list_head list;
	276
	277	/* Indicate if this is the 1st packet of a payload */
	278	bool first_segment;
	279
	280	/* SMBD packet header and payload follows this structure */
	281	u8 packet[];
	282	};
	283
399f9539 LL	284	/* Create a SMBDirect session */
	285	struct smbd_connection *smbd_get_connection(
	286	struct TCP_Server_Info server, struct sockaddr dstaddr);
	287
ad57b8e1 LL	288	/* Reconnect SMBDirect session */
ad57b8e1 LL	289	int smbd_reconnect(struct TCP_Server_Info *server);
8ef130f9 LL	290	/* Destroy SMBDirect session */
8ef130f9 LL	291	void smbd_destroy(struct smbd_connection *info);
ad57b8e1	292
f64b78fd LL	293	/* Interface for carrying upper layer I/O through send/recv */
f64b78fd LL	294	int smbd_recv(struct smbd_connection info, struct msghdr msg);
81f39f95	295	int smbd_send(struct TCP_Server_Info server, struct smb_rqst rqst);
f64b78fd	296
c7398583 LL	297	enum mr_state {
	298	MR_READY,
	299	MR_REGISTERED,
	300	MR_INVALIDATED,
	301	MR_ERROR
	302	};
	303
	304	struct smbd_mr {
	305	struct smbd_connection *conn;
	306	struct list_head list;
	307	enum mr_state state;
	308	struct ib_mr *mr;
	309	struct scatterlist *sgl;
	310	int sgl_count;
	311	enum dma_data_direction dir;
	312	union {
	313	struct ib_reg_wr wr;
	314	struct ib_send_wr inv_wr;
	315	};
	316	struct ib_cqe cqe;
	317	bool need_invalidate;
	318	struct completion invalidate_done;
	319	};
	320
	321	/* Interfaces to register and deregister MR for RDMA read/write */
	322	struct smbd_mr *smbd_register_mr(
	323	struct smbd_connection info, struct page pages[], int num_pages,
7cf20bce	324	int offset, int tailsz, bool writing, bool need_invalidate);
c7398583 LL	325	int smbd_deregister_mr(struct smbd_mr *mr);
c7398583 LL	326
f198186a LL	327	#else
	328	#define cifs_rdma_enabled(server) 0
	329	struct smbd_connection {};
399f9539 LL	330	static inline void *smbd_get_connection(
399f9539 LL	331	struct TCP_Server_Info server, struct sockaddr dstaddr) {return NULL;}
ad57b8e1	332	static inline int smbd_reconnect(struct TCP_Server_Info *server) {return -1; }
8ef130f9	333	static inline void smbd_destroy(struct smbd_connection *info) {}
f64b78fd	334	static inline int smbd_recv(struct smbd_connection info, struct msghdr msg) {return -1; }
81f39f95	335	static inline int smbd_send(struct TCP_Server_Info server, struct smb_rqst rqst) {return -1; }
f198186a LL	336	#endif
f198186a LL	337
03bee01d	338	#endif