[linux.git] / drivers / crypto / qat / qat_common / qat_crypto.c

// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2014 - 2020 Intel Corporation */
#include <linux/module.h>
#include <linux/slab.h>
#include "adf_accel_devices.h"
#include "adf_common_drv.h"
#include "adf_transport.h"
#include "adf_cfg.h"
#include "adf_cfg_strings.h"
#include "adf_gen2_hw_data.h"
#include "qat_crypto.h"
#include "icp_qat_fw.h"

#define SEC ADF_KERNEL_SEC

static struct service_hndl qat_crypto;

void qat_crypto_put_instance(struct qat_crypto_instance *inst)
{
	atomic_dec(&inst->refctr);
	adf_dev_put(inst->accel_dev);
}

static int qat_crypto_free_instances(struct adf_accel_dev *accel_dev)
{
	struct qat_crypto_instance *inst, *tmp;
	int i;

	list_for_each_entry_safe(inst, tmp, &accel_dev->crypto_list, list) {
		for (i = 0; i < atomic_read(&inst->refctr); i++)
			qat_crypto_put_instance(inst);

		if (inst->sym_tx)
			adf_remove_ring(inst->sym_tx);

		if (inst->sym_rx)
			adf_remove_ring(inst->sym_rx);

		if (inst->pke_tx)
			adf_remove_ring(inst->pke_tx);

		if (inst->pke_rx)
			adf_remove_ring(inst->pke_rx);

		list_del(&inst->list);
		kfree(inst);
	}
	return 0;
}

struct qat_crypto_instance *qat_crypto_get_instance_node(int node)
{
	struct adf_accel_dev *accel_dev = NULL, *tmp_dev;
	struct qat_crypto_instance *inst = NULL, *tmp_inst;
	unsigned long best = ~0;

	list_for_each_entry(tmp_dev, adf_devmgr_get_head(), list) {
		unsigned long ctr;

		if ((node == dev_to_node(&GET_DEV(tmp_dev)) ||
		     dev_to_node(&GET_DEV(tmp_dev)) < 0) &&
		    adf_dev_started(tmp_dev) &&
		    !list_empty(&tmp_dev->crypto_list)) {
			ctr = atomic_read(&tmp_dev->ref_count);
			if (best > ctr) {
				accel_dev = tmp_dev;
				best = ctr;
			}
		}
	}

	if (!accel_dev) {
		pr_debug_ratelimited("QAT: Could not find a device on node %d\n", node);
		/* Get any started device */
		list_for_each_entry(tmp_dev, adf_devmgr_get_head(), list) {
			if (adf_dev_started(tmp_dev) &&
			    !list_empty(&tmp_dev->crypto_list)) {
				accel_dev = tmp_dev;
				break;
			}
		}
	}

	if (!accel_dev)
		return NULL;

	best = ~0;
	list_for_each_entry(tmp_inst, &accel_dev->crypto_list, list) {
		unsigned long ctr;

		ctr = atomic_read(&tmp_inst->refctr);
		if (best > ctr) {
			inst = tmp_inst;
			best = ctr;
		}
	}
	if (inst) {
		if (adf_dev_get(accel_dev)) {
			dev_err(&GET_DEV(accel_dev), "Could not increment dev refctr\n");
			return NULL;
		}
		atomic_inc(&inst->refctr);
	}
	return inst;
}

/**
 * qat_crypto_vf_dev_config()
 *     create dev config required to create crypto inst.
 *
 * @accel_dev: Pointer to acceleration device.
 *
 * Function creates device configuration required to create
 * asym, sym or, crypto instances
 *
 * Return: 0 on success, error code otherwise.
 */
int qat_crypto_vf_dev_config(struct adf_accel_dev *accel_dev)
{
	u16 ring_to_svc_map = GET_HW_DATA(accel_dev)->ring_to_svc_map;

	if (ring_to_svc_map != ADF_GEN2_DEFAULT_RING_TO_SRV_MAP) {
		dev_err(&GET_DEV(accel_dev),
			"Unsupported ring/service mapping present on PF");
		return -EFAULT;
	}

	return GET_HW_DATA(accel_dev)->dev_config(accel_dev);
}

static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev)
{
	unsigned long num_inst, num_msg_sym, num_msg_asym;
	char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
	char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
	unsigned long sym_bank, asym_bank;
	struct qat_crypto_instance *inst;
	int msg_size;
	int ret;
	int i;

	INIT_LIST_HEAD(&accel_dev->crypto_list);
	ret = adf_cfg_get_param_value(accel_dev, SEC, ADF_NUM_CY, val);
	if (ret)
		return ret;

	ret = kstrtoul(val, 0, &num_inst);
	if (ret)
		return ret;

	for (i = 0; i < num_inst; i++) {
		inst = kzalloc_node(sizeof(*inst), GFP_KERNEL,
				    dev_to_node(&GET_DEV(accel_dev)));
		if (!inst) {
			ret = -ENOMEM;
			goto err;
		}

		list_add_tail(&inst->list, &accel_dev->crypto_list);
		inst->id = i;
		atomic_set(&inst->refctr, 0);
		inst->accel_dev = accel_dev;

		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i);
		ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);
		if (ret)
			goto err;

		ret = kstrtoul(val, 10, &sym_bank);
		if (ret)
			goto err;

		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i);
		ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);
		if (ret)
			goto err;

		ret = kstrtoul(val, 10, &asym_bank);
		if (ret)
			goto err;

		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i);
		ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);
		if (ret)
			goto err;

		ret = kstrtoul(val, 10, &num_msg_sym);
		if (ret)
			goto err;

		num_msg_sym = num_msg_sym >> 1;

		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i);
		ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);
		if (ret)
			goto err;

		ret = kstrtoul(val, 10, &num_msg_asym);
		if (ret)
			goto err;
		num_msg_asym = num_msg_asym >> 1;

		msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i);
		ret = adf_create_ring(accel_dev, SEC, sym_bank, num_msg_sym,
				      msg_size, key, NULL, 0, &inst->sym_tx);
		if (ret)
			goto err;

		msg_size = msg_size >> 1;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i);
		ret = adf_create_ring(accel_dev, SEC, asym_bank, num_msg_asym,
				      msg_size, key, NULL, 0, &inst->pke_tx);
		if (ret)
			goto err;

		msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ;
		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i);
		ret = adf_create_ring(accel_dev, SEC, sym_bank, num_msg_sym,
				      msg_size, key, qat_alg_callback, 0,
				      &inst->sym_rx);
		if (ret)
			goto err;

		snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i);
		ret = adf_create_ring(accel_dev, SEC, asym_bank, num_msg_asym,
				      msg_size, key, qat_alg_asym_callback, 0,
				      &inst->pke_rx);
		if (ret)
			goto err;

		INIT_LIST_HEAD(&inst->backlog.list);
		spin_lock_init(&inst->backlog.lock);
	}
	return 0;
err:
	qat_crypto_free_instances(accel_dev);
	return ret;
}

static int qat_crypto_init(struct adf_accel_dev *accel_dev)
{
	if (qat_crypto_create_instances(accel_dev))
		return -EFAULT;

	return 0;
}

static int qat_crypto_shutdown(struct adf_accel_dev *accel_dev)
{
	return qat_crypto_free_instances(accel_dev);
}

static int qat_crypto_event_handler(struct adf_accel_dev *accel_dev,
				    enum adf_event event)
{
	int ret;

	switch (event) {
	case ADF_EVENT_INIT:
		ret = qat_crypto_init(accel_dev);
		break;
	case ADF_EVENT_SHUTDOWN:
		ret = qat_crypto_shutdown(accel_dev);
		break;
	case ADF_EVENT_RESTARTING:
	case ADF_EVENT_RESTARTED:
	case ADF_EVENT_START:
	case ADF_EVENT_STOP:
	default:
		ret = 0;
	}
	return ret;
}

int qat_crypto_register(void)
{
	memset(&qat_crypto, 0, sizeof(qat_crypto));
	qat_crypto.event_hld = qat_crypto_event_handler;
	qat_crypto.name = "qat_crypto";
	return adf_service_register(&qat_crypto);
}

int qat_crypto_unregister(void)
{
	return adf_service_unregister(&qat_crypto);
}
Commit	Line	Data
1532e31f GC	1	// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
1532e31f GC	2	/* Copyright(c) 2014 - 2020 Intel Corporation */
d370cec3 TS	3	#include <linux/module.h>
	4	#include <linux/slab.h>
	5	#include "adf_accel_devices.h"
	6	#include "adf_common_drv.h"
	7	#include "adf_transport.h"
	8	#include "adf_cfg.h"
	9	#include "adf_cfg_strings.h"
925b3069	10	#include "adf_gen2_hw_data.h"
d370cec3 TS	11	#include "qat_crypto.h"
	12	#include "icp_qat_fw.h"
	13
	14	#define SEC ADF_KERNEL_SEC
	15
	16	static struct service_hndl qat_crypto;
	17
	18	void qat_crypto_put_instance(struct qat_crypto_instance *inst)
	19	{
3cc43a0a TS	20	atomic_dec(&inst->refctr);
3cc43a0a TS	21	adf_dev_put(inst->accel_dev);
d370cec3 TS	22	}
	23
	24	static int qat_crypto_free_instances(struct adf_accel_dev *accel_dev)
	25	{
dc2c6322	26	struct qat_crypto_instance inst, tmp;
d370cec3 TS	27	int i;
d370cec3 TS	28
dc2c6322	29	list_for_each_entry_safe(inst, tmp, &accel_dev->crypto_list, list) {
d370cec3 TS	30	for (i = 0; i < atomic_read(&inst->refctr); i++)
	31	qat_crypto_put_instance(inst);
	32
	33	if (inst->sym_tx)
	34	adf_remove_ring(inst->sym_tx);
	35
	36	if (inst->sym_rx)
	37	adf_remove_ring(inst->sym_rx);
	38
	39	if (inst->pke_tx)
	40	adf_remove_ring(inst->pke_tx);
	41
	42	if (inst->pke_rx)
	43	adf_remove_ring(inst->pke_rx);
	44
dc2c6322	45	list_del(&inst->list);
d370cec3 TS	46	kfree(inst);
	47	}
	48	return 0;
	49	}
	50
	51	struct qat_crypto_instance *qat_crypto_get_instance_node(int node)
	52	{
dc2c6322 GT	53	struct adf_accel_dev accel_dev = NULL, tmp_dev;
dc2c6322 GT	54	struct qat_crypto_instance inst = NULL, tmp_inst;
d370cec3 TS	55	unsigned long best = ~0;
d370cec3 TS	56
dc2c6322	57	list_for_each_entry(tmp_dev, adf_devmgr_get_head(), list) {
3cc43a0a TS	58	unsigned long ctr;
3cc43a0a TS	59
3cc43a0a TS	60	if ((node == dev_to_node(&GET_DEV(tmp_dev)) \|\|
	61	dev_to_node(&GET_DEV(tmp_dev)) < 0) &&
	62	adf_dev_started(tmp_dev) &&
	63	!list_empty(&tmp_dev->crypto_list)) {
	64	ctr = atomic_read(&tmp_dev->ref_count);
	65	if (best > ctr) {
	66	accel_dev = tmp_dev;
	67	best = ctr;
	68	}
	69	}
d370cec3	70	}
4a4b0bad	71
d956fed7	72	if (!accel_dev) {
0ceb587d	73	pr_debug_ratelimited("QAT: Could not find a device on node %d\n", node);
d956fed7	74	/* Get any started device */
dc2c6322	75	list_for_each_entry(tmp_dev, adf_devmgr_get_head(), list) {
d956fed7 TS	76	if (adf_dev_started(tmp_dev) &&
	77	!list_empty(&tmp_dev->crypto_list)) {
	78	accel_dev = tmp_dev;
	79	break;
	80	}
4a4b0bad	81	}
d370cec3	82	}
4a4b0bad TS	83
4a4b0bad TS	84	if (!accel_dev)
d370cec3 TS	85	return NULL;
d370cec3 TS	86
3cc43a0a	87	best = ~0;
dc2c6322	88	list_for_each_entry(tmp_inst, &accel_dev->crypto_list, list) {
3cc43a0a TS	89	unsigned long ctr;
3cc43a0a TS	90
3cc43a0a TS	91	ctr = atomic_read(&tmp_inst->refctr);
	92	if (best > ctr) {
	93	inst = tmp_inst;
	94	best = ctr;
d370cec3 TS	95	}
d370cec3 TS	96	}
3cc43a0a TS	97	if (inst) {
	98	if (adf_dev_get(accel_dev)) {
	99	dev_err(&GET_DEV(accel_dev), "Could not increment dev refctr\n");
	100	return NULL;
d370cec3	101	}
3cc43a0a	102	atomic_inc(&inst->refctr);
d370cec3	103	}
3cc43a0a	104	return inst;
d370cec3 TS	105	}
d370cec3 TS	106
925b3069 MC	107	/**
	108	* qat_crypto_vf_dev_config()
	109	* create dev config required to create crypto inst.
	110	*
	111	* @accel_dev: Pointer to acceleration device.
	112	*
	113	* Function creates device configuration required to create
	114	* asym, sym or, crypto instances
	115	*
	116	* Return: 0 on success, error code otherwise.
	117	*/
	118	int qat_crypto_vf_dev_config(struct adf_accel_dev *accel_dev)
	119	{
	120	u16 ring_to_svc_map = GET_HW_DATA(accel_dev)->ring_to_svc_map;
	121
	122	if (ring_to_svc_map != ADF_GEN2_DEFAULT_RING_TO_SRV_MAP) {
	123	dev_err(&GET_DEV(accel_dev),
	124	"Unsupported ring/service mapping present on PF");
	125	return -EFAULT;
	126	}
	127
93b2f579	128	return GET_HW_DATA(accel_dev)->dev_config(accel_dev);
925b3069 MC	129	}
925b3069 MC	130
d370cec3 TS	131	static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev)
d370cec3 TS	132	{
657ad678	133	unsigned long num_inst, num_msg_sym, num_msg_asym;
d370cec3 TS	134	char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
d370cec3 TS	135	char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
657ad678	136	unsigned long sym_bank, asym_bank;
533179ce GC	137	struct qat_crypto_instance *inst;
	138	int msg_size;
	139	int ret;
	140	int i;
d370cec3 TS	141
d370cec3 TS	142	INIT_LIST_HEAD(&accel_dev->crypto_list);
533179ce GC	143	ret = adf_cfg_get_param_value(accel_dev, SEC, ADF_NUM_CY, val);
	144	if (ret)
	145	return ret;
d370cec3	146
533179ce GC	147	ret = kstrtoul(val, 0, &num_inst);
	148	if (ret)
	149	return ret;
d370cec3 TS	150
	151	for (i = 0; i < num_inst; i++) {
	152	inst = kzalloc_node(sizeof(*inst), GFP_KERNEL,
09adc878	153	dev_to_node(&GET_DEV(accel_dev)));
533179ce GC	154	if (!inst) {
533179ce GC	155	ret = -ENOMEM;
d370cec3	156	goto err;
533179ce	157	}
d370cec3 TS	158
	159	list_add_tail(&inst->list, &accel_dev->crypto_list);
	160	inst->id = i;
	161	atomic_set(&inst->refctr, 0);
	162	inst->accel_dev = accel_dev;
657ad678 GC	163
	164	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i);
	165	ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);
	166	if (ret)
	167	goto err;
	168
	169	ret = kstrtoul(val, 10, &sym_bank);
	170	if (ret)
	171	goto err;
	172
	173	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i);
533179ce GC	174	ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);
533179ce GC	175	if (ret)
d370cec3 TS	176	goto err;
d370cec3 TS	177
657ad678	178	ret = kstrtoul(val, 10, &asym_bank);
533179ce	179	if (ret)
d370cec3	180	goto err;
657ad678	181
d370cec3	182	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i);
533179ce GC	183	ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);
533179ce GC	184	if (ret)
d370cec3 TS	185	goto err;
d370cec3 TS	186
533179ce GC	187	ret = kstrtoul(val, 10, &num_msg_sym);
533179ce GC	188	if (ret)
d370cec3	189	goto err;
a5733139	190
d370cec3	191	num_msg_sym = num_msg_sym >> 1;
a5733139	192
d370cec3	193	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i);
533179ce GC	194	ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);
533179ce GC	195	if (ret)
d370cec3 TS	196	goto err;
d370cec3 TS	197
533179ce GC	198	ret = kstrtoul(val, 10, &num_msg_asym);
533179ce GC	199	if (ret)
d370cec3 TS	200	goto err;
	201	num_msg_asym = num_msg_asym >> 1;
	202
	203	msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ;
	204	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i);
657ad678	205	ret = adf_create_ring(accel_dev, SEC, sym_bank, num_msg_sym,
533179ce GC	206	msg_size, key, NULL, 0, &inst->sym_tx);
533179ce GC	207	if (ret)
d370cec3 TS	208	goto err;
d370cec3 TS	209
d370cec3 TS	210	msg_size = msg_size >> 1;
d370cec3 TS	211	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i);
657ad678	212	ret = adf_create_ring(accel_dev, SEC, asym_bank, num_msg_asym,
533179ce GC	213	msg_size, key, NULL, 0, &inst->pke_tx);
533179ce GC	214	if (ret)
d370cec3 TS	215	goto err;
	216
	217	msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ;
	218	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i);
657ad678	219	ret = adf_create_ring(accel_dev, SEC, sym_bank, num_msg_sym,
533179ce GC	220	msg_size, key, qat_alg_callback, 0,
	221	&inst->sym_rx);
	222	if (ret)
d370cec3 TS	223	goto err;
d370cec3 TS	224
d370cec3	225	snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i);
657ad678	226	ret = adf_create_ring(accel_dev, SEC, asym_bank, num_msg_asym,
533179ce GC	227	msg_size, key, qat_alg_asym_callback, 0,
	228	&inst->pke_rx);
	229	if (ret)
d370cec3	230	goto err;
38682383 GC	231
	232	INIT_LIST_HEAD(&inst->backlog.list);
	233	spin_lock_init(&inst->backlog.lock);
d370cec3 TS	234	}
	235	return 0;
	236	err:
	237	qat_crypto_free_instances(accel_dev);
533179ce	238	return ret;
d370cec3 TS	239	}
	240
	241	static int qat_crypto_init(struct adf_accel_dev *accel_dev)
	242	{
	243	if (qat_crypto_create_instances(accel_dev))
	244	return -EFAULT;
	245
	246	return 0;
	247	}
	248
	249	static int qat_crypto_shutdown(struct adf_accel_dev *accel_dev)
	250	{
	251	return qat_crypto_free_instances(accel_dev);
	252	}
	253
	254	static int qat_crypto_event_handler(struct adf_accel_dev *accel_dev,
	255	enum adf_event event)
	256	{
	257	int ret;
	258
	259	switch (event) {
	260	case ADF_EVENT_INIT:
	261	ret = qat_crypto_init(accel_dev);
	262	break;
	263	case ADF_EVENT_SHUTDOWN:
	264	ret = qat_crypto_shutdown(accel_dev);
	265	break;
	266	case ADF_EVENT_RESTARTING:
	267	case ADF_EVENT_RESTARTED:
	268	case ADF_EVENT_START:
	269	case ADF_EVENT_STOP:
	270	default:
	271	ret = 0;
	272	}
	273	return ret;
	274	}
	275
	276	int qat_crypto_register(void)
	277	{
	278	memset(&qat_crypto, 0, sizeof(qat_crypto));
	279	qat_crypto.event_hld = qat_crypto_event_handler;
	280	qat_crypto.name = "qat_crypto";
	281	return adf_service_register(&qat_crypto);
	282	}
	283
	284	int qat_crypto_unregister(void)
	285	{
	286	return adf_service_unregister(&qat_crypto);
	287	}