[linux.git] / crypto / mcryptd.c

/*
 * Software multibuffer async crypto daemon.
 *
 * Copyright (c) 2014 Tim Chen <[email protected]>
 *
 * Adapted from crypto daemon.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 */

#include <crypto/algapi.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/aead.h>
#include <crypto/mcryptd.h>
#include <crypto/crypto_wq.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/hardirq.h>

#define MCRYPTD_MAX_CPU_QLEN 100
#define MCRYPTD_BATCH 9

static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
				   unsigned int tail);

struct mcryptd_flush_list {
	struct list_head list;
	struct mutex lock;
};

static struct mcryptd_flush_list __percpu *mcryptd_flist;

struct hashd_instance_ctx {
	struct crypto_shash_spawn spawn;
	struct mcryptd_queue *queue;
};

static void mcryptd_queue_worker(struct work_struct *work);

void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay)
{
	struct mcryptd_flush_list *flist;

	if (!cstate->flusher_engaged) {
		/* put the flusher on the flush list */
		flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
		mutex_lock(&flist->lock);
		list_add_tail(&cstate->flush_list, &flist->list);
		cstate->flusher_engaged = true;
		cstate->next_flush = jiffies + delay;
		queue_delayed_work_on(smp_processor_id(), kcrypto_wq,
			&cstate->flush, delay);
		mutex_unlock(&flist->lock);
	}
}
EXPORT_SYMBOL(mcryptd_arm_flusher);

static int mcryptd_init_queue(struct mcryptd_queue *queue,
			     unsigned int max_cpu_qlen)
{
	int cpu;
	struct mcryptd_cpu_queue *cpu_queue;

	queue->cpu_queue = alloc_percpu(struct mcryptd_cpu_queue);
	pr_debug("mqueue:%p mcryptd_cpu_queue %p\n", queue, queue->cpu_queue);
	if (!queue->cpu_queue)
		return -ENOMEM;
	for_each_possible_cpu(cpu) {
		cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
		pr_debug("cpu_queue #%d %p\n", cpu, queue->cpu_queue);
		crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
		INIT_WORK(&cpu_queue->work, mcryptd_queue_worker);
	}
	return 0;
}

static void mcryptd_fini_queue(struct mcryptd_queue *queue)
{
	int cpu;
	struct mcryptd_cpu_queue *cpu_queue;

	for_each_possible_cpu(cpu) {
		cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
		BUG_ON(cpu_queue->queue.qlen);
	}
	free_percpu(queue->cpu_queue);
}

static int mcryptd_enqueue_request(struct mcryptd_queue *queue,
				  struct crypto_async_request *request,
				  struct mcryptd_hash_request_ctx *rctx)
{
	int cpu, err;
	struct mcryptd_cpu_queue *cpu_queue;

	cpu = get_cpu();
	cpu_queue = this_cpu_ptr(queue->cpu_queue);
	rctx->tag.cpu = cpu;

	err = crypto_enqueue_request(&cpu_queue->queue, request);
	pr_debug("enqueue request: cpu %d cpu_queue %p request %p\n",
		 cpu, cpu_queue, request);
	queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
	put_cpu();

	return err;
}

/*
 * Try to opportunisticlly flush the partially completed jobs if
 * crypto daemon is the only task running.
 */
static void mcryptd_opportunistic_flush(void)
{
	struct mcryptd_flush_list *flist;
	struct mcryptd_alg_cstate *cstate;

	flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
	while (single_task_running()) {
		mutex_lock(&flist->lock);
		if (list_empty(&flist->list)) {
			mutex_unlock(&flist->lock);
			return;
		}
		cstate = list_entry(flist->list.next,
				struct mcryptd_alg_cstate, flush_list);
		if (!cstate->flusher_engaged) {
			mutex_unlock(&flist->lock);
			return;
		}
		list_del(&cstate->flush_list);
		cstate->flusher_engaged = false;
		mutex_unlock(&flist->lock);
		cstate->alg_state->flusher(cstate);
	}
}

/*
 * Called in workqueue context, do one real cryption work (via
 * req->complete) and reschedule itself if there are more work to
 * do.
 */
static void mcryptd_queue_worker(struct work_struct *work)
{
	struct mcryptd_cpu_queue *cpu_queue;
	struct crypto_async_request *req, *backlog;
	int i;

	/*
	 * Need to loop through more than once for multi-buffer to
	 * be effective.
	 */

	cpu_queue = container_of(work, struct mcryptd_cpu_queue, work);
	i = 0;
	while (i < MCRYPTD_BATCH || single_task_running()) {
		/*
		 * preempt_disable/enable is used to prevent
		 * being preempted by mcryptd_enqueue_request()
		 */
		local_bh_disable();
		preempt_disable();
		backlog = crypto_get_backlog(&cpu_queue->queue);
		req = crypto_dequeue_request(&cpu_queue->queue);
		preempt_enable();
		local_bh_enable();

		if (!req) {
			mcryptd_opportunistic_flush();
			return;
		}

		if (backlog)
			backlog->complete(backlog, -EINPROGRESS);
		req->complete(req, 0);
		if (!cpu_queue->queue.qlen)
			return;
		++i;
	}
	if (cpu_queue->queue.qlen)
		queue_work(kcrypto_wq, &cpu_queue->work);
}

void mcryptd_flusher(struct work_struct *__work)
{
	struct	mcryptd_alg_cstate	*alg_cpu_state;
	struct	mcryptd_alg_state	*alg_state;
	struct	mcryptd_flush_list	*flist;
	int	cpu;

	cpu = smp_processor_id();
	alg_cpu_state = container_of(to_delayed_work(__work),
				     struct mcryptd_alg_cstate, flush);
	alg_state = alg_cpu_state->alg_state;
	if (alg_cpu_state->cpu != cpu)
		pr_debug("mcryptd error: work on cpu %d, should be cpu %d\n",
				cpu, alg_cpu_state->cpu);

	if (alg_cpu_state->flusher_engaged) {
		flist = per_cpu_ptr(mcryptd_flist, cpu);
		mutex_lock(&flist->lock);
		list_del(&alg_cpu_state->flush_list);
		alg_cpu_state->flusher_engaged = false;
		mutex_unlock(&flist->lock);
		alg_state->flusher(alg_cpu_state);
	}
}
EXPORT_SYMBOL_GPL(mcryptd_flusher);

static inline struct mcryptd_queue *mcryptd_get_queue(struct crypto_tfm *tfm)
{
	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
	struct mcryptd_instance_ctx *ictx = crypto_instance_ctx(inst);

	return ictx->queue;
}

static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
				   unsigned int tail)
{
	char *p;
	struct crypto_instance *inst;
	int err;

	p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
	if (!p)
		return ERR_PTR(-ENOMEM);

	inst = (void *)(p + head);

	err = -ENAMETOOLONG;
	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
		    "mcryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
		goto out_free_inst;

	memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);

	inst->alg.cra_priority = alg->cra_priority + 50;
	inst->alg.cra_blocksize = alg->cra_blocksize;
	inst->alg.cra_alignmask = alg->cra_alignmask;

out:
	return p;

out_free_inst:
	kfree(p);
	p = ERR_PTR(err);
	goto out;
}

static inline void mcryptd_check_internal(struct rtattr **tb, u32 *type,
					  u32 *mask)
{
	struct crypto_attr_type *algt;

	algt = crypto_get_attr_type(tb);
	if (IS_ERR(algt))
		return;
	if ((algt->type & CRYPTO_ALG_INTERNAL))
		*type |= CRYPTO_ALG_INTERNAL;
	if ((algt->mask & CRYPTO_ALG_INTERNAL))
		*mask |= CRYPTO_ALG_INTERNAL;
}

static int mcryptd_hash_init_tfm(struct crypto_tfm *tfm)
{
	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
	struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
	struct crypto_shash_spawn *spawn = &ictx->spawn;
	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
	struct crypto_shash *hash;

	hash = crypto_spawn_shash(spawn);
	if (IS_ERR(hash))
		return PTR_ERR(hash);

	ctx->child = hash;
	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
				 sizeof(struct mcryptd_hash_request_ctx) +
				 crypto_shash_descsize(hash));
	return 0;
}

static void mcryptd_hash_exit_tfm(struct crypto_tfm *tfm)
{
	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);

	crypto_free_shash(ctx->child);
}

static int mcryptd_hash_setkey(struct crypto_ahash *parent,
				   const u8 *key, unsigned int keylen)
{
	struct mcryptd_hash_ctx *ctx   = crypto_ahash_ctx(parent);
	struct crypto_shash *child = ctx->child;
	int err;

	crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
	crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) &
				      CRYPTO_TFM_REQ_MASK);
	err = crypto_shash_setkey(child, key, keylen);
	crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) &
				       CRYPTO_TFM_RES_MASK);
	return err;
}

static int mcryptd_hash_enqueue(struct ahash_request *req,
				crypto_completion_t complete)
{
	int ret;

	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct mcryptd_queue *queue =
		mcryptd_get_queue(crypto_ahash_tfm(tfm));

	rctx->complete = req->base.complete;
	req->base.complete = complete;

	ret = mcryptd_enqueue_request(queue, &req->base, rctx);

	return ret;
}

static void mcryptd_hash_init(struct crypto_async_request *req_async, int err)
{
	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
	struct crypto_shash *child = ctx->child;
	struct ahash_request *req = ahash_request_cast(req_async);
	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
	struct shash_desc *desc = &rctx->desc;

	if (unlikely(err == -EINPROGRESS))
		goto out;

	desc->tfm = child;
	desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;

	err = crypto_shash_init(desc);

	req->base.complete = rctx->complete;

out:
	local_bh_disable();
	rctx->complete(&req->base, err);
	local_bh_enable();
}

static int mcryptd_hash_init_enqueue(struct ahash_request *req)
{
	return mcryptd_hash_enqueue(req, mcryptd_hash_init);
}

static void mcryptd_hash_update(struct crypto_async_request *req_async, int err)
{
	struct ahash_request *req = ahash_request_cast(req_async);
	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);

	if (unlikely(err == -EINPROGRESS))
		goto out;

	err = shash_ahash_mcryptd_update(req, &rctx->desc);
	if (err) {
		req->base.complete = rctx->complete;
		goto out;
	}

	return;
out:
	local_bh_disable();
	rctx->complete(&req->base, err);
	local_bh_enable();
}

static int mcryptd_hash_update_enqueue(struct ahash_request *req)
{
	return mcryptd_hash_enqueue(req, mcryptd_hash_update);
}

static void mcryptd_hash_final(struct crypto_async_request *req_async, int err)
{
	struct ahash_request *req = ahash_request_cast(req_async);
	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);

	if (unlikely(err == -EINPROGRESS))
		goto out;

	err = shash_ahash_mcryptd_final(req, &rctx->desc);
	if (err) {
		req->base.complete = rctx->complete;
		goto out;
	}

	return;
out:
	local_bh_disable();
	rctx->complete(&req->base, err);
	local_bh_enable();
}

static int mcryptd_hash_final_enqueue(struct ahash_request *req)
{
	return mcryptd_hash_enqueue(req, mcryptd_hash_final);
}

static void mcryptd_hash_finup(struct crypto_async_request *req_async, int err)
{
	struct ahash_request *req = ahash_request_cast(req_async);
	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);

	if (unlikely(err == -EINPROGRESS))
		goto out;

	err = shash_ahash_mcryptd_finup(req, &rctx->desc);

	if (err) {
		req->base.complete = rctx->complete;
		goto out;
	}

	return;
out:
	local_bh_disable();
	rctx->complete(&req->base, err);
	local_bh_enable();
}

static int mcryptd_hash_finup_enqueue(struct ahash_request *req)
{
	return mcryptd_hash_enqueue(req, mcryptd_hash_finup);
}

static void mcryptd_hash_digest(struct crypto_async_request *req_async, int err)
{
	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
	struct crypto_shash *child = ctx->child;
	struct ahash_request *req = ahash_request_cast(req_async);
	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
	struct shash_desc *desc = &rctx->desc;

	if (unlikely(err == -EINPROGRESS))
		goto out;

	desc->tfm = child;
	desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;  /* check this again */

	err = shash_ahash_mcryptd_digest(req, desc);

	if (err) {
		req->base.complete = rctx->complete;
		goto out;
	}

	return;
out:
	local_bh_disable();
	rctx->complete(&req->base, err);
	local_bh_enable();
}

static int mcryptd_hash_digest_enqueue(struct ahash_request *req)
{
	return mcryptd_hash_enqueue(req, mcryptd_hash_digest);
}

static int mcryptd_hash_export(struct ahash_request *req, void *out)
{
	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);

	return crypto_shash_export(&rctx->desc, out);
}

static int mcryptd_hash_import(struct ahash_request *req, const void *in)
{
	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);

	return crypto_shash_import(&rctx->desc, in);
}

static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
			      struct mcryptd_queue *queue)
{
	struct hashd_instance_ctx *ctx;
	struct ahash_instance *inst;
	struct shash_alg *salg;
	struct crypto_alg *alg;
	u32 type = 0;
	u32 mask = 0;
	int err;

	mcryptd_check_internal(tb, &type, &mask);

	salg = shash_attr_alg(tb[1], type, mask);
	if (IS_ERR(salg))
		return PTR_ERR(salg);

	alg = &salg->base;
	pr_debug("crypto: mcryptd hash alg: %s\n", alg->cra_name);
	inst = mcryptd_alloc_instance(alg, ahash_instance_headroom(),
					sizeof(*ctx));
	err = PTR_ERR(inst);
	if (IS_ERR(inst))
		goto out_put_alg;

	ctx = ahash_instance_ctx(inst);
	ctx->queue = queue;

	err = crypto_init_shash_spawn(&ctx->spawn, salg,
				      ahash_crypto_instance(inst));
	if (err)
		goto out_free_inst;

	type = CRYPTO_ALG_ASYNC;
	if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
		type |= CRYPTO_ALG_INTERNAL;
	inst->alg.halg.base.cra_flags = type;

	inst->alg.halg.digestsize = salg->digestsize;
	inst->alg.halg.base.cra_ctxsize = sizeof(struct mcryptd_hash_ctx);

	inst->alg.halg.base.cra_init = mcryptd_hash_init_tfm;
	inst->alg.halg.base.cra_exit = mcryptd_hash_exit_tfm;

	inst->alg.init   = mcryptd_hash_init_enqueue;
	inst->alg.update = mcryptd_hash_update_enqueue;
	inst->alg.final  = mcryptd_hash_final_enqueue;
	inst->alg.finup  = mcryptd_hash_finup_enqueue;
	inst->alg.export = mcryptd_hash_export;
	inst->alg.import = mcryptd_hash_import;
	inst->alg.setkey = mcryptd_hash_setkey;
	inst->alg.digest = mcryptd_hash_digest_enqueue;

	err = ahash_register_instance(tmpl, inst);
	if (err) {
		crypto_drop_shash(&ctx->spawn);
out_free_inst:
		kfree(inst);
	}

out_put_alg:
	crypto_mod_put(alg);
	return err;
}

static struct mcryptd_queue mqueue;

static int mcryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
{
	struct crypto_attr_type *algt;

	algt = crypto_get_attr_type(tb);
	if (IS_ERR(algt))
		return PTR_ERR(algt);

	switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
	case CRYPTO_ALG_TYPE_DIGEST:
		return mcryptd_create_hash(tmpl, tb, &mqueue);
	break;
	}

	return -EINVAL;
}

static void mcryptd_free(struct crypto_instance *inst)
{
	struct mcryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
	struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);

	switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
	case CRYPTO_ALG_TYPE_AHASH:
		crypto_drop_shash(&hctx->spawn);
		kfree(ahash_instance(inst));
		return;
	default:
		crypto_drop_spawn(&ctx->spawn);
		kfree(inst);
	}
}

static struct crypto_template mcryptd_tmpl = {
	.name = "mcryptd",
	.create = mcryptd_create,
	.free = mcryptd_free,
	.module = THIS_MODULE,
};

struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name,
					u32 type, u32 mask)
{
	char mcryptd_alg_name[CRYPTO_MAX_ALG_NAME];
	struct crypto_ahash *tfm;

	if (snprintf(mcryptd_alg_name, CRYPTO_MAX_ALG_NAME,
		     "mcryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
		return ERR_PTR(-EINVAL);
	tfm = crypto_alloc_ahash(mcryptd_alg_name, type, mask);
	if (IS_ERR(tfm))
		return ERR_CAST(tfm);
	if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
		crypto_free_ahash(tfm);
		return ERR_PTR(-EINVAL);
	}

	return __mcryptd_ahash_cast(tfm);
}
EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash);

int shash_ahash_mcryptd_digest(struct ahash_request *req,
			       struct shash_desc *desc)
{
	int err;

	err = crypto_shash_init(desc) ?:
	      shash_ahash_mcryptd_finup(req, desc);

	return err;
}
EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_digest);

int shash_ahash_mcryptd_update(struct ahash_request *req,
			       struct shash_desc *desc)
{
	struct crypto_shash *tfm = desc->tfm;
	struct shash_alg *shash = crypto_shash_alg(tfm);

	/* alignment is to be done by multi-buffer crypto algorithm if needed */

	return shash->update(desc, NULL, 0);
}
EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_update);

int shash_ahash_mcryptd_finup(struct ahash_request *req,
			      struct shash_desc *desc)
{
	struct crypto_shash *tfm = desc->tfm;
	struct shash_alg *shash = crypto_shash_alg(tfm);

	/* alignment is to be done by multi-buffer crypto algorithm if needed */

	return shash->finup(desc, NULL, 0, req->result);
}
EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_finup);

int shash_ahash_mcryptd_final(struct ahash_request *req,
			      struct shash_desc *desc)
{
	struct crypto_shash *tfm = desc->tfm;
	struct shash_alg *shash = crypto_shash_alg(tfm);

	/* alignment is to be done by multi-buffer crypto algorithm if needed */

	return shash->final(desc, req->result);
}
EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_final);

struct crypto_shash *mcryptd_ahash_child(struct mcryptd_ahash *tfm)
{
	struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);

	return ctx->child;
}
EXPORT_SYMBOL_GPL(mcryptd_ahash_child);

struct shash_desc *mcryptd_shash_desc(struct ahash_request *req)
{
	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
	return &rctx->desc;
}
EXPORT_SYMBOL_GPL(mcryptd_shash_desc);

void mcryptd_free_ahash(struct mcryptd_ahash *tfm)
{
	crypto_free_ahash(&tfm->base);
}
EXPORT_SYMBOL_GPL(mcryptd_free_ahash);


static int __init mcryptd_init(void)
{
	int err, cpu;
	struct mcryptd_flush_list *flist;

	mcryptd_flist = alloc_percpu(struct mcryptd_flush_list);
	for_each_possible_cpu(cpu) {
		flist = per_cpu_ptr(mcryptd_flist, cpu);
		INIT_LIST_HEAD(&flist->list);
		mutex_init(&flist->lock);
	}

	err = mcryptd_init_queue(&mqueue, MCRYPTD_MAX_CPU_QLEN);
	if (err) {
		free_percpu(mcryptd_flist);
		return err;
	}

	err = crypto_register_template(&mcryptd_tmpl);
	if (err) {
		mcryptd_fini_queue(&mqueue);
		free_percpu(mcryptd_flist);
	}

	return err;
}

static void __exit mcryptd_exit(void)
{
	mcryptd_fini_queue(&mqueue);
	crypto_unregister_template(&mcryptd_tmpl);
	free_percpu(mcryptd_flist);
}

subsys_initcall(mcryptd_init);
module_exit(mcryptd_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Software async multibuffer crypto daemon");
MODULE_ALIAS_CRYPTO("mcryptd");
Commit	Line	Data
1e65b81a TC	1	/*
	2	* Software multibuffer async crypto daemon.
	3	*
	4	* Copyright (c) 2014 Tim Chen <[email protected]>
	5	*
	6	* Adapted from crypto daemon.
	7	*
	8	* This program is free software; you can redistribute it and/or modify it
	9	* under the terms of the GNU General Public License as published by the Free
	10	* Software Foundation; either version 2 of the License, or (at your option)
	11	* any later version.
	12	*
	13	*/
	14
	15	#include <crypto/algapi.h>
	16	#include <crypto/internal/hash.h>
	17	#include <crypto/internal/aead.h>
	18	#include <crypto/mcryptd.h>
	19	#include <crypto/crypto_wq.h>
	20	#include <linux/err.h>
	21	#include <linux/init.h>
	22	#include <linux/kernel.h>
	23	#include <linux/list.h>
	24	#include <linux/module.h>
	25	#include <linux/scatterlist.h>
	26	#include <linux/sched.h>
	27	#include <linux/slab.h>
	28	#include <linux/hardirq.h>
	29
	30	#define MCRYPTD_MAX_CPU_QLEN 100
	31	#define MCRYPTD_BATCH 9
	32
	33	static void mcryptd_alloc_instance(struct crypto_alg alg, unsigned int head,
	34	unsigned int tail);
	35
	36	struct mcryptd_flush_list {
	37	struct list_head list;
	38	struct mutex lock;
	39	};
	40
1f6e97f6	41	static struct mcryptd_flush_list __percpu *mcryptd_flist;
1e65b81a TC	42
	43	struct hashd_instance_ctx {
	44	struct crypto_shash_spawn spawn;
	45	struct mcryptd_queue *queue;
	46	};
	47
	48	static void mcryptd_queue_worker(struct work_struct *work);
	49
	50	void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay)
	51	{
	52	struct mcryptd_flush_list *flist;
	53
	54	if (!cstate->flusher_engaged) {
	55	/* put the flusher on the flush list */
	56	flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
	57	mutex_lock(&flist->lock);
	58	list_add_tail(&cstate->flush_list, &flist->list);
	59	cstate->flusher_engaged = true;
	60	cstate->next_flush = jiffies + delay;
	61	queue_delayed_work_on(smp_processor_id(), kcrypto_wq,
	62	&cstate->flush, delay);
	63	mutex_unlock(&flist->lock);
	64	}
	65	}
	66	EXPORT_SYMBOL(mcryptd_arm_flusher);
	67
	68	static int mcryptd_init_queue(struct mcryptd_queue *queue,
	69	unsigned int max_cpu_qlen)
	70	{
	71	int cpu;
	72	struct mcryptd_cpu_queue *cpu_queue;
	73
	74	queue->cpu_queue = alloc_percpu(struct mcryptd_cpu_queue);
	75	pr_debug("mqueue:%p mcryptd_cpu_queue %p\n", queue, queue->cpu_queue);
	76	if (!queue->cpu_queue)
	77	return -ENOMEM;
	78	for_each_possible_cpu(cpu) {
	79	cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
	80	pr_debug("cpu_queue #%d %p\n", cpu, queue->cpu_queue);
	81	crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
	82	INIT_WORK(&cpu_queue->work, mcryptd_queue_worker);
	83	}
	84	return 0;
	85	}
	86
	87	static void mcryptd_fini_queue(struct mcryptd_queue *queue)
	88	{
	89	int cpu;
	90	struct mcryptd_cpu_queue *cpu_queue;
	91
	92	for_each_possible_cpu(cpu) {
	93	cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
	94	BUG_ON(cpu_queue->queue.qlen);
	95	}
	96	free_percpu(queue->cpu_queue);
	97	}
	98
	99	static int mcryptd_enqueue_request(struct mcryptd_queue *queue,
	100	struct crypto_async_request *request,
	101	struct mcryptd_hash_request_ctx *rctx)
	102	{
	103	int cpu, err;
	104	struct mcryptd_cpu_queue *cpu_queue;
	105
106	cpu = get_cpu();
107	cpu_queue = this_cpu_ptr(queue->cpu_queue);
108	rctx->tag.cpu = cpu;
109
110	err = crypto_enqueue_request(&cpu_queue->queue, request);
111	pr_debug("enqueue request: cpu %d cpu_queue %p request %p\n",
112	cpu, cpu_queue, request);
113	queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
114	put_cpu();
115
116	return err;
117	}
118
119	/*
120	* Try to opportunisticlly flush the partially completed jobs if
121	* crypto daemon is the only task running.
122	*/
123	static void mcryptd_opportunistic_flush(void)
124	{
125	struct mcryptd_flush_list *flist;
126	struct mcryptd_alg_cstate *cstate;
127
128	flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
129	while (single_task_running()) {
130	mutex_lock(&flist->lock);
131	if (list_empty(&flist->list)) {
132	mutex_unlock(&flist->lock);
133	return;
134	}
135	cstate = list_entry(flist->list.next,
136	struct mcryptd_alg_cstate, flush_list);
137	if (!cstate->flusher_engaged) {
138	mutex_unlock(&flist->lock);
139	return;
140	}
141	list_del(&cstate->flush_list);
142	cstate->flusher_engaged = false;
143	mutex_unlock(&flist->lock);
144	cstate->alg_state->flusher(cstate);
145	}
146	}
147
148	/*
149	* Called in workqueue context, do one real cryption work (via
150	* req->complete) and reschedule itself if there are more work to
151	* do.
152	*/
153	static void mcryptd_queue_worker(struct work_struct *work)
154	{
155	struct mcryptd_cpu_queue *cpu_queue;
156	struct crypto_async_request req, backlog;
157	int i;
158
159	/*
160	* Need to loop through more than once for multi-buffer to
161	* be effective.
162	*/
163
164	cpu_queue = container_of(work, struct mcryptd_cpu_queue, work);
165	i = 0;
166	while (i < MCRYPTD_BATCH \|\| single_task_running()) {
167	/*
168	* preempt_disable/enable is used to prevent
169	* being preempted by mcryptd_enqueue_request()
170	*/
171	local_bh_disable();
172	preempt_disable();
173	backlog = crypto_get_backlog(&cpu_queue->queue);
174	req = crypto_dequeue_request(&cpu_queue->queue);
175	preempt_enable();
176	local_bh_enable();
177
178	if (!req) {
179	mcryptd_opportunistic_flush();
180	return;
181	}
182
183	if (backlog)
184	backlog->complete(backlog, -EINPROGRESS);
185	req->complete(req, 0);
186	if (!cpu_queue->queue.qlen)
187	return;
188	++i;
189	}
190	if (cpu_queue->queue.qlen)
191	queue_work(kcrypto_wq, &cpu_queue->work);
192	}
193
194	void mcryptd_flusher(struct work_struct *__work)
195	{
196	struct mcryptd_alg_cstate *alg_cpu_state;
197	struct mcryptd_alg_state *alg_state;
198	struct mcryptd_flush_list *flist;
199	int cpu;
200
201	cpu = smp_processor_id();
202	alg_cpu_state = container_of(to_delayed_work(__work),
203	struct mcryptd_alg_cstate, flush);
204	alg_state = alg_cpu_state->alg_state;
205	if (alg_cpu_state->cpu != cpu)
206	pr_debug("mcryptd error: work on cpu %d, should be cpu %d\n",
207	cpu, alg_cpu_state->cpu);
208
209	if (alg_cpu_state->flusher_engaged) {
210	flist = per_cpu_ptr(mcryptd_flist, cpu);
211	mutex_lock(&flist->lock);
212	list_del(&alg_cpu_state->flush_list);
213	alg_cpu_state->flusher_engaged = false;
214	mutex_unlock(&flist->lock);
215	alg_state->flusher(alg_cpu_state);
216	}
217	}
218	EXPORT_SYMBOL_GPL(mcryptd_flusher);
219
220	static inline struct mcryptd_queue mcryptd_get_queue(struct crypto_tfm tfm)
221	{
222	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
223	struct mcryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
224
225	return ictx->queue;
226	}
227
228	static void mcryptd_alloc_instance(struct crypto_alg alg, unsigned int head,
229	unsigned int tail)
230	{
231	char *p;
232	struct crypto_instance *inst;
233	int err;
234
235	p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
236	if (!p)
237	return ERR_PTR(-ENOMEM);
238
239	inst = (void *)(p + head);
240
241	err = -ENAMETOOLONG;
242	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
243	"mcryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
244	goto out_free_inst;
245
246	memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
247
248	inst->alg.cra_priority = alg->cra_priority + 50;
249	inst->alg.cra_blocksize = alg->cra_blocksize;
250	inst->alg.cra_alignmask = alg->cra_alignmask;
251
252	out:
253	return p;
254
255	out_free_inst:
256	kfree(p);
257	p = ERR_PTR(err);
258	goto out;
259	}
260
f52bbf55 SM	261	static inline void mcryptd_check_internal(struct rtattr *tb, u32 type,
	262	u32 *mask)
	263	{
	264	struct crypto_attr_type *algt;
	265
	266	algt = crypto_get_attr_type(tb);
	267	if (IS_ERR(algt))
	268	return;
	269	if ((algt->type & CRYPTO_ALG_INTERNAL))
	270	*type \|= CRYPTO_ALG_INTERNAL;
	271	if ((algt->mask & CRYPTO_ALG_INTERNAL))
	272	*mask \|= CRYPTO_ALG_INTERNAL;
	273	}
	274
1e65b81a TC	275	static int mcryptd_hash_init_tfm(struct crypto_tfm *tfm)
	276	{
	277	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
	278	struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
	279	struct crypto_shash_spawn *spawn = &ictx->spawn;
	280	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
	281	struct crypto_shash *hash;
	282
	283	hash = crypto_spawn_shash(spawn);
	284	if (IS_ERR(hash))
	285	return PTR_ERR(hash);
	286
	287	ctx->child = hash;
	288	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
	289	sizeof(struct mcryptd_hash_request_ctx) +
	290	crypto_shash_descsize(hash));
	291	return 0;
	292	}
	293
	294	static void mcryptd_hash_exit_tfm(struct crypto_tfm *tfm)
	295	{
	296	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
	297
	298	crypto_free_shash(ctx->child);
	299	}
	300
	301	static int mcryptd_hash_setkey(struct crypto_ahash *parent,
	302	const u8 *key, unsigned int keylen)
	303	{
	304	struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(parent);
	305	struct crypto_shash *child = ctx->child;
	306	int err;
	307
	308	crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
	309	crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) &
	310	CRYPTO_TFM_REQ_MASK);
	311	err = crypto_shash_setkey(child, key, keylen);
	312	crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) &
	313	CRYPTO_TFM_RES_MASK);
	314	return err;
	315	}
	316
	317	static int mcryptd_hash_enqueue(struct ahash_request *req,
	318	crypto_completion_t complete)
	319	{
	320	int ret;
	321
	322	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
	323	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	324	struct mcryptd_queue *queue =
	325	mcryptd_get_queue(crypto_ahash_tfm(tfm));
	326
	327	rctx->complete = req->base.complete;
	328	req->base.complete = complete;
	329
	330	ret = mcryptd_enqueue_request(queue, &req->base, rctx);
	331
	332	return ret;
	333	}
	334
	335	static void mcryptd_hash_init(struct crypto_async_request *req_async, int err)
	336	{
	337	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
	338	struct crypto_shash *child = ctx->child;
339	struct ahash_request *req = ahash_request_cast(req_async);
340	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
341	struct shash_desc *desc = &rctx->desc;
342
343	if (unlikely(err == -EINPROGRESS))
344	goto out;
345
346	desc->tfm = child;
347	desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
348
349	err = crypto_shash_init(desc);
350
351	req->base.complete = rctx->complete;
352
353	out:
354	local_bh_disable();
355	rctx->complete(&req->base, err);
356	local_bh_enable();
357	}
358
359	static int mcryptd_hash_init_enqueue(struct ahash_request *req)
360	{
361	return mcryptd_hash_enqueue(req, mcryptd_hash_init);
362	}
363
364	static void mcryptd_hash_update(struct crypto_async_request *req_async, int err)
365	{
366	struct ahash_request *req = ahash_request_cast(req_async);
367	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
368
369	if (unlikely(err == -EINPROGRESS))
370	goto out;
371
372	err = shash_ahash_mcryptd_update(req, &rctx->desc);
373	if (err) {
374	req->base.complete = rctx->complete;
375	goto out;
376	}
377
378	return;
379	out:
380	local_bh_disable();
381	rctx->complete(&req->base, err);
382	local_bh_enable();
383	}
384
385	static int mcryptd_hash_update_enqueue(struct ahash_request *req)
386	{
387	return mcryptd_hash_enqueue(req, mcryptd_hash_update);
388	}
389
390	static void mcryptd_hash_final(struct crypto_async_request *req_async, int err)
391	{
392	struct ahash_request *req = ahash_request_cast(req_async);
393	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
394
395	if (unlikely(err == -EINPROGRESS))
396	goto out;
397
398	err = shash_ahash_mcryptd_final(req, &rctx->desc);
399	if (err) {
400	req->base.complete = rctx->complete;
401	goto out;
402	}
403
404	return;
405	out:
406	local_bh_disable();
407	rctx->complete(&req->base, err);
408	local_bh_enable();
409	}
410
411	static int mcryptd_hash_final_enqueue(struct ahash_request *req)
412	{
413	return mcryptd_hash_enqueue(req, mcryptd_hash_final);
414	}
415
416	static void mcryptd_hash_finup(struct crypto_async_request *req_async, int err)
417	{
418	struct ahash_request *req = ahash_request_cast(req_async);
419	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
420
421	if (unlikely(err == -EINPROGRESS))
422	goto out;
423
424	err = shash_ahash_mcryptd_finup(req, &rctx->desc);
425
426	if (err) {
427	req->base.complete = rctx->complete;
428	goto out;
429	}
430
431	return;
432	out:
433	local_bh_disable();
434	rctx->complete(&req->base, err);
435	local_bh_enable();
436	}
437
438	static int mcryptd_hash_finup_enqueue(struct ahash_request *req)
439	{
440	return mcryptd_hash_enqueue(req, mcryptd_hash_finup);
441	}
442
443	static void mcryptd_hash_digest(struct crypto_async_request *req_async, int err)
444	{
445	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
446	struct crypto_shash *child = ctx->child;
447	struct ahash_request *req = ahash_request_cast(req_async);
448	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
449	struct shash_desc *desc = &rctx->desc;
450
451	if (unlikely(err == -EINPROGRESS))
452	goto out;
453
454	desc->tfm = child;
455	desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; /* check this again */
456
457	err = shash_ahash_mcryptd_digest(req, desc);
458
459	if (err) {
460	req->base.complete = rctx->complete;
461	goto out;
462	}
463
464	return;
465	out:
466	local_bh_disable();
467	rctx->complete(&req->base, err);
468	local_bh_enable();
469	}
470
471	static int mcryptd_hash_digest_enqueue(struct ahash_request *req)
472	{
473	return mcryptd_hash_enqueue(req, mcryptd_hash_digest);
474	}
475
476	static int mcryptd_hash_export(struct ahash_request req, void out)
477	{
478	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
479
480	return crypto_shash_export(&rctx->desc, out);
481	}
482
483	static int mcryptd_hash_import(struct ahash_request req, const void in)
484	{
485	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
486
487	return crypto_shash_import(&rctx->desc, in);
488	}
489
490	static int mcryptd_create_hash(struct crypto_template tmpl, struct rtattr *tb,
491	struct mcryptd_queue *queue)
492	{
493	struct hashd_instance_ctx *ctx;
494	struct ahash_instance *inst;
495	struct shash_alg *salg;
496	struct crypto_alg *alg;
f52bbf55 SM	497	u32 type = 0;
f52bbf55 SM	498	u32 mask = 0;
1e65b81a TC	499	int err;
1e65b81a TC	500
f52bbf55 SM	501	mcryptd_check_internal(tb, &type, &mask);
	502
	503	salg = shash_attr_alg(tb[1], type, mask);
1e65b81a TC	504	if (IS_ERR(salg))
	505	return PTR_ERR(salg);
	506
	507	alg = &salg->base;
	508	pr_debug("crypto: mcryptd hash alg: %s\n", alg->cra_name);
	509	inst = mcryptd_alloc_instance(alg, ahash_instance_headroom(),
	510	sizeof(*ctx));
	511	err = PTR_ERR(inst);
	512	if (IS_ERR(inst))
	513	goto out_put_alg;
	514
	515	ctx = ahash_instance_ctx(inst);
	516	ctx->queue = queue;
	517
	518	err = crypto_init_shash_spawn(&ctx->spawn, salg,
	519	ahash_crypto_instance(inst));
	520	if (err)
	521	goto out_free_inst;
	522
f52bbf55 SM	523	type = CRYPTO_ALG_ASYNC;
	524	if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
	525	type \|= CRYPTO_ALG_INTERNAL;
	526	inst->alg.halg.base.cra_flags = type;
1e65b81a TC	527
	528	inst->alg.halg.digestsize = salg->digestsize;
	529	inst->alg.halg.base.cra_ctxsize = sizeof(struct mcryptd_hash_ctx);
	530
	531	inst->alg.halg.base.cra_init = mcryptd_hash_init_tfm;
	532	inst->alg.halg.base.cra_exit = mcryptd_hash_exit_tfm;
	533
	534	inst->alg.init = mcryptd_hash_init_enqueue;
	535	inst->alg.update = mcryptd_hash_update_enqueue;
	536	inst->alg.final = mcryptd_hash_final_enqueue;
	537	inst->alg.finup = mcryptd_hash_finup_enqueue;
	538	inst->alg.export = mcryptd_hash_export;
	539	inst->alg.import = mcryptd_hash_import;
	540	inst->alg.setkey = mcryptd_hash_setkey;
	541	inst->alg.digest = mcryptd_hash_digest_enqueue;
	542
	543	err = ahash_register_instance(tmpl, inst);
	544	if (err) {
	545	crypto_drop_shash(&ctx->spawn);
	546	out_free_inst:
	547	kfree(inst);
	548	}
	549
	550	out_put_alg:
	551	crypto_mod_put(alg);
	552	return err;
	553	}
	554
	555	static struct mcryptd_queue mqueue;
	556
	557	static int mcryptd_create(struct crypto_template tmpl, struct rtattr *tb)
	558	{
	559	struct crypto_attr_type *algt;
	560
	561	algt = crypto_get_attr_type(tb);
	562	if (IS_ERR(algt))
	563	return PTR_ERR(algt);
	564
	565	switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
	566	case CRYPTO_ALG_TYPE_DIGEST:
	567	return mcryptd_create_hash(tmpl, tb, &mqueue);
	568	break;
	569	}
	570
	571	return -EINVAL;
	572	}
	573
	574	static void mcryptd_free(struct crypto_instance *inst)
	575	{
	576	struct mcryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
	577	struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
	578
	579	switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
	580	case CRYPTO_ALG_TYPE_AHASH:
	581	crypto_drop_shash(&hctx->spawn);
	582	kfree(ahash_instance(inst));
	583	return;
	584	default:
	585	crypto_drop_spawn(&ctx->spawn);
	586	kfree(inst);
	587	}
	588	}
	589
	590	static struct crypto_template mcryptd_tmpl = {
591	.name = "mcryptd",
592	.create = mcryptd_create,
593	.free = mcryptd_free,
594	.module = THIS_MODULE,
595	};
596
597	struct mcryptd_ahash mcryptd_alloc_ahash(const char alg_name,
598	u32 type, u32 mask)
599	{
600	char mcryptd_alg_name[CRYPTO_MAX_ALG_NAME];
601	struct crypto_ahash *tfm;
602
603	if (snprintf(mcryptd_alg_name, CRYPTO_MAX_ALG_NAME,
604	"mcryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
605	return ERR_PTR(-EINVAL);
606	tfm = crypto_alloc_ahash(mcryptd_alg_name, type, mask);
607	if (IS_ERR(tfm))
608	return ERR_CAST(tfm);
609	if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
610	crypto_free_ahash(tfm);
611	return ERR_PTR(-EINVAL);
612	}
613
614	return __mcryptd_ahash_cast(tfm);
615	}
616	EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash);
617
618	int shash_ahash_mcryptd_digest(struct ahash_request *req,
619	struct shash_desc *desc)
620	{
621	int err;
622
623	err = crypto_shash_init(desc) ?:
624	shash_ahash_mcryptd_finup(req, desc);
625
626	return err;
627	}
628	EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_digest);
629
630	int shash_ahash_mcryptd_update(struct ahash_request *req,
631	struct shash_desc *desc)
632	{
633	struct crypto_shash *tfm = desc->tfm;
634	struct shash_alg *shash = crypto_shash_alg(tfm);
635
636	/* alignment is to be done by multi-buffer crypto algorithm if needed */
637
638	return shash->update(desc, NULL, 0);
639	}
640	EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_update);
641
642	int shash_ahash_mcryptd_finup(struct ahash_request *req,
643	struct shash_desc *desc)
644	{
645	struct crypto_shash *tfm = desc->tfm;
646	struct shash_alg *shash = crypto_shash_alg(tfm);
647
648	/* alignment is to be done by multi-buffer crypto algorithm if needed */
649
650	return shash->finup(desc, NULL, 0, req->result);
651	}
652	EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_finup);
653
654	int shash_ahash_mcryptd_final(struct ahash_request *req,
655	struct shash_desc *desc)
656	{
657	struct crypto_shash *tfm = desc->tfm;
658	struct shash_alg *shash = crypto_shash_alg(tfm);
659
660	/* alignment is to be done by multi-buffer crypto algorithm if needed */
661
662	return shash->final(desc, req->result);
663	}
664	EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_final);
665
666	struct crypto_shash mcryptd_ahash_child(struct mcryptd_ahash tfm)
667	{
668	struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
669
670	return ctx->child;
671	}
672	EXPORT_SYMBOL_GPL(mcryptd_ahash_child);
673
674	struct shash_desc mcryptd_shash_desc(struct ahash_request req)
675	{
676	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
677	return &rctx->desc;
678	}
679	EXPORT_SYMBOL_GPL(mcryptd_shash_desc);
680
681	void mcryptd_free_ahash(struct mcryptd_ahash *tfm)
682	{
683	crypto_free_ahash(&tfm->base);
684	}
685	EXPORT_SYMBOL_GPL(mcryptd_free_ahash);
686
687
688	static int __init mcryptd_init(void)
689	{
690	int err, cpu;
691	struct mcryptd_flush_list *flist;
692
693	mcryptd_flist = alloc_percpu(struct mcryptd_flush_list);
694	for_each_possible_cpu(cpu) {
695	flist = per_cpu_ptr(mcryptd_flist, cpu);
696	INIT_LIST_HEAD(&flist->list);
697	mutex_init(&flist->lock);
698	}
699
700	err = mcryptd_init_queue(&mqueue, MCRYPTD_MAX_CPU_QLEN);
701	if (err) {
702	free_percpu(mcryptd_flist);
703	return err;
704	}
705
706	err = crypto_register_template(&mcryptd_tmpl);
707	if (err) {
708	mcryptd_fini_queue(&mqueue);
709	free_percpu(mcryptd_flist);
710	}
711
712	return err;
713	}
714
715	static void __exit mcryptd_exit(void)
716	{
717	mcryptd_fini_queue(&mqueue);
718	crypto_unregister_template(&mcryptd_tmpl);
719	free_percpu(mcryptd_flist);
720	}
721
722	subsys_initcall(mcryptd_init);
723	module_exit(mcryptd_exit);
724
725	MODULE_LICENSE("GPL");
726	MODULE_DESCRIPTION("Software async multibuffer crypto daemon");
4943ba16	727	MODULE_ALIAS_CRYPTO("mcryptd");