[linux.git] / arch / sparc / crypto / des_glue.c

// SPDX-License-Identifier: GPL-2.0-only
/* Glue code for DES encryption optimized for sparc64 crypto opcodes.
 *
 * Copyright (C) 2012 David S. Miller <[email protected]>
 */

#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt

#include <linux/crypto.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/algapi.h>
#include <crypto/internal/des.h>
#include <crypto/internal/skcipher.h>

#include <asm/fpumacro.h>
#include <asm/pstate.h>
#include <asm/elf.h>

#include "opcodes.h"

struct des_sparc64_ctx {
	u64 encrypt_expkey[DES_EXPKEY_WORDS / 2];
	u64 decrypt_expkey[DES_EXPKEY_WORDS / 2];
};

struct des3_ede_sparc64_ctx {
	u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
	u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
};

static void encrypt_to_decrypt(u64 *d, const u64 *e)
{
	const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1;
	int i;

	for (i = 0; i < DES_EXPKEY_WORDS / 2; i++)
		*d++ = *s--;
}

extern void des_sparc64_key_expand(const u32 *input_key, u64 *key);

static int des_set_key(struct crypto_tfm *tfm, const u8 *key,
		       unsigned int keylen)
{
	struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
	int err;

	/* Even though we have special instructions for key expansion,
	 * we call des_verify_key() so that we don't have to write our own
	 * weak key detection code.
	 */
	err = crypto_des_verify_key(tfm, key);
	if (err)
		return err;

	des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]);
	encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]);

	return 0;
}

static int des_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *key,
				unsigned int keylen)
{
	return des_set_key(crypto_skcipher_tfm(tfm), key, keylen);
}

extern void des_sparc64_crypt(const u64 *key, const u64 *input,
			      u64 *output);

static void sparc_des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
	struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
	const u64 *K = ctx->encrypt_expkey;

	des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
}

static void sparc_des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
	struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
	const u64 *K = ctx->decrypt_expkey;

	des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
}

extern void des_sparc64_load_keys(const u64 *key);

extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output,
				  unsigned int len);

static int __ecb_crypt(struct skcipher_request *req, bool encrypt)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
	struct skcipher_walk walk;
	unsigned int nbytes;
	int err;

	err = skcipher_walk_virt(&walk, req, true);
	if (err)
		return err;

	if (encrypt)
		des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
	else
		des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
	while ((nbytes = walk.nbytes) != 0) {
		des_sparc64_ecb_crypt(walk.src.virt.addr, walk.dst.virt.addr,
				      round_down(nbytes, DES_BLOCK_SIZE));
		err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
	}
	fprs_write(0);
	return err;
}

static int ecb_encrypt(struct skcipher_request *req)
{
	return __ecb_crypt(req, true);
}

static int ecb_decrypt(struct skcipher_request *req)
{
	return __ecb_crypt(req, false);
}

extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output,
				    unsigned int len, u64 *iv);

extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output,
				    unsigned int len, u64 *iv);

static int __cbc_crypt(struct skcipher_request *req, bool encrypt)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
	struct skcipher_walk walk;
	unsigned int nbytes;
	int err;

	err = skcipher_walk_virt(&walk, req, true);
	if (err)
		return err;

	if (encrypt)
		des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
	else
		des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
	while ((nbytes = walk.nbytes) != 0) {
		if (encrypt)
			des_sparc64_cbc_encrypt(walk.src.virt.addr,
						walk.dst.virt.addr,
						round_down(nbytes,
							   DES_BLOCK_SIZE),
						walk.iv);
		else
			des_sparc64_cbc_decrypt(walk.src.virt.addr,
						walk.dst.virt.addr,
						round_down(nbytes,
							   DES_BLOCK_SIZE),
						walk.iv);
		err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
	}
	fprs_write(0);
	return err;
}

static int cbc_encrypt(struct skcipher_request *req)
{
	return __cbc_crypt(req, true);
}

static int cbc_decrypt(struct skcipher_request *req)
{
	return __cbc_crypt(req, false);
}

static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key,
			    unsigned int keylen)
{
	struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
	u64 k1[DES_EXPKEY_WORDS / 2];
	u64 k2[DES_EXPKEY_WORDS / 2];
	u64 k3[DES_EXPKEY_WORDS / 2];
	int err;

	err = crypto_des3_ede_verify_key(tfm, key);
	if (err)
		return err;

	des_sparc64_key_expand((const u32 *)key, k1);
	key += DES_KEY_SIZE;
	des_sparc64_key_expand((const u32 *)key, k2);
	key += DES_KEY_SIZE;
	des_sparc64_key_expand((const u32 *)key, k3);

	memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1));
	encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]);
	memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
	       &k3[0], sizeof(k3));

	encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]);
	memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2],
	       &k2[0], sizeof(k2));
	encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
			   &k1[0]);

	return 0;
}

static int des3_ede_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *key,
				     unsigned int keylen)
{
	return des3_ede_set_key(crypto_skcipher_tfm(tfm), key, keylen);
}

extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input,
				   u64 *output);

static void sparc_des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
	struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
	const u64 *K = ctx->encrypt_expkey;

	des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
}

static void sparc_des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
	struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
	const u64 *K = ctx->decrypt_expkey;

	des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
}

extern void des3_ede_sparc64_load_keys(const u64 *key);

extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input,
				       u64 *output, unsigned int len);

static int __ecb3_crypt(struct skcipher_request *req, bool encrypt)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
	struct skcipher_walk walk;
	const u64 *K;
	unsigned int nbytes;
	int err;

	err = skcipher_walk_virt(&walk, req, true);
	if (err)
		return err;

	if (encrypt)
		K = &ctx->encrypt_expkey[0];
	else
		K = &ctx->decrypt_expkey[0];
	des3_ede_sparc64_load_keys(K);
	while ((nbytes = walk.nbytes) != 0) {
		des3_ede_sparc64_ecb_crypt(K, walk.src.virt.addr,
					   walk.dst.virt.addr,
					   round_down(nbytes, DES_BLOCK_SIZE));
		err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
	}
	fprs_write(0);
	return err;
}

static int ecb3_encrypt(struct skcipher_request *req)
{
	return __ecb3_crypt(req, true);
}

static int ecb3_decrypt(struct skcipher_request *req)
{
	return __ecb3_crypt(req, false);
}

extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input,
					 u64 *output, unsigned int len,
					 u64 *iv);

extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input,
					 u64 *output, unsigned int len,
					 u64 *iv);

static int __cbc3_crypt(struct skcipher_request *req, bool encrypt)
{
	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
	struct skcipher_walk walk;
	const u64 *K;
	unsigned int nbytes;
	int err;

	err = skcipher_walk_virt(&walk, req, true);
	if (err)
		return err;

	if (encrypt)
		K = &ctx->encrypt_expkey[0];
	else
		K = &ctx->decrypt_expkey[0];
	des3_ede_sparc64_load_keys(K);
	while ((nbytes = walk.nbytes) != 0) {
		if (encrypt)
			des3_ede_sparc64_cbc_encrypt(K, walk.src.virt.addr,
						     walk.dst.virt.addr,
						     round_down(nbytes,
								DES_BLOCK_SIZE),
						     walk.iv);
		else
			des3_ede_sparc64_cbc_decrypt(K, walk.src.virt.addr,
						     walk.dst.virt.addr,
						     round_down(nbytes,
								DES_BLOCK_SIZE),
						     walk.iv);
		err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
	}
	fprs_write(0);
	return err;
}

static int cbc3_encrypt(struct skcipher_request *req)
{
	return __cbc3_crypt(req, true);
}

static int cbc3_decrypt(struct skcipher_request *req)
{
	return __cbc3_crypt(req, false);
}

static struct crypto_alg cipher_algs[] = {
	{
		.cra_name		= "des",
		.cra_driver_name	= "des-sparc64",
		.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
		.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
		.cra_blocksize		= DES_BLOCK_SIZE,
		.cra_ctxsize		= sizeof(struct des_sparc64_ctx),
		.cra_alignmask		= 7,
		.cra_module		= THIS_MODULE,
		.cra_u	= {
			.cipher	= {
				.cia_min_keysize	= DES_KEY_SIZE,
				.cia_max_keysize	= DES_KEY_SIZE,
				.cia_setkey		= des_set_key,
				.cia_encrypt		= sparc_des_encrypt,
				.cia_decrypt		= sparc_des_decrypt
			}
		}
	}, {
		.cra_name		= "des3_ede",
		.cra_driver_name	= "des3_ede-sparc64",
		.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
		.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
		.cra_blocksize		= DES3_EDE_BLOCK_SIZE,
		.cra_ctxsize		= sizeof(struct des3_ede_sparc64_ctx),
		.cra_alignmask		= 7,
		.cra_module		= THIS_MODULE,
		.cra_u	= {
			.cipher	= {
				.cia_min_keysize	= DES3_EDE_KEY_SIZE,
				.cia_max_keysize	= DES3_EDE_KEY_SIZE,
				.cia_setkey		= des3_ede_set_key,
				.cia_encrypt		= sparc_des3_ede_encrypt,
				.cia_decrypt		= sparc_des3_ede_decrypt
			}
		}
	}
};

static struct skcipher_alg skcipher_algs[] = {
	{
		.base.cra_name		= "ecb(des)",
		.base.cra_driver_name	= "ecb-des-sparc64",
		.base.cra_priority	= SPARC_CR_OPCODE_PRIORITY,
		.base.cra_blocksize	= DES_BLOCK_SIZE,
		.base.cra_ctxsize	= sizeof(struct des_sparc64_ctx),
		.base.cra_alignmask	= 7,
		.base.cra_module	= THIS_MODULE,
		.min_keysize		= DES_KEY_SIZE,
		.max_keysize		= DES_KEY_SIZE,
		.setkey			= des_set_key_skcipher,
		.encrypt		= ecb_encrypt,
		.decrypt		= ecb_decrypt,
	}, {
		.base.cra_name		= "cbc(des)",
		.base.cra_driver_name	= "cbc-des-sparc64",
		.base.cra_priority	= SPARC_CR_OPCODE_PRIORITY,
		.base.cra_blocksize	= DES_BLOCK_SIZE,
		.base.cra_ctxsize	= sizeof(struct des_sparc64_ctx),
		.base.cra_alignmask	= 7,
		.base.cra_module	= THIS_MODULE,
		.min_keysize		= DES_KEY_SIZE,
		.max_keysize		= DES_KEY_SIZE,
		.ivsize			= DES_BLOCK_SIZE,
		.setkey			= des_set_key_skcipher,
		.encrypt		= cbc_encrypt,
		.decrypt		= cbc_decrypt,
	}, {
		.base.cra_name		= "ecb(des3_ede)",
		.base.cra_driver_name	= "ecb-des3_ede-sparc64",
		.base.cra_priority	= SPARC_CR_OPCODE_PRIORITY,
		.base.cra_blocksize	= DES3_EDE_BLOCK_SIZE,
		.base.cra_ctxsize	= sizeof(struct des3_ede_sparc64_ctx),
		.base.cra_alignmask	= 7,
		.base.cra_module	= THIS_MODULE,
		.min_keysize		= DES3_EDE_KEY_SIZE,
		.max_keysize		= DES3_EDE_KEY_SIZE,
		.setkey			= des3_ede_set_key_skcipher,
		.encrypt		= ecb3_encrypt,
		.decrypt		= ecb3_decrypt,
	}, {
		.base.cra_name		= "cbc(des3_ede)",
		.base.cra_driver_name	= "cbc-des3_ede-sparc64",
		.base.cra_priority	= SPARC_CR_OPCODE_PRIORITY,
		.base.cra_blocksize	= DES3_EDE_BLOCK_SIZE,
		.base.cra_ctxsize	= sizeof(struct des3_ede_sparc64_ctx),
		.base.cra_alignmask	= 7,
		.base.cra_module	= THIS_MODULE,
		.min_keysize		= DES3_EDE_KEY_SIZE,
		.max_keysize		= DES3_EDE_KEY_SIZE,
		.ivsize			= DES3_EDE_BLOCK_SIZE,
		.setkey			= des3_ede_set_key_skcipher,
		.encrypt		= cbc3_encrypt,
		.decrypt		= cbc3_decrypt,
	}
};

static bool __init sparc64_has_des_opcode(void)
{
	unsigned long cfr;

	if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
		return false;

	__asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
	if (!(cfr & CFR_DES))
		return false;

	return true;
}

static int __init des_sparc64_mod_init(void)
{
	int err;

	if (!sparc64_has_des_opcode()) {
		pr_info("sparc64 des opcodes not available.\n");
		return -ENODEV;
	}
	pr_info("Using sparc64 des opcodes optimized DES implementation\n");
	err = crypto_register_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
	if (err)
		return err;
	err = crypto_register_skciphers(skcipher_algs,
					ARRAY_SIZE(skcipher_algs));
	if (err)
		crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
	return err;
}

static void __exit des_sparc64_mod_fini(void)
{
	crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
	crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
}

module_init(des_sparc64_mod_init);
module_exit(des_sparc64_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");

MODULE_ALIAS_CRYPTO("des");
MODULE_ALIAS_CRYPTO("des3_ede");

#include "crop_devid.c"
Commit	Line	Data
09c434b8	1	// SPDX-License-Identifier: GPL-2.0-only
c5aac2df DM	2	/* Glue code for DES encryption optimized for sparc64 crypto opcodes.
	3	*
	4	* Copyright (C) 2012 David S. Miller <[email protected]>
	5	*/
	6
71741680 DM	7	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
71741680 DM	8
c5aac2df DM	9	#include <linux/crypto.h>
	10	#include <linux/init.h>
	11	#include <linux/module.h>
	12	#include <linux/mm.h>
	13	#include <linux/types.h>
	14	#include <crypto/algapi.h>
8f467cf2	15	#include <crypto/internal/des.h>
cd5d2f84	16	#include <crypto/internal/skcipher.h>
c5aac2df DM	17
	18	#include <asm/fpumacro.h>
	19	#include <asm/pstate.h>
	20	#include <asm/elf.h>
	21
10803624 DM	22	#include "opcodes.h"
10803624 DM	23
c5aac2df DM	24	struct des_sparc64_ctx {
	25	u64 encrypt_expkey[DES_EXPKEY_WORDS / 2];
	26	u64 decrypt_expkey[DES_EXPKEY_WORDS / 2];
	27	};
	28
	29	struct des3_ede_sparc64_ctx {
	30	u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
	31	u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
	32	};
	33
	34	static void encrypt_to_decrypt(u64 d, const u64 e)
	35	{
	36	const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1;
	37	int i;
	38
	39	for (i = 0; i < DES_EXPKEY_WORDS / 2; i++)
	40	d++ = s--;
	41	}
	42
	43	extern void des_sparc64_key_expand(const u32 input_key, u64 key);
	44
	45	static int des_set_key(struct crypto_tfm tfm, const u8 key,
	46	unsigned int keylen)
	47	{
	48	struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
8f467cf2	49	int err;
c5aac2df DM	50
c5aac2df DM	51	/* Even though we have special instructions for key expansion,
8f467cf2	52	* we call des_verify_key() so that we don't have to write our own
c5aac2df DM	53	* weak key detection code.
c5aac2df DM	54	*/
8f467cf2 AB	55	err = crypto_des_verify_key(tfm, key);
	56	if (err)
	57	return err;
c5aac2df DM	58
	59	des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]);
	60	encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]);
	61
	62	return 0;
	63	}
	64
cd5d2f84 EB	65	static int des_set_key_skcipher(struct crypto_skcipher tfm, const u8 key,
	66	unsigned int keylen)
	67	{
	68	return des_set_key(crypto_skcipher_tfm(tfm), key, keylen);
	69	}
	70
c5aac2df DM	71	extern void des_sparc64_crypt(const u64 key, const u64 input,
	72	u64 *output);
	73
8f467cf2	74	static void sparc_des_encrypt(struct crypto_tfm tfm, u8 dst, const u8 *src)
c5aac2df DM	75	{
	76	struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
	77	const u64 *K = ctx->encrypt_expkey;
	78
	79	des_sparc64_crypt(K, (const u64 ) src, (u64 ) dst);
	80	}
	81
8f467cf2	82	static void sparc_des_decrypt(struct crypto_tfm tfm, u8 dst, const u8 *src)
c5aac2df DM	83	{
	84	struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
	85	const u64 *K = ctx->decrypt_expkey;
	86
	87	des_sparc64_crypt(K, (const u64 ) src, (u64 ) dst);
	88	}
	89
	90	extern void des_sparc64_load_keys(const u64 *key);
	91
	92	extern void des_sparc64_ecb_crypt(const u64 input, u64 output,
	93	unsigned int len);
	94
cd5d2f84	95	static int __ecb_crypt(struct skcipher_request *req, bool encrypt)
c5aac2df	96	{
cd5d2f84 EB	97	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	98	const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
	99	struct skcipher_walk walk;
	100	unsigned int nbytes;
c5aac2df DM	101	int err;
c5aac2df DM	102
cd5d2f84 EB	103	err = skcipher_walk_virt(&walk, req, true);
	104	if (err)
	105	return err;
c5aac2df DM	106
	107	if (encrypt)
	108	des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
	109	else
	110	des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
cd5d2f84 EB	111	while ((nbytes = walk.nbytes) != 0) {
	112	des_sparc64_ecb_crypt(walk.src.virt.addr, walk.dst.virt.addr,
	113	round_down(nbytes, DES_BLOCK_SIZE));
	114	err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
c5aac2df DM	115	}
	116	fprs_write(0);
	117	return err;
	118	}
	119
cd5d2f84	120	static int ecb_encrypt(struct skcipher_request *req)
c5aac2df	121	{
cd5d2f84	122	return __ecb_crypt(req, true);
c5aac2df DM	123	}
c5aac2df DM	124
cd5d2f84	125	static int ecb_decrypt(struct skcipher_request *req)
c5aac2df	126	{
cd5d2f84	127	return __ecb_crypt(req, false);
c5aac2df DM	128	}
	129
	130	extern void des_sparc64_cbc_encrypt(const u64 input, u64 output,
	131	unsigned int len, u64 *iv);
	132
cd5d2f84 EB	133	extern void des_sparc64_cbc_decrypt(const u64 input, u64 output,
	134	unsigned int len, u64 *iv);
	135
	136	static int __cbc_crypt(struct skcipher_request *req, bool encrypt)
c5aac2df	137	{
cd5d2f84 EB	138	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	139	const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
	140	struct skcipher_walk walk;
	141	unsigned int nbytes;
c5aac2df DM	142	int err;
c5aac2df DM	143
cd5d2f84 EB	144	err = skcipher_walk_virt(&walk, req, true);
	145	if (err)
	146	return err;
c5aac2df	147
cd5d2f84 EB	148	if (encrypt)
	149	des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
	150	else
	151	des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
	152	while ((nbytes = walk.nbytes) != 0) {
	153	if (encrypt)
	154	des_sparc64_cbc_encrypt(walk.src.virt.addr,
	155	walk.dst.virt.addr,
	156	round_down(nbytes,
	157	DES_BLOCK_SIZE),
	158	walk.iv);
	159	else
	160	des_sparc64_cbc_decrypt(walk.src.virt.addr,
	161	walk.dst.virt.addr,
	162	round_down(nbytes,
	163	DES_BLOCK_SIZE),
	164	walk.iv);
	165	err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
c5aac2df DM	166	}
	167	fprs_write(0);
	168	return err;
	169	}
	170
cd5d2f84	171	static int cbc_encrypt(struct skcipher_request *req)
c5aac2df	172	{
cd5d2f84 EB	173	return __cbc_crypt(req, true);
cd5d2f84 EB	174	}
c5aac2df	175
cd5d2f84 EB	176	static int cbc_decrypt(struct skcipher_request *req)
	177	{
	178	return __cbc_crypt(req, false);
c5aac2df DM	179	}
	180
	181	static int des3_ede_set_key(struct crypto_tfm tfm, const u8 key,
	182	unsigned int keylen)
	183	{
	184	struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
c5aac2df DM	185	u64 k1[DES_EXPKEY_WORDS / 2];
	186	u64 k2[DES_EXPKEY_WORDS / 2];
	187	u64 k3[DES_EXPKEY_WORDS / 2];
eee25da5	188	int err;
c5aac2df	189
8f467cf2 AB	190	err = crypto_des3_ede_verify_key(tfm, key);
8f467cf2 AB	191	if (err)
eee25da5	192	return err;
c5aac2df DM	193
	194	des_sparc64_key_expand((const u32 *)key, k1);
	195	key += DES_KEY_SIZE;
	196	des_sparc64_key_expand((const u32 *)key, k2);
	197	key += DES_KEY_SIZE;
	198	des_sparc64_key_expand((const u32 *)key, k3);
	199
	200	memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1));
	201	encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]);
	202	memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
	203	&k3[0], sizeof(k3));
	204
	205	encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]);
	206	memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2],
	207	&k2[0], sizeof(k2));
	208	encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
	209	&k1[0]);
	210
	211	return 0;
	212	}
	213
cd5d2f84 EB	214	static int des3_ede_set_key_skcipher(struct crypto_skcipher tfm, const u8 key,
	215	unsigned int keylen)
	216	{
	217	return des3_ede_set_key(crypto_skcipher_tfm(tfm), key, keylen);
	218	}
	219
c5aac2df DM	220	extern void des3_ede_sparc64_crypt(const u64 key, const u64 input,
	221	u64 *output);
	222
8f467cf2	223	static void sparc_des3_ede_encrypt(struct crypto_tfm tfm, u8 dst, const u8 *src)
c5aac2df DM	224	{
	225	struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
	226	const u64 *K = ctx->encrypt_expkey;
	227
	228	des3_ede_sparc64_crypt(K, (const u64 ) src, (u64 ) dst);
	229	}
	230
8f467cf2	231	static void sparc_des3_ede_decrypt(struct crypto_tfm tfm, u8 dst, const u8 *src)
c5aac2df DM	232	{
	233	struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
	234	const u64 *K = ctx->decrypt_expkey;
	235
	236	des3_ede_sparc64_crypt(K, (const u64 ) src, (u64 ) dst);
	237	}
	238
	239	extern void des3_ede_sparc64_load_keys(const u64 *key);
	240
	241	extern void des3_ede_sparc64_ecb_crypt(const u64 expkey, const u64 input,
	242	u64 *output, unsigned int len);
	243
cd5d2f84	244	static int __ecb3_crypt(struct skcipher_request *req, bool encrypt)
c5aac2df	245	{
cd5d2f84 EB	246	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	247	const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
	248	struct skcipher_walk walk;
c5aac2df	249	const u64 *K;
cd5d2f84	250	unsigned int nbytes;
c5aac2df DM	251	int err;
c5aac2df DM	252
cd5d2f84 EB	253	err = skcipher_walk_virt(&walk, req, true);
	254	if (err)
	255	return err;
c5aac2df DM	256
	257	if (encrypt)
	258	K = &ctx->encrypt_expkey[0];
	259	else
	260	K = &ctx->decrypt_expkey[0];
	261	des3_ede_sparc64_load_keys(K);
cd5d2f84 EB	262	while ((nbytes = walk.nbytes) != 0) {
	263	des3_ede_sparc64_ecb_crypt(K, walk.src.virt.addr,
	264	walk.dst.virt.addr,
	265	round_down(nbytes, DES_BLOCK_SIZE));
	266	err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
c5aac2df DM	267	}
	268	fprs_write(0);
	269	return err;
	270	}
	271
cd5d2f84	272	static int ecb3_encrypt(struct skcipher_request *req)
c5aac2df	273	{
cd5d2f84	274	return __ecb3_crypt(req, true);
c5aac2df DM	275	}
c5aac2df DM	276
cd5d2f84	277	static int ecb3_decrypt(struct skcipher_request *req)
c5aac2df	278	{
cd5d2f84	279	return __ecb3_crypt(req, false);
c5aac2df DM	280	}
	281
	282	extern void des3_ede_sparc64_cbc_encrypt(const u64 expkey, const u64 input,
	283	u64 *output, unsigned int len,
	284	u64 *iv);
	285
c5aac2df DM	286	extern void des3_ede_sparc64_cbc_decrypt(const u64 expkey, const u64 input,
	287	u64 *output, unsigned int len,
	288	u64 *iv);
	289
cd5d2f84	290	static int __cbc3_crypt(struct skcipher_request *req, bool encrypt)
c5aac2df	291	{
cd5d2f84 EB	292	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
	293	const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
	294	struct skcipher_walk walk;
c5aac2df	295	const u64 *K;
cd5d2f84	296	unsigned int nbytes;
c5aac2df DM	297	int err;
c5aac2df DM	298
cd5d2f84 EB	299	err = skcipher_walk_virt(&walk, req, true);
	300	if (err)
	301	return err;
c5aac2df	302
cd5d2f84 EB	303	if (encrypt)
	304	K = &ctx->encrypt_expkey[0];
	305	else
	306	K = &ctx->decrypt_expkey[0];
c5aac2df	307	des3_ede_sparc64_load_keys(K);
cd5d2f84 EB	308	while ((nbytes = walk.nbytes) != 0) {
	309	if (encrypt)
	310	des3_ede_sparc64_cbc_encrypt(K, walk.src.virt.addr,
	311	walk.dst.virt.addr,
	312	round_down(nbytes,
	313	DES_BLOCK_SIZE),
	314	walk.iv);
	315	else
	316	des3_ede_sparc64_cbc_decrypt(K, walk.src.virt.addr,
	317	walk.dst.virt.addr,
	318	round_down(nbytes,
	319	DES_BLOCK_SIZE),
	320	walk.iv);
	321	err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
c5aac2df DM	322	}
	323	fprs_write(0);
	324	return err;
	325	}
	326
cd5d2f84 EB	327	static int cbc3_encrypt(struct skcipher_request *req)
	328	{
	329	return __cbc3_crypt(req, true);
	330	}
	331
	332	static int cbc3_decrypt(struct skcipher_request *req)
	333	{
	334	return __cbc3_crypt(req, false);
	335	}
	336
	337	static struct crypto_alg cipher_algs[] = {
	338	{
	339	.cra_name = "des",
	340	.cra_driver_name = "des-sparc64",
	341	.cra_priority = SPARC_CR_OPCODE_PRIORITY,
	342	.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
	343	.cra_blocksize = DES_BLOCK_SIZE,
	344	.cra_ctxsize = sizeof(struct des_sparc64_ctx),
	345	.cra_alignmask = 7,
	346	.cra_module = THIS_MODULE,
	347	.cra_u = {
	348	.cipher = {
	349	.cia_min_keysize = DES_KEY_SIZE,
	350	.cia_max_keysize = DES_KEY_SIZE,
	351	.cia_setkey = des_set_key,
	352	.cia_encrypt = sparc_des_encrypt,
	353	.cia_decrypt = sparc_des_decrypt
	354	}
c5aac2df	355	}
cd5d2f84 EB	356	}, {
	357	.cra_name = "des3_ede",
	358	.cra_driver_name = "des3_ede-sparc64",
	359	.cra_priority = SPARC_CR_OPCODE_PRIORITY,
	360	.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
	361	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
	362	.cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
	363	.cra_alignmask = 7,
	364	.cra_module = THIS_MODULE,
	365	.cra_u = {
	366	.cipher = {
	367	.cia_min_keysize = DES3_EDE_KEY_SIZE,
	368	.cia_max_keysize = DES3_EDE_KEY_SIZE,
	369	.cia_setkey = des3_ede_set_key,
	370	.cia_encrypt = sparc_des3_ede_encrypt,
	371	.cia_decrypt = sparc_des3_ede_decrypt
	372	}
c5aac2df DM	373	}
c5aac2df DM	374	}
cd5d2f84 EB	375	};
	376
	377	static struct skcipher_alg skcipher_algs[] = {
	378	{
	379	.base.cra_name = "ecb(des)",
	380	.base.cra_driver_name = "ecb-des-sparc64",
	381	.base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
	382	.base.cra_blocksize = DES_BLOCK_SIZE,
	383	.base.cra_ctxsize = sizeof(struct des_sparc64_ctx),
	384	.base.cra_alignmask = 7,
	385	.base.cra_module = THIS_MODULE,
	386	.min_keysize = DES_KEY_SIZE,
	387	.max_keysize = DES_KEY_SIZE,
	388	.setkey = des_set_key_skcipher,
	389	.encrypt = ecb_encrypt,
	390	.decrypt = ecb_decrypt,
	391	}, {
	392	.base.cra_name = "cbc(des)",
	393	.base.cra_driver_name = "cbc-des-sparc64",
	394	.base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
	395	.base.cra_blocksize = DES_BLOCK_SIZE,
	396	.base.cra_ctxsize = sizeof(struct des_sparc64_ctx),
	397	.base.cra_alignmask = 7,
	398	.base.cra_module = THIS_MODULE,
	399	.min_keysize = DES_KEY_SIZE,
	400	.max_keysize = DES_KEY_SIZE,
	401	.ivsize = DES_BLOCK_SIZE,
	402	.setkey = des_set_key_skcipher,
	403	.encrypt = cbc_encrypt,
	404	.decrypt = cbc_decrypt,
	405	}, {
	406	.base.cra_name = "ecb(des3_ede)",
	407	.base.cra_driver_name = "ecb-des3_ede-sparc64",
	408	.base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
	409	.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
	410	.base.cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
	411	.base.cra_alignmask = 7,
	412	.base.cra_module = THIS_MODULE,
	413	.min_keysize = DES3_EDE_KEY_SIZE,
	414	.max_keysize = DES3_EDE_KEY_SIZE,
	415	.setkey = des3_ede_set_key_skcipher,
	416	.encrypt = ecb3_encrypt,
	417	.decrypt = ecb3_decrypt,
	418	}, {
	419	.base.cra_name = "cbc(des3_ede)",
	420	.base.cra_driver_name = "cbc-des3_ede-sparc64",
	421	.base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
	422	.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
	423	.base.cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
	424	.base.cra_alignmask = 7,
	425	.base.cra_module = THIS_MODULE,
	426	.min_keysize = DES3_EDE_KEY_SIZE,
	427	.max_keysize = DES3_EDE_KEY_SIZE,
	428	.ivsize = DES3_EDE_BLOCK_SIZE,
	429	.setkey = des3_ede_set_key_skcipher,
	430	.encrypt = cbc3_encrypt,
	431	.decrypt = cbc3_decrypt,
	432	}
	433	};
c5aac2df DM	434
	435	static bool __init sparc64_has_des_opcode(void)
	436	{
	437	unsigned long cfr;
	438
	439	if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
	440	return false;
	441
	442	__asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
	443	if (!(cfr & CFR_DES))
	444	return false;
	445
	446	return true;
	447	}
	448
	449	static int __init des_sparc64_mod_init(void)
	450	{
cd5d2f84 EB	451	int err;
	452
	453	if (!sparc64_has_des_opcode()) {
	454	pr_info("sparc64 des opcodes not available.\n");
	455	return -ENODEV;
c5aac2df	456	}
cd5d2f84 EB	457	pr_info("Using sparc64 des opcodes optimized DES implementation\n");
	458	err = crypto_register_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
	459	if (err)
	460	return err;
	461	err = crypto_register_skciphers(skcipher_algs,
	462	ARRAY_SIZE(skcipher_algs));
	463	if (err)
	464	crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
	465	return err;
c5aac2df DM	466	}
	467
	468	static void __exit des_sparc64_mod_fini(void)
	469	{
cd5d2f84 EB	470	crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
cd5d2f84 EB	471	crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
c5aac2df DM	472	}
	473
	474	module_init(des_sparc64_mod_init);
	475	module_exit(des_sparc64_mod_fini);
	476
	477	MODULE_LICENSE("GPL");
	478	MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");
	479
5d26a105	480	MODULE_ALIAS_CRYPTO("des");
5001323b	481	MODULE_ALIAS_CRYPTO("des3_ede");
226f7cea DM	482
226f7cea DM	483	#include "crop_devid.c"