arch/x86/crypto/serpent_avx_glue.c

   1 /*
   2  * Glue Code for AVX assembler versions of Serpent Cipher
   3  *
   4  * Copyright (C) 2012 Johannes Goetzfried
   5  *     <[email protected]>
   6  *
   7  * Copyright © 2011-2013 Jussi Kivilinna <[email protected]>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  22  * USA
  23  *
  24  */
  25
  26 #include <linux/module.h>
  27 #include <linux/types.h>
  28 #include <linux/crypto.h>
  29 #include <linux/err.h>
  30 #include <crypto/algapi.h>
  31 #include <crypto/internal/simd.h>
  32 #include <crypto/serpent.h>
  33 #include <crypto/xts.h>
  34 #include <asm/crypto/glue_helper.h>
  35 #include <asm/crypto/serpent-avx.h>
  36
  37 /* 8-way parallel cipher functions */
  38 asmlinkage void serpent_ecb_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst,
  39                                          const u8 *src);
  40 EXPORT_SYMBOL_GPL(serpent_ecb_enc_8way_avx);
  41
  42 asmlinkage void serpent_ecb_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
  43                                          const u8 *src);
  44 EXPORT_SYMBOL_GPL(serpent_ecb_dec_8way_avx);
  45
  46 asmlinkage void serpent_cbc_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
  47                                          const u8 *src);
  48 EXPORT_SYMBOL_GPL(serpent_cbc_dec_8way_avx);
  49
  50 asmlinkage void serpent_ctr_8way_avx(struct serpent_ctx *ctx, u8 *dst,
  51                                      const u8 *src, le128 *iv);
  52 EXPORT_SYMBOL_GPL(serpent_ctr_8way_avx);
  53
  54 asmlinkage void serpent_xts_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst,
  55                                          const u8 *src, le128 *iv);
  56 EXPORT_SYMBOL_GPL(serpent_xts_enc_8way_avx);
  57
  58 asmlinkage void serpent_xts_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
  59                                          const u8 *src, le128 *iv);
  60 EXPORT_SYMBOL_GPL(serpent_xts_dec_8way_avx);
  61
  62 void __serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
  63 {
  64         be128 ctrblk;
  65
  66         le128_to_be128(&ctrblk, iv);
  67         le128_inc(iv);
  68
  69         __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
  70         u128_xor(dst, src, (u128 *)&ctrblk);
  71 }
  72 EXPORT_SYMBOL_GPL(__serpent_crypt_ctr);
  73
  74 void serpent_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
  75 {
  76         glue_xts_crypt_128bit_one(ctx, dst, src, iv,
  77                                   GLUE_FUNC_CAST(__serpent_encrypt));
  78 }
  79 EXPORT_SYMBOL_GPL(serpent_xts_enc);
  80
  81 void serpent_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
  82 {
  83         glue_xts_crypt_128bit_one(ctx, dst, src, iv,
  84                                   GLUE_FUNC_CAST(__serpent_decrypt));
  85 }
  86 EXPORT_SYMBOL_GPL(serpent_xts_dec);
  87
  88 static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
  89                                    const u8 *key, unsigned int keylen)
  90 {
  91         return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen);
  92 }
  93
  94 int xts_serpent_setkey(struct crypto_skcipher *tfm, const u8 *key,
  95                        unsigned int keylen)
  96 {
  97         struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
  98         int err;
  99
 100         err = xts_verify_key(tfm, key, keylen);
 101         if (err)
 102                 return err;
 103
 104         /* first half of xts-key is for crypt */
 105         err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2);
 106         if (err)
 107                 return err;
 108
 109         /* second half of xts-key is for tweak */
 110         return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
 111 }
 112 EXPORT_SYMBOL_GPL(xts_serpent_setkey);
 113
 114 static const struct common_glue_ctx serpent_enc = {
 115         .num_funcs = 2,
 116         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 117
 118         .funcs = { {
 119                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 120                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) }
 121         }, {
 122                 .num_blocks = 1,
 123                 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
 124         } }
 125 };
 126
 127 static const struct common_glue_ctx serpent_ctr = {
 128         .num_funcs = 2,
 129         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 130
 131         .funcs = { {
 132                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 133                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) }
 134         }, {
 135                 .num_blocks = 1,
 136                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) }
 137         } }
 138 };
 139
 140 static const struct common_glue_ctx serpent_enc_xts = {
 141         .num_funcs = 2,
 142         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 143
 144         .funcs = { {
 145                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 146                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) }
 147         }, {
 148                 .num_blocks = 1,
 149                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) }
 150         } }
 151 };
 152
 153 static const struct common_glue_ctx serpent_dec = {
 154         .num_funcs = 2,
 155         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 156
 157         .funcs = { {
 158                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 159                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) }
 160         }, {
 161                 .num_blocks = 1,
 162                 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
 163         } }
 164 };
 165
 166 static const struct common_glue_ctx serpent_dec_cbc = {
 167         .num_funcs = 2,
 168         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 169
 170         .funcs = { {
 171                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 172                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) }
 173         }, {
 174                 .num_blocks = 1,
 175                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
 176         } }
 177 };
 178
 179 static const struct common_glue_ctx serpent_dec_xts = {
 180         .num_funcs = 2,
 181         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 182
 183         .funcs = { {
 184                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 185                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) }
 186         }, {
 187                 .num_blocks = 1,
 188                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) }
 189         } }
 190 };
 191
 192 static int ecb_encrypt(struct skcipher_request *req)
 193 {
 194         return glue_ecb_req_128bit(&serpent_enc, req);
 195 }
 196
 197 static int ecb_decrypt(struct skcipher_request *req)
 198 {
 199         return glue_ecb_req_128bit(&serpent_dec, req);
 200 }
 201
 202 static int cbc_encrypt(struct skcipher_request *req)
 203 {
 204         return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(__serpent_encrypt),
 205                                            req);
 206 }
 207
 208 static int cbc_decrypt(struct skcipher_request *req)
 209 {
 210         return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req);
 211 }
 212
 213 static int ctr_crypt(struct skcipher_request *req)
 214 {
 215         return glue_ctr_req_128bit(&serpent_ctr, req);
 216 }
 217
 218 static int xts_encrypt(struct skcipher_request *req)
 219 {
 220         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 221         struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 222
 223         return glue_xts_req_128bit(&serpent_enc_xts, req,
 224                                    XTS_TWEAK_CAST(__serpent_encrypt),
 225                                    &ctx->tweak_ctx, &ctx->crypt_ctx);
 226 }
 227
 228 static int xts_decrypt(struct skcipher_request *req)
 229 {
 230         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 231         struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 232
 233         return glue_xts_req_128bit(&serpent_dec_xts, req,
 234                                    XTS_TWEAK_CAST(__serpent_encrypt),
 235                                    &ctx->tweak_ctx, &ctx->crypt_ctx);
 236 }
 237
 238 static struct skcipher_alg serpent_algs[] = {
 239         {
 240                 .base.cra_name          = "__ecb(serpent)",
 241                 .base.cra_driver_name   = "__ecb-serpent-avx",
 242                 .base.cra_priority      = 500,
 243                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 244                 .base.cra_blocksize     = SERPENT_BLOCK_SIZE,
 245                 .base.cra_ctxsize       = sizeof(struct serpent_ctx),
 246                 .base.cra_module        = THIS_MODULE,
 247                 .min_keysize            = SERPENT_MIN_KEY_SIZE,
 248                 .max_keysize            = SERPENT_MAX_KEY_SIZE,
 249                 .setkey                 = serpent_setkey_skcipher,
 250                 .encrypt                = ecb_encrypt,
 251                 .decrypt                = ecb_decrypt,
 252         }, {
 253                 .base.cra_name          = "__cbc(serpent)",
 254                 .base.cra_driver_name   = "__cbc-serpent-avx",
 255                 .base.cra_priority      = 500,
 256                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 257                 .base.cra_blocksize     = SERPENT_BLOCK_SIZE,
 258                 .base.cra_ctxsize       = sizeof(struct serpent_ctx),
 259                 .base.cra_module        = THIS_MODULE,
 260                 .min_keysize            = SERPENT_MIN_KEY_SIZE,
 261                 .max_keysize            = SERPENT_MAX_KEY_SIZE,
 262                 .ivsize                 = SERPENT_BLOCK_SIZE,
 263                 .setkey                 = serpent_setkey_skcipher,
 264                 .encrypt                = cbc_encrypt,
 265                 .decrypt                = cbc_decrypt,
 266         }, {
 267                 .base.cra_name          = "__ctr(serpent)",
 268                 .base.cra_driver_name   = "__ctr-serpent-avx",
 269                 .base.cra_priority      = 500,
 270                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 271                 .base.cra_blocksize     = 1,
 272                 .base.cra_ctxsize       = sizeof(struct serpent_ctx),
 273                 .base.cra_module        = THIS_MODULE,
 274                 .min_keysize            = SERPENT_MIN_KEY_SIZE,
 275                 .max_keysize            = SERPENT_MAX_KEY_SIZE,
 276                 .ivsize                 = SERPENT_BLOCK_SIZE,
 277                 .chunksize              = SERPENT_BLOCK_SIZE,
 278                 .setkey                 = serpent_setkey_skcipher,
 279                 .encrypt                = ctr_crypt,
 280                 .decrypt                = ctr_crypt,
 281         }, {
 282                 .base.cra_name          = "__xts(serpent)",
 283                 .base.cra_driver_name   = "__xts-serpent-avx",
 284                 .base.cra_priority      = 500,
 285                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 286                 .base.cra_blocksize     = SERPENT_BLOCK_SIZE,
 287                 .base.cra_ctxsize       = sizeof(struct serpent_xts_ctx),
 288                 .base.cra_module        = THIS_MODULE,
 289                 .min_keysize            = 2 * SERPENT_MIN_KEY_SIZE,
 290                 .max_keysize            = 2 * SERPENT_MAX_KEY_SIZE,
 291                 .ivsize                 = SERPENT_BLOCK_SIZE,
 292                 .setkey                 = xts_serpent_setkey,
 293                 .encrypt                = xts_encrypt,
 294                 .decrypt                = xts_decrypt,
 295         },
 296 };
 297
 298 static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];
 299
 300 static int __init serpent_init(void)
 301 {
 302         const char *feature_name;
 303
 304         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
 305                                 &feature_name)) {
 306                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
 307                 return -ENODEV;
 308         }
 309
 310         return simd_register_skciphers_compat(serpent_algs,
 311                                               ARRAY_SIZE(serpent_algs),
 312                                               serpent_simd_algs);
 313 }
 314
 315 static void __exit serpent_exit(void)
 316 {
 317         simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
 318                                   serpent_simd_algs);
 319 }
 320
 321 module_init(serpent_init);
 322 module_exit(serpent_exit);
 323
 324 MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX optimized");
 325 MODULE_LICENSE("GPL");
 326 MODULE_ALIAS_CRYPTO("serpent");