arch/x86/crypto/twofish_glue_3way.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Glue Code for 3-way parallel assembler optimized version of Twofish
   4  *
   5  * Copyright (c) 2011 Jussi Kivilinna <[email protected]>
   6  */
   7
   8 #include <crypto/algapi.h>
   9 #include <crypto/twofish.h>
  10 #include <linux/crypto.h>
  11 #include <linux/init.h>
  12 #include <linux/module.h>
  13 #include <linux/types.h>
  14
  15 #include "twofish.h"
  16 #include "ecb_cbc_helpers.h"
  17
  18 EXPORT_SYMBOL_GPL(__twofish_enc_blk_3way);
  19 EXPORT_SYMBOL_GPL(twofish_dec_blk_3way);
  20
  21 static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
  22                                    const u8 *key, unsigned int keylen)
  23 {
  24         return twofish_setkey(&tfm->base, key, keylen);
  25 }
  26
  27 static inline void twofish_enc_blk_3way(const void *ctx, u8 *dst, const u8 *src)
  28 {
  29         __twofish_enc_blk_3way(ctx, dst, src, false);
  30 }
  31
  32 void twofish_dec_blk_cbc_3way(const void *ctx, u8 *dst, const u8 *src)
  33 {
  34         u8 buf[2][TF_BLOCK_SIZE];
  35         const u8 *s = src;
  36
  37         if (dst == src)
  38                 s = memcpy(buf, src, sizeof(buf));
  39         twofish_dec_blk_3way(ctx, dst, src);
  40         crypto_xor(dst + TF_BLOCK_SIZE, s, sizeof(buf));
  41
  42 }
  43 EXPORT_SYMBOL_GPL(twofish_dec_blk_cbc_3way);
  44
  45 static int ecb_encrypt(struct skcipher_request *req)
  46 {
  47         ECB_WALK_START(req, TF_BLOCK_SIZE, -1);
  48         ECB_BLOCK(3, twofish_enc_blk_3way);
  49         ECB_BLOCK(1, twofish_enc_blk);
  50         ECB_WALK_END();
  51 }
  52
  53 static int ecb_decrypt(struct skcipher_request *req)
  54 {
  55         ECB_WALK_START(req, TF_BLOCK_SIZE, -1);
  56         ECB_BLOCK(3, twofish_dec_blk_3way);
  57         ECB_BLOCK(1, twofish_dec_blk);
  58         ECB_WALK_END();
  59 }
  60
  61 static int cbc_encrypt(struct skcipher_request *req)
  62 {
  63         CBC_WALK_START(req, TF_BLOCK_SIZE, -1);
  64         CBC_ENC_BLOCK(twofish_enc_blk);
  65         CBC_WALK_END();
  66 }
  67
  68 static int cbc_decrypt(struct skcipher_request *req)
  69 {
  70         CBC_WALK_START(req, TF_BLOCK_SIZE, -1);
  71         CBC_DEC_BLOCK(3, twofish_dec_blk_cbc_3way);
  72         CBC_DEC_BLOCK(1, twofish_dec_blk);
  73         CBC_WALK_END();
  74 }
  75
  76 static struct skcipher_alg tf_skciphers[] = {
  77         {
  78                 .base.cra_name          = "ecb(twofish)",
  79                 .base.cra_driver_name   = "ecb-twofish-3way",
  80                 .base.cra_priority      = 300,
  81                 .base.cra_blocksize     = TF_BLOCK_SIZE,
  82                 .base.cra_ctxsize       = sizeof(struct twofish_ctx),
  83                 .base.cra_module        = THIS_MODULE,
  84                 .min_keysize            = TF_MIN_KEY_SIZE,
  85                 .max_keysize            = TF_MAX_KEY_SIZE,
  86                 .setkey                 = twofish_setkey_skcipher,
  87                 .encrypt                = ecb_encrypt,
  88                 .decrypt                = ecb_decrypt,
  89         }, {
  90                 .base.cra_name          = "cbc(twofish)",
  91                 .base.cra_driver_name   = "cbc-twofish-3way",
  92                 .base.cra_priority      = 300,
  93                 .base.cra_blocksize     = TF_BLOCK_SIZE,
  94                 .base.cra_ctxsize       = sizeof(struct twofish_ctx),
  95                 .base.cra_module        = THIS_MODULE,
  96                 .min_keysize            = TF_MIN_KEY_SIZE,
  97                 .max_keysize            = TF_MAX_KEY_SIZE,
  98                 .ivsize                 = TF_BLOCK_SIZE,
  99                 .setkey                 = twofish_setkey_skcipher,
 100                 .encrypt                = cbc_encrypt,
 101                 .decrypt                = cbc_decrypt,
 102         },
 103 };
 104
 105 static bool is_blacklisted_cpu(void)
 106 {
 107         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
 108                 return false;
 109
 110         if (boot_cpu_data.x86 == 0x06 &&
 111                 (boot_cpu_data.x86_model == 0x1c ||
 112                  boot_cpu_data.x86_model == 0x26 ||
 113                  boot_cpu_data.x86_model == 0x36)) {
 114                 /*
 115                  * On Atom, twofish-3way is slower than original assembler
 116                  * implementation. Twofish-3way trades off some performance in
 117                  * storing blocks in 64bit registers to allow three blocks to
 118                  * be processed parallel. Parallel operation then allows gaining
 119                  * more performance than was trade off, on out-of-order CPUs.
 120                  * However Atom does not benefit from this parallelism and
 121                  * should be blacklisted.
 122                  */
 123                 return true;
 124         }
 125
 126         if (boot_cpu_data.x86 == 0x0f) {
 127                 /*
 128                  * On Pentium 4, twofish-3way is slower than original assembler
 129                  * implementation because excessive uses of 64bit rotate and
 130                  * left-shifts (which are really slow on P4) needed to store and
 131                  * handle 128bit block in two 64bit registers.
 132                  */
 133                 return true;
 134         }
 135
 136         return false;
 137 }
 138
 139 static int force;
 140 module_param(force, int, 0);
 141 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");
 142
 143 static int __init init(void)
 144 {
 145         if (!force && is_blacklisted_cpu()) {
 146                 printk(KERN_INFO
 147                         "twofish-x86_64-3way: performance on this CPU "
 148                         "would be suboptimal: disabling "
 149                         "twofish-x86_64-3way.\n");
 150                 return -ENODEV;
 151         }
 152
 153         return crypto_register_skciphers(tf_skciphers,
 154                                          ARRAY_SIZE(tf_skciphers));
 155 }
 156
 157 static void __exit fini(void)
 158 {
 159         crypto_unregister_skciphers(tf_skciphers, ARRAY_SIZE(tf_skciphers));
 160 }
 161
 162 module_init(init);
 163 module_exit(fini);
 164
 165 MODULE_LICENSE("GPL");
 166 MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized");
 167 MODULE_ALIAS_CRYPTO("twofish");
 168 MODULE_ALIAS_CRYPTO("twofish-asm");