arch/x86/crypto/sha256_ssse3_glue.c

   1 /*
   2  * Cryptographic API.
   3  *
   4  * Glue code for the SHA256 Secure Hash Algorithm assembler
   5  * implementation using supplemental SSE3 / AVX / AVX2 instructions.
   6  *
   7  * This file is based on sha256_generic.c
   8  *
   9  * Copyright (C) 2013 Intel Corporation.
  10  *
  11  * Author:
  12  *     Tim Chen <[email protected]>
  13  *
  14  * This program is free software; you can redistribute it and/or modify it
  15  * under the terms of the GNU General Public License as published by the Free
  16  * Software Foundation; either version 2 of the License, or (at your option)
  17  * any later version.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  20  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  22  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  23  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  24  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  25  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  26  * SOFTWARE.
  27  */
  28
  29
  30 #define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
  31
  32 #include <crypto/internal/hash.h>
  33 #include <crypto/internal/simd.h>
  34 #include <linux/init.h>
  35 #include <linux/module.h>
  36 #include <linux/mm.h>
  37 #include <linux/cryptohash.h>
  38 #include <linux/types.h>
  39 #include <crypto/sha.h>
  40 #include <crypto/sha256_base.h>
  41 #include <linux/string.h>
  42 #include <asm/simd.h>
  43
  44 asmlinkage void sha256_transform_ssse3(struct sha256_state *state,
  45                                        const u8 *data, int blocks);
  46
  47 static int _sha256_update(struct shash_desc *desc, const u8 *data,
  48                           unsigned int len, sha256_block_fn *sha256_xform)
  49 {
  50         struct sha256_state *sctx = shash_desc_ctx(desc);
  51
  52         if (!crypto_simd_usable() ||
  53             (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
  54                 return crypto_sha256_update(desc, data, len);
  55
  56         /*
  57          * Make sure struct sha256_state begins directly with the SHA256
  58          * 256-bit internal state, as this is what the asm functions expect.
  59          */
  60         BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0);
  61
  62         kernel_fpu_begin();
  63         sha256_base_do_update(desc, data, len, sha256_xform);
  64         kernel_fpu_end();
  65
  66         return 0;
  67 }
  68
  69 static int sha256_finup(struct shash_desc *desc, const u8 *data,
  70               unsigned int len, u8 *out, sha256_block_fn *sha256_xform)
  71 {
  72         if (!crypto_simd_usable())
  73                 return crypto_sha256_finup(desc, data, len, out);
  74
  75         kernel_fpu_begin();
  76         if (len)
  77                 sha256_base_do_update(desc, data, len, sha256_xform);
  78         sha256_base_do_finalize(desc, sha256_xform);
  79         kernel_fpu_end();
  80
  81         return sha256_base_finish(desc, out);
  82 }
  83
  84 static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data,
  85                          unsigned int len)
  86 {
  87         return _sha256_update(desc, data, len, sha256_transform_ssse3);
  88 }
  89
  90 static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data,
  91               unsigned int len, u8 *out)
  92 {
  93         return sha256_finup(desc, data, len, out, sha256_transform_ssse3);
  94 }
  95
  96 /* Add padding and return the message digest. */
  97 static int sha256_ssse3_final(struct shash_desc *desc, u8 *out)
  98 {
  99         return sha256_ssse3_finup(desc, NULL, 0, out);
 100 }
 101
 102 static struct shash_alg sha256_ssse3_algs[] = { {
 103         .digestsize     =       SHA256_DIGEST_SIZE,
 104         .init           =       sha256_base_init,
 105         .update         =       sha256_ssse3_update,
 106         .final          =       sha256_ssse3_final,
 107         .finup          =       sha256_ssse3_finup,
 108         .descsize       =       sizeof(struct sha256_state),
 109         .base           =       {
 110                 .cra_name       =       "sha256",
 111                 .cra_driver_name =      "sha256-ssse3",
 112                 .cra_priority   =       150,
 113                 .cra_blocksize  =       SHA256_BLOCK_SIZE,
 114                 .cra_module     =       THIS_MODULE,
 115         }
 116 }, {
 117         .digestsize     =       SHA224_DIGEST_SIZE,
 118         .init           =       sha224_base_init,
 119         .update         =       sha256_ssse3_update,
 120         .final          =       sha256_ssse3_final,
 121         .finup          =       sha256_ssse3_finup,
 122         .descsize       =       sizeof(struct sha256_state),
 123         .base           =       {
 124                 .cra_name       =       "sha224",
 125                 .cra_driver_name =      "sha224-ssse3",
 126                 .cra_priority   =       150,
 127                 .cra_blocksize  =       SHA224_BLOCK_SIZE,
 128                 .cra_module     =       THIS_MODULE,
 129         }
 130 } };
 131
 132 static int register_sha256_ssse3(void)
 133 {
 134         if (boot_cpu_has(X86_FEATURE_SSSE3))
 135                 return crypto_register_shashes(sha256_ssse3_algs,
 136                                 ARRAY_SIZE(sha256_ssse3_algs));
 137         return 0;
 138 }
 139
 140 static void unregister_sha256_ssse3(void)
 141 {
 142         if (boot_cpu_has(X86_FEATURE_SSSE3))
 143                 crypto_unregister_shashes(sha256_ssse3_algs,
 144                                 ARRAY_SIZE(sha256_ssse3_algs));
 145 }
 146
 147 #ifdef CONFIG_AS_AVX
 148 asmlinkage void sha256_transform_avx(struct sha256_state *state,
 149                                      const u8 *data, int blocks);
 150
 151 static int sha256_avx_update(struct shash_desc *desc, const u8 *data,
 152                          unsigned int len)
 153 {
 154         return _sha256_update(desc, data, len, sha256_transform_avx);
 155 }
 156
 157 static int sha256_avx_finup(struct shash_desc *desc, const u8 *data,
 158                       unsigned int len, u8 *out)
 159 {
 160         return sha256_finup(desc, data, len, out, sha256_transform_avx);
 161 }
 162
 163 static int sha256_avx_final(struct shash_desc *desc, u8 *out)
 164 {
 165         return sha256_avx_finup(desc, NULL, 0, out);
 166 }
 167
 168 static struct shash_alg sha256_avx_algs[] = { {
 169         .digestsize     =       SHA256_DIGEST_SIZE,
 170         .init           =       sha256_base_init,
 171         .update         =       sha256_avx_update,
 172         .final          =       sha256_avx_final,
 173         .finup          =       sha256_avx_finup,
 174         .descsize       =       sizeof(struct sha256_state),
 175         .base           =       {
 176                 .cra_name       =       "sha256",
 177                 .cra_driver_name =      "sha256-avx",
 178                 .cra_priority   =       160,
 179                 .cra_blocksize  =       SHA256_BLOCK_SIZE,
 180                 .cra_module     =       THIS_MODULE,
 181         }
 182 }, {
 183         .digestsize     =       SHA224_DIGEST_SIZE,
 184         .init           =       sha224_base_init,
 185         .update         =       sha256_avx_update,
 186         .final          =       sha256_avx_final,
 187         .finup          =       sha256_avx_finup,
 188         .descsize       =       sizeof(struct sha256_state),
 189         .base           =       {
 190                 .cra_name       =       "sha224",
 191                 .cra_driver_name =      "sha224-avx",
 192                 .cra_priority   =       160,
 193                 .cra_blocksize  =       SHA224_BLOCK_SIZE,
 194                 .cra_module     =       THIS_MODULE,
 195         }
 196 } };
 197
 198 static bool avx_usable(void)
 199 {
 200         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
 201                 if (boot_cpu_has(X86_FEATURE_AVX))
 202                         pr_info("AVX detected but unusable.\n");
 203                 return false;
 204         }
 205
 206         return true;
 207 }
 208
 209 static int register_sha256_avx(void)
 210 {
 211         if (avx_usable())
 212                 return crypto_register_shashes(sha256_avx_algs,
 213                                 ARRAY_SIZE(sha256_avx_algs));
 214         return 0;
 215 }
 216
 217 static void unregister_sha256_avx(void)
 218 {
 219         if (avx_usable())
 220                 crypto_unregister_shashes(sha256_avx_algs,
 221                                 ARRAY_SIZE(sha256_avx_algs));
 222 }
 223
 224 #else
 225 static inline int register_sha256_avx(void) { return 0; }
 226 static inline void unregister_sha256_avx(void) { }
 227 #endif
 228
 229 #if defined(CONFIG_AS_AVX2) && defined(CONFIG_AS_AVX)
 230 asmlinkage void sha256_transform_rorx(struct sha256_state *state,
 231                                       const u8 *data, int blocks);
 232
 233 static int sha256_avx2_update(struct shash_desc *desc, const u8 *data,
 234                          unsigned int len)
 235 {
 236         return _sha256_update(desc, data, len, sha256_transform_rorx);
 237 }
 238
 239 static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data,
 240                       unsigned int len, u8 *out)
 241 {
 242         return sha256_finup(desc, data, len, out, sha256_transform_rorx);
 243 }
 244
 245 static int sha256_avx2_final(struct shash_desc *desc, u8 *out)
 246 {
 247         return sha256_avx2_finup(desc, NULL, 0, out);
 248 }
 249
 250 static struct shash_alg sha256_avx2_algs[] = { {
 251         .digestsize     =       SHA256_DIGEST_SIZE,
 252         .init           =       sha256_base_init,
 253         .update         =       sha256_avx2_update,
 254         .final          =       sha256_avx2_final,
 255         .finup          =       sha256_avx2_finup,
 256         .descsize       =       sizeof(struct sha256_state),
 257         .base           =       {
 258                 .cra_name       =       "sha256",
 259                 .cra_driver_name =      "sha256-avx2",
 260                 .cra_priority   =       170,
 261                 .cra_blocksize  =       SHA256_BLOCK_SIZE,
 262                 .cra_module     =       THIS_MODULE,
 263         }
 264 }, {
 265         .digestsize     =       SHA224_DIGEST_SIZE,
 266         .init           =       sha224_base_init,
 267         .update         =       sha256_avx2_update,
 268         .final          =       sha256_avx2_final,
 269         .finup          =       sha256_avx2_finup,
 270         .descsize       =       sizeof(struct sha256_state),
 271         .base           =       {
 272                 .cra_name       =       "sha224",
 273                 .cra_driver_name =      "sha224-avx2",
 274                 .cra_priority   =       170,
 275                 .cra_blocksize  =       SHA224_BLOCK_SIZE,
 276                 .cra_module     =       THIS_MODULE,
 277         }
 278 } };
 279
 280 static bool avx2_usable(void)
 281 {
 282         if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
 283                     boot_cpu_has(X86_FEATURE_BMI2))
 284                 return true;
 285
 286         return false;
 287 }
 288
 289 static int register_sha256_avx2(void)
 290 {
 291         if (avx2_usable())
 292                 return crypto_register_shashes(sha256_avx2_algs,
 293                                 ARRAY_SIZE(sha256_avx2_algs));
 294         return 0;
 295 }
 296
 297 static void unregister_sha256_avx2(void)
 298 {
 299         if (avx2_usable())
 300                 crypto_unregister_shashes(sha256_avx2_algs,
 301                                 ARRAY_SIZE(sha256_avx2_algs));
 302 }
 303
 304 #else
 305 static inline int register_sha256_avx2(void) { return 0; }
 306 static inline void unregister_sha256_avx2(void) { }
 307 #endif
 308
 309 #ifdef CONFIG_AS_SHA256_NI
 310 asmlinkage void sha256_ni_transform(struct sha256_state *digest,
 311                                     const u8 *data, int rounds);
 312
 313 static int sha256_ni_update(struct shash_desc *desc, const u8 *data,
 314                          unsigned int len)
 315 {
 316         return _sha256_update(desc, data, len, sha256_ni_transform);
 317 }
 318
 319 static int sha256_ni_finup(struct shash_desc *desc, const u8 *data,
 320                       unsigned int len, u8 *out)
 321 {
 322         return sha256_finup(desc, data, len, out, sha256_ni_transform);
 323 }
 324
 325 static int sha256_ni_final(struct shash_desc *desc, u8 *out)
 326 {
 327         return sha256_ni_finup(desc, NULL, 0, out);
 328 }
 329
 330 static struct shash_alg sha256_ni_algs[] = { {
 331         .digestsize     =       SHA256_DIGEST_SIZE,
 332         .init           =       sha256_base_init,
 333         .update         =       sha256_ni_update,
 334         .final          =       sha256_ni_final,
 335         .finup          =       sha256_ni_finup,
 336         .descsize       =       sizeof(struct sha256_state),
 337         .base           =       {
 338                 .cra_name       =       "sha256",
 339                 .cra_driver_name =      "sha256-ni",
 340                 .cra_priority   =       250,
 341                 .cra_blocksize  =       SHA256_BLOCK_SIZE,
 342                 .cra_module     =       THIS_MODULE,
 343         }
 344 }, {
 345         .digestsize     =       SHA224_DIGEST_SIZE,
 346         .init           =       sha224_base_init,
 347         .update         =       sha256_ni_update,
 348         .final          =       sha256_ni_final,
 349         .finup          =       sha256_ni_finup,
 350         .descsize       =       sizeof(struct sha256_state),
 351         .base           =       {
 352                 .cra_name       =       "sha224",
 353                 .cra_driver_name =      "sha224-ni",
 354                 .cra_priority   =       250,
 355                 .cra_blocksize  =       SHA224_BLOCK_SIZE,
 356                 .cra_module     =       THIS_MODULE,
 357         }
 358 } };
 359
 360 static int register_sha256_ni(void)
 361 {
 362         if (boot_cpu_has(X86_FEATURE_SHA_NI))
 363                 return crypto_register_shashes(sha256_ni_algs,
 364                                 ARRAY_SIZE(sha256_ni_algs));
 365         return 0;
 366 }
 367
 368 static void unregister_sha256_ni(void)
 369 {
 370         if (boot_cpu_has(X86_FEATURE_SHA_NI))
 371                 crypto_unregister_shashes(sha256_ni_algs,
 372                                 ARRAY_SIZE(sha256_ni_algs));
 373 }
 374
 375 #else
 376 static inline int register_sha256_ni(void) { return 0; }
 377 static inline void unregister_sha256_ni(void) { }
 378 #endif
 379
 380 static int __init sha256_ssse3_mod_init(void)
 381 {
 382         if (register_sha256_ssse3())
 383                 goto fail;
 384
 385         if (register_sha256_avx()) {
 386                 unregister_sha256_ssse3();
 387                 goto fail;
 388         }
 389
 390         if (register_sha256_avx2()) {
 391                 unregister_sha256_avx();
 392                 unregister_sha256_ssse3();
 393                 goto fail;
 394         }
 395
 396         if (register_sha256_ni()) {
 397                 unregister_sha256_avx2();
 398                 unregister_sha256_avx();
 399                 unregister_sha256_ssse3();
 400                 goto fail;
 401         }
 402
 403         return 0;
 404 fail:
 405         return -ENODEV;
 406 }
 407
 408 static void __exit sha256_ssse3_mod_fini(void)
 409 {
 410         unregister_sha256_ni();
 411         unregister_sha256_avx2();
 412         unregister_sha256_avx();
 413         unregister_sha256_ssse3();
 414 }
 415
 416 module_init(sha256_ssse3_mod_init);
 417 module_exit(sha256_ssse3_mod_fini);
 418
 419 MODULE_LICENSE("GPL");
 420 MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");
 421
 422 MODULE_ALIAS_CRYPTO("sha256");
 423 MODULE_ALIAS_CRYPTO("sha256-ssse3");
 424 MODULE_ALIAS_CRYPTO("sha256-avx");
 425 MODULE_ALIAS_CRYPTO("sha256-avx2");
 426 MODULE_ALIAS_CRYPTO("sha224");
 427 MODULE_ALIAS_CRYPTO("sha224-ssse3");
 428 MODULE_ALIAS_CRYPTO("sha224-avx");
 429 MODULE_ALIAS_CRYPTO("sha224-avx2");
 430 #ifdef CONFIG_AS_SHA256_NI
 431 MODULE_ALIAS_CRYPTO("sha256-ni");
 432 MODULE_ALIAS_CRYPTO("sha224-ni");
 433 #endif