arch/arm/crypto/ghash-ce-glue.c

   1 /*
   2  * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions.
   3  *
   4  * Copyright (C) 2015 Linaro Ltd. <[email protected]>
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation.
   9  */
  10
  11 #include <asm/hwcap.h>
  12 #include <asm/neon.h>
  13 #include <asm/simd.h>
  14 #include <asm/unaligned.h>
  15 #include <crypto/cryptd.h>
  16 #include <crypto/internal/hash.h>
  17 #include <crypto/gf128mul.h>
  18 #include <linux/crypto.h>
  19 #include <linux/module.h>
  20
  21 MODULE_DESCRIPTION("GHASH secure hash using ARMv8 Crypto Extensions");
  22 MODULE_AUTHOR("Ard Biesheuvel <[email protected]>");
  23 MODULE_LICENSE("GPL v2");
  24
  25 #define GHASH_BLOCK_SIZE        16
  26 #define GHASH_DIGEST_SIZE       16
  27
  28 struct ghash_key {
  29         u64     a;
  30         u64     b;
  31 };
  32
  33 struct ghash_desc_ctx {
  34         u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
  35         u8 buf[GHASH_BLOCK_SIZE];
  36         u32 count;
  37 };
  38
  39 struct ghash_async_ctx {
  40         struct cryptd_ahash *cryptd_tfm;
  41 };
  42
  43 asmlinkage void pmull_ghash_update(int blocks, u64 dg[], const char *src,
  44                                    struct ghash_key const *k, const char *head);
  45
  46 static int ghash_init(struct shash_desc *desc)
  47 {
  48         struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
  49
  50         *ctx = (struct ghash_desc_ctx){};
  51         return 0;
  52 }
  53
  54 static int ghash_update(struct shash_desc *desc, const u8 *src,
  55                         unsigned int len)
  56 {
  57         struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
  58         unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
  59
  60         ctx->count += len;
  61
  62         if ((partial + len) >= GHASH_BLOCK_SIZE) {
  63                 struct ghash_key *key = crypto_shash_ctx(desc->tfm);
  64                 int blocks;
  65
  66                 if (partial) {
  67                         int p = GHASH_BLOCK_SIZE - partial;
  68
  69                         memcpy(ctx->buf + partial, src, p);
  70                         src += p;
  71                         len -= p;
  72                 }
  73
  74                 blocks = len / GHASH_BLOCK_SIZE;
  75                 len %= GHASH_BLOCK_SIZE;
  76
  77                 kernel_neon_begin();
  78                 pmull_ghash_update(blocks, ctx->digest, src, key,
  79                                    partial ? ctx->buf : NULL);
  80                 kernel_neon_end();
  81                 src += blocks * GHASH_BLOCK_SIZE;
  82                 partial = 0;
  83         }
  84         if (len)
  85                 memcpy(ctx->buf + partial, src, len);
  86         return 0;
  87 }
  88
  89 static int ghash_final(struct shash_desc *desc, u8 *dst)
  90 {
  91         struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
  92         unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
  93
  94         if (partial) {
  95                 struct ghash_key *key = crypto_shash_ctx(desc->tfm);
  96
  97                 memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
  98                 kernel_neon_begin();
  99                 pmull_ghash_update(1, ctx->digest, ctx->buf, key, NULL);
 100                 kernel_neon_end();
 101         }
 102         put_unaligned_be64(ctx->digest[1], dst);
 103         put_unaligned_be64(ctx->digest[0], dst + 8);
 104
 105         *ctx = (struct ghash_desc_ctx){};
 106         return 0;
 107 }
 108
 109 static int ghash_setkey(struct crypto_shash *tfm,
 110                         const u8 *inkey, unsigned int keylen)
 111 {
 112         struct ghash_key *key = crypto_shash_ctx(tfm);
 113         u64 a, b;
 114
 115         if (keylen != GHASH_BLOCK_SIZE) {
 116                 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 117                 return -EINVAL;
 118         }
 119
 120         /* perform multiplication by 'x' in GF(2^128) */
 121         b = get_unaligned_be64(inkey);
 122         a = get_unaligned_be64(inkey + 8);
 123
 124         key->a = (a << 1) | (b >> 63);
 125         key->b = (b << 1) | (a >> 63);
 126
 127         if (b >> 63)
 128                 key->b ^= 0xc200000000000000UL;
 129
 130         return 0;
 131 }
 132
 133 static struct shash_alg ghash_alg = {
 134         .digestsize             = GHASH_DIGEST_SIZE,
 135         .init                   = ghash_init,
 136         .update                 = ghash_update,
 137         .final                  = ghash_final,
 138         .setkey                 = ghash_setkey,
 139         .descsize               = sizeof(struct ghash_desc_ctx),
 140         .base                   = {
 141                 .cra_name       = "ghash",
 142                 .cra_driver_name = "__driver-ghash-ce",
 143                 .cra_priority   = 0,
 144                 .cra_flags      = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_INTERNAL,
 145                 .cra_blocksize  = GHASH_BLOCK_SIZE,
 146                 .cra_ctxsize    = sizeof(struct ghash_key),
 147                 .cra_module     = THIS_MODULE,
 148         },
 149 };
 150
 151 static int ghash_async_init(struct ahash_request *req)
 152 {
 153         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 154         struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
 155         struct ahash_request *cryptd_req = ahash_request_ctx(req);
 156         struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
 157
 158         if (!may_use_simd()) {
 159                 memcpy(cryptd_req, req, sizeof(*req));
 160                 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
 161                 return crypto_ahash_init(cryptd_req);
 162         } else {
 163                 struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
 164                 struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
 165
 166                 desc->tfm = child;
 167                 desc->flags = req->base.flags;
 168                 return crypto_shash_init(desc);
 169         }
 170 }
 171
 172 static int ghash_async_update(struct ahash_request *req)
 173 {
 174         struct ahash_request *cryptd_req = ahash_request_ctx(req);
 175
 176         if (!may_use_simd()) {
 177                 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 178                 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
 179                 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
 180
 181                 memcpy(cryptd_req, req, sizeof(*req));
 182                 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
 183                 return crypto_ahash_update(cryptd_req);
 184         } else {
 185                 struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
 186                 return shash_ahash_update(req, desc);
 187         }
 188 }
 189
 190 static int ghash_async_final(struct ahash_request *req)
 191 {
 192         struct ahash_request *cryptd_req = ahash_request_ctx(req);
 193
 194         if (!may_use_simd()) {
 195                 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 196                 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
 197                 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
 198
 199                 memcpy(cryptd_req, req, sizeof(*req));
 200                 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
 201                 return crypto_ahash_final(cryptd_req);
 202         } else {
 203                 struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
 204                 return crypto_shash_final(desc, req->result);
 205         }
 206 }
 207
 208 static int ghash_async_digest(struct ahash_request *req)
 209 {
 210         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 211         struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
 212         struct ahash_request *cryptd_req = ahash_request_ctx(req);
 213         struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
 214
 215         if (!may_use_simd()) {
 216                 memcpy(cryptd_req, req, sizeof(*req));
 217                 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
 218                 return crypto_ahash_digest(cryptd_req);
 219         } else {
 220                 struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
 221                 struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
 222
 223                 desc->tfm = child;
 224                 desc->flags = req->base.flags;
 225                 return shash_ahash_digest(req, desc);
 226         }
 227 }
 228
 229 static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key,
 230                               unsigned int keylen)
 231 {
 232         struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
 233         struct crypto_ahash *child = &ctx->cryptd_tfm->base;
 234         int err;
 235
 236         crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 237         crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
 238                                & CRYPTO_TFM_REQ_MASK);
 239         err = crypto_ahash_setkey(child, key, keylen);
 240         crypto_ahash_set_flags(tfm, crypto_ahash_get_flags(child)
 241                                & CRYPTO_TFM_RES_MASK);
 242
 243         return err;
 244 }
 245
 246 static int ghash_async_init_tfm(struct crypto_tfm *tfm)
 247 {
 248         struct cryptd_ahash *cryptd_tfm;
 249         struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
 250
 251         cryptd_tfm = cryptd_alloc_ahash("__driver-ghash-ce",
 252                                         CRYPTO_ALG_INTERNAL,
 253                                         CRYPTO_ALG_INTERNAL);
 254         if (IS_ERR(cryptd_tfm))
 255                 return PTR_ERR(cryptd_tfm);
 256         ctx->cryptd_tfm = cryptd_tfm;
 257         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 258                                  sizeof(struct ahash_request) +
 259                                  crypto_ahash_reqsize(&cryptd_tfm->base));
 260
 261         return 0;
 262 }
 263
 264 static void ghash_async_exit_tfm(struct crypto_tfm *tfm)
 265 {
 266         struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
 267
 268         cryptd_free_ahash(ctx->cryptd_tfm);
 269 }
 270
 271 static struct ahash_alg ghash_async_alg = {
 272         .init                   = ghash_async_init,
 273         .update                 = ghash_async_update,
 274         .final                  = ghash_async_final,
 275         .setkey                 = ghash_async_setkey,
 276         .digest                 = ghash_async_digest,
 277         .halg.digestsize        = GHASH_DIGEST_SIZE,
 278         .halg.base              = {
 279                 .cra_name       = "ghash",
 280                 .cra_driver_name = "ghash-ce",
 281                 .cra_priority   = 300,
 282                 .cra_flags      = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
 283                 .cra_blocksize  = GHASH_BLOCK_SIZE,
 284                 .cra_type       = &crypto_ahash_type,
 285                 .cra_ctxsize    = sizeof(struct ghash_async_ctx),
 286                 .cra_module     = THIS_MODULE,
 287                 .cra_init       = ghash_async_init_tfm,
 288                 .cra_exit       = ghash_async_exit_tfm,
 289         },
 290 };
 291
 292 static int __init ghash_ce_mod_init(void)
 293 {
 294         int err;
 295
 296         if (!(elf_hwcap2 & HWCAP2_PMULL))
 297                 return -ENODEV;
 298
 299         err = crypto_register_shash(&ghash_alg);
 300         if (err)
 301                 return err;
 302         err = crypto_register_ahash(&ghash_async_alg);
 303         if (err)
 304                 goto err_shash;
 305
 306         return 0;
 307
 308 err_shash:
 309         crypto_unregister_shash(&ghash_alg);
 310         return err;
 311 }
 312
 313 static void __exit ghash_ce_mod_exit(void)
 314 {
 315         crypto_unregister_ahash(&ghash_async_alg);
 316         crypto_unregister_shash(&ghash_alg);
 317 }
 318
 319 module_init(ghash_ce_mod_init);
 320 module_exit(ghash_ce_mod_exit);