drivers/crypto/vmx/ghash.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * GHASH routines supporting VMX instructions on the Power 8
   4  *
   5  * Copyright (C) 2015, 2019 International Business Machines Inc.
   6  *
   7  * Author: Marcelo Henrique Cerri <[email protected]>
   8  *
   9  * Extended by Daniel Axtens <[email protected]> to replace the fallback
  10  * mechanism. The new approach is based on arm64 code, which is:
  11  *   Copyright (C) 2014 - 2018 Linaro Ltd. <[email protected]>
  12  */
  13
  14 #include <linux/types.h>
  15 #include <linux/err.h>
  16 #include <linux/crypto.h>
  17 #include <linux/delay.h>
  18 #include <asm/simd.h>
  19 #include <asm/switch_to.h>
  20 #include <crypto/aes.h>
  21 #include <crypto/ghash.h>
  22 #include <crypto/scatterwalk.h>
  23 #include <crypto/internal/hash.h>
  24 #include <crypto/internal/simd.h>
  25 #include <crypto/b128ops.h>
  26
  27 void gcm_init_p8(u128 htable[16], const u64 Xi[2]);
  28 void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]);
  29 void gcm_ghash_p8(u64 Xi[2], const u128 htable[16],
  30                   const u8 *in, size_t len);
  31
  32 struct p8_ghash_ctx {
  33         /* key used by vector asm */
  34         u128 htable[16];
  35         /* key used by software fallback */
  36         be128 key;
  37 };
  38
  39 struct p8_ghash_desc_ctx {
  40         u64 shash[2];
  41         u8 buffer[GHASH_DIGEST_SIZE];
  42         int bytes;
  43 };
  44
  45 static int p8_ghash_init(struct shash_desc *desc)
  46 {
  47         struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
  48
  49         dctx->bytes = 0;
  50         memset(dctx->shash, 0, GHASH_DIGEST_SIZE);
  51         return 0;
  52 }
  53
  54 static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
  55                            unsigned int keylen)
  56 {
  57         struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(tfm));
  58
  59         if (keylen != GHASH_BLOCK_SIZE)
  60                 return -EINVAL;
  61
  62         preempt_disable();
  63         pagefault_disable();
  64         enable_kernel_vsx();
  65         gcm_init_p8(ctx->htable, (const u64 *) key);
  66         disable_kernel_vsx();
  67         pagefault_enable();
  68         preempt_enable();
  69
  70         memcpy(&ctx->key, key, GHASH_BLOCK_SIZE);
  71
  72         return 0;
  73 }
  74
  75 static inline void __ghash_block(struct p8_ghash_ctx *ctx,
  76                                  struct p8_ghash_desc_ctx *dctx)
  77 {
  78         if (crypto_simd_usable()) {
  79                 preempt_disable();
  80                 pagefault_disable();
  81                 enable_kernel_vsx();
  82                 gcm_ghash_p8(dctx->shash, ctx->htable,
  83                                 dctx->buffer, GHASH_DIGEST_SIZE);
  84                 disable_kernel_vsx();
  85                 pagefault_enable();
  86                 preempt_enable();
  87         } else {
  88                 crypto_xor((u8 *)dctx->shash, dctx->buffer, GHASH_BLOCK_SIZE);
  89                 gf128mul_lle((be128 *)dctx->shash, &ctx->key);
  90         }
  91 }
  92
  93 static inline void __ghash_blocks(struct p8_ghash_ctx *ctx,
  94                                   struct p8_ghash_desc_ctx *dctx,
  95                                   const u8 *src, unsigned int srclen)
  96 {
  97         if (crypto_simd_usable()) {
  98                 preempt_disable();
  99                 pagefault_disable();
 100                 enable_kernel_vsx();
 101                 gcm_ghash_p8(dctx->shash, ctx->htable,
 102                                 src, srclen);
 103                 disable_kernel_vsx();
 104                 pagefault_enable();
 105                 preempt_enable();
 106         } else {
 107                 while (srclen >= GHASH_BLOCK_SIZE) {
 108                         crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE);
 109                         gf128mul_lle((be128 *)dctx->shash, &ctx->key);
 110                         srclen -= GHASH_BLOCK_SIZE;
 111                         src += GHASH_BLOCK_SIZE;
 112                 }
 113         }
 114 }
 115
 116 static int p8_ghash_update(struct shash_desc *desc,
 117                            const u8 *src, unsigned int srclen)
 118 {
 119         unsigned int len;
 120         struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
 121         struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
 122
 123         if (dctx->bytes) {
 124                 if (dctx->bytes + srclen < GHASH_DIGEST_SIZE) {
 125                         memcpy(dctx->buffer + dctx->bytes, src,
 126                                 srclen);
 127                         dctx->bytes += srclen;
 128                         return 0;
 129                 }
 130                 memcpy(dctx->buffer + dctx->bytes, src,
 131                         GHASH_DIGEST_SIZE - dctx->bytes);
 132
 133                 __ghash_block(ctx, dctx);
 134
 135                 src += GHASH_DIGEST_SIZE - dctx->bytes;
 136                 srclen -= GHASH_DIGEST_SIZE - dctx->bytes;
 137                 dctx->bytes = 0;
 138         }
 139         len = srclen & ~(GHASH_DIGEST_SIZE - 1);
 140         if (len) {
 141                 __ghash_blocks(ctx, dctx, src, len);
 142                 src += len;
 143                 srclen -= len;
 144         }
 145         if (srclen) {
 146                 memcpy(dctx->buffer, src, srclen);
 147                 dctx->bytes = srclen;
 148         }
 149         return 0;
 150 }
 151
 152 static int p8_ghash_final(struct shash_desc *desc, u8 *out)
 153 {
 154         int i;
 155         struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
 156         struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
 157
 158         if (dctx->bytes) {
 159                 for (i = dctx->bytes; i < GHASH_DIGEST_SIZE; i++)
 160                         dctx->buffer[i] = 0;
 161                 __ghash_block(ctx, dctx);
 162                 dctx->bytes = 0;
 163         }
 164         memcpy(out, dctx->shash, GHASH_DIGEST_SIZE);
 165         return 0;
 166 }
 167
 168 struct shash_alg p8_ghash_alg = {
 169         .digestsize = GHASH_DIGEST_SIZE,
 170         .init = p8_ghash_init,
 171         .update = p8_ghash_update,
 172         .final = p8_ghash_final,
 173         .setkey = p8_ghash_setkey,
 174         .descsize = sizeof(struct p8_ghash_desc_ctx)
 175                 + sizeof(struct ghash_desc_ctx),
 176         .base = {
 177                  .cra_name = "ghash",
 178                  .cra_driver_name = "p8_ghash",
 179                  .cra_priority = 1000,
 180                  .cra_blocksize = GHASH_BLOCK_SIZE,
 181                  .cra_ctxsize = sizeof(struct p8_ghash_ctx),
 182                  .cra_module = THIS_MODULE,
 183         },
 184 };