arch/powerpc/crypto/poly1305-p10-glue.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Poly1305 authenticator algorithm, RFC7539.
   4  *
   5  * Copyright 2023- IBM Corp. All rights reserved.
   6  */
   7
   8 #include <crypto/algapi.h>
   9 #include <linux/crypto.h>
  10 #include <linux/kernel.h>
  11 #include <linux/module.h>
  12 #include <linux/jump_label.h>
  13 #include <crypto/internal/hash.h>
  14 #include <crypto/internal/poly1305.h>
  15 #include <crypto/internal/simd.h>
  16 #include <linux/cpufeature.h>
  17 #include <asm/unaligned.h>
  18 #include <asm/simd.h>
  19 #include <asm/switch_to.h>
  20
  21 asmlinkage void poly1305_p10le_4blocks(void *h, const u8 *m, u32 mlen);
  22 asmlinkage void poly1305_64s(void *h, const u8 *m, u32 mlen, int highbit);
  23 asmlinkage void poly1305_emit_64(void *h, void *s, u8 *dst);
  24
  25 static void vsx_begin(void)
  26 {
  27         preempt_disable();
  28         enable_kernel_vsx();
  29 }
  30
  31 static void vsx_end(void)
  32 {
  33         disable_kernel_vsx();
  34         preempt_enable();
  35 }
  36
  37 static int crypto_poly1305_p10_init(struct shash_desc *desc)
  38 {
  39         struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
  40
  41         poly1305_core_init(&dctx->h);
  42         dctx->buflen = 0;
  43         dctx->rset = 0;
  44         dctx->sset = false;
  45
  46         return 0;
  47 }
  48
  49 static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx,
  50                                                const u8 *inp, unsigned int len)
  51 {
  52         unsigned int acc = 0;
  53
  54         if (unlikely(!dctx->sset)) {
  55                 if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) {
  56                         struct poly1305_core_key *key = &dctx->core_r;
  57
  58                         key->key.r64[0] = get_unaligned_le64(&inp[0]);
  59                         key->key.r64[1] = get_unaligned_le64(&inp[8]);
  60                         inp += POLY1305_BLOCK_SIZE;
  61                         len -= POLY1305_BLOCK_SIZE;
  62                         acc += POLY1305_BLOCK_SIZE;
  63                         dctx->rset = 1;
  64                 }
  65                 if (len >= POLY1305_BLOCK_SIZE) {
  66                         dctx->s[0] = get_unaligned_le32(&inp[0]);
  67                         dctx->s[1] = get_unaligned_le32(&inp[4]);
  68                         dctx->s[2] = get_unaligned_le32(&inp[8]);
  69                         dctx->s[3] = get_unaligned_le32(&inp[12]);
  70                         acc += POLY1305_BLOCK_SIZE;
  71                         dctx->sset = true;
  72                 }
  73         }
  74         return acc;
  75 }
  76
  77 static int crypto_poly1305_p10_update(struct shash_desc *desc,
  78                                       const u8 *src, unsigned int srclen)
  79 {
  80         struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
  81         unsigned int bytes, used;
  82
  83         if (unlikely(dctx->buflen)) {
  84                 bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen);
  85                 memcpy(dctx->buf + dctx->buflen, src, bytes);
  86                 src += bytes;
  87                 srclen -= bytes;
  88                 dctx->buflen += bytes;
  89
  90                 if (dctx->buflen == POLY1305_BLOCK_SIZE) {
  91                         if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf,
  92                                                                POLY1305_BLOCK_SIZE))) {
  93                                 vsx_begin();
  94                                 poly1305_64s(&dctx->h, dctx->buf,
  95                                                   POLY1305_BLOCK_SIZE, 1);
  96                                 vsx_end();
  97                         }
  98                         dctx->buflen = 0;
  99                 }
 100         }
 101
 102         if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
 103                 bytes = round_down(srclen, POLY1305_BLOCK_SIZE);
 104                 used = crypto_poly1305_setdctxkey(dctx, src, bytes);
 105                 if (likely(used)) {
 106                         srclen -= used;
 107                         src += used;
 108                 }
 109                 if (crypto_simd_usable() && (srclen >= POLY1305_BLOCK_SIZE*4)) {
 110                         vsx_begin();
 111                         poly1305_p10le_4blocks(&dctx->h, src, srclen);
 112                         vsx_end();
 113                         src += srclen - (srclen % (POLY1305_BLOCK_SIZE * 4));
 114                         srclen %= POLY1305_BLOCK_SIZE * 4;
 115                 }
 116                 while (srclen >= POLY1305_BLOCK_SIZE) {
 117                         vsx_begin();
 118                         poly1305_64s(&dctx->h, src, POLY1305_BLOCK_SIZE, 1);
 119                         vsx_end();
 120                         srclen -= POLY1305_BLOCK_SIZE;
 121                         src += POLY1305_BLOCK_SIZE;
 122                 }
 123         }
 124
 125         if (unlikely(srclen)) {
 126                 dctx->buflen = srclen;
 127                 memcpy(dctx->buf, src, srclen);
 128         }
 129
 130         return 0;
 131 }
 132
 133 static int crypto_poly1305_p10_final(struct shash_desc *desc, u8 *dst)
 134 {
 135         struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
 136
 137         if (unlikely(!dctx->sset))
 138                 return -ENOKEY;
 139
 140         if ((dctx->buflen)) {
 141                 dctx->buf[dctx->buflen++] = 1;
 142                 memset(dctx->buf + dctx->buflen, 0,
 143                        POLY1305_BLOCK_SIZE - dctx->buflen);
 144                 vsx_begin();
 145                 poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);
 146                 vsx_end();
 147                 dctx->buflen = 0;
 148         }
 149
 150         poly1305_emit_64(&dctx->h, &dctx->s, dst);
 151         return 0;
 152 }
 153
 154 static struct shash_alg poly1305_alg = {
 155         .digestsize     = POLY1305_DIGEST_SIZE,
 156         .init           = crypto_poly1305_p10_init,
 157         .update         = crypto_poly1305_p10_update,
 158         .final          = crypto_poly1305_p10_final,
 159         .descsize       = sizeof(struct poly1305_desc_ctx),
 160         .base           = {
 161                 .cra_name               = "poly1305",
 162                 .cra_driver_name        = "poly1305-p10",
 163                 .cra_priority           = 300,
 164                 .cra_blocksize          = POLY1305_BLOCK_SIZE,
 165                 .cra_module             = THIS_MODULE,
 166         },
 167 };
 168
 169 static int __init poly1305_p10_init(void)
 170 {
 171         return crypto_register_shash(&poly1305_alg);
 172 }
 173
 174 static void __exit poly1305_p10_exit(void)
 175 {
 176         crypto_unregister_shash(&poly1305_alg);
 177 }
 178
 179 module_cpu_feature_match(PPC_MODULE_FEATURE_P10, poly1305_p10_init);
 180 module_exit(poly1305_p10_exit);
 181
 182 MODULE_LICENSE("GPL");
 183 MODULE_AUTHOR("Danny Tsen <[email protected]>");
 184 MODULE_DESCRIPTION("Optimized Poly1305 for P10");
 185 MODULE_ALIAS_CRYPTO("poly1305");
 186 MODULE_ALIAS_CRYPTO("poly1305-p10");