drivers/crypto/inside-secure/safexcel_hash.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2017 Marvell
   4  *
   5  * Antoine Tenart <[email protected]>
   6  */
   7
   8 #include <crypto/aes.h>
   9 #include <crypto/hmac.h>
  10 #include <crypto/md5.h>
  11 #include <crypto/sha1.h>
  12 #include <crypto/sha2.h>
  13 #include <crypto/sha3.h>
  14 #include <crypto/skcipher.h>
  15 #include <crypto/sm3.h>
  16 #include <linux/device.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/dmapool.h>
  19
  20 #include "safexcel.h"
  21
  22 struct safexcel_ahash_ctx {
  23         struct safexcel_context base;
  24
  25         u32 alg;
  26         u8  key_sz;
  27         bool cbcmac;
  28         bool do_fallback;
  29         bool fb_init_done;
  30         bool fb_do_setkey;
  31
  32         struct crypto_cipher *kaes;
  33         struct crypto_ahash *fback;
  34         struct crypto_shash *shpre;
  35         struct shash_desc *shdesc;
  36 };
  37
  38 struct safexcel_ahash_req {
  39         bool last_req;
  40         bool finish;
  41         bool hmac;
  42         bool needs_inv;
  43         bool hmac_zlen;
  44         bool len_is_le;
  45         bool not_first;
  46         bool xcbcmac;
  47
  48         int nents;
  49         dma_addr_t result_dma;
  50
  51         u32 digest;
  52
  53         u8 state_sz;    /* expected state size, only set once */
  54         u8 block_sz;    /* block size, only set once */
  55         u8 digest_sz;   /* output digest size, only set once */
  56         __le32 state[SHA3_512_BLOCK_SIZE /
  57                      sizeof(__le32)] __aligned(sizeof(__le32));
  58
  59         u64 len;
  60         u64 processed;
  61
  62         u8 cache[HASH_CACHE_SIZE] __aligned(sizeof(u32));
  63         dma_addr_t cache_dma;
  64         unsigned int cache_sz;
  65
  66         u8 cache_next[HASH_CACHE_SIZE] __aligned(sizeof(u32));
  67 };
  68
  69 static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req)
  70 {
  71         return req->len - req->processed;
  72 }
  73
  74 static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
  75                                 u32 input_length, u32 result_length,
  76                                 bool cbcmac)
  77 {
  78         struct safexcel_token *token =
  79                 (struct safexcel_token *)cdesc->control_data.token;
  80
  81         token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
  82         token[0].packet_length = input_length;
  83         token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH;
  84
  85         input_length &= 15;
  86         if (unlikely(cbcmac && input_length)) {
  87                 token[0].stat =  0;
  88                 token[1].opcode = EIP197_TOKEN_OPCODE_INSERT;
  89                 token[1].packet_length = 16 - input_length;
  90                 token[1].stat = EIP197_TOKEN_STAT_LAST_HASH;
  91                 token[1].instructions = EIP197_TOKEN_INS_TYPE_HASH;
  92         } else {
  93                 token[0].stat = EIP197_TOKEN_STAT_LAST_HASH;
  94                 eip197_noop_token(&token[1]);
  95         }
  96
  97         token[2].opcode = EIP197_TOKEN_OPCODE_INSERT;
  98         token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
  99                         EIP197_TOKEN_STAT_LAST_PACKET;
 100         token[2].packet_length = result_length;
 101         token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
 102                                 EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
 103
 104         eip197_noop_token(&token[3]);
 105 }
 106
 107 static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
 108                                      struct safexcel_ahash_req *req,
 109                                      struct safexcel_command_desc *cdesc)
 110 {
 111         struct safexcel_crypto_priv *priv = ctx->base.priv;
 112         u64 count = 0;
 113
 114         cdesc->control_data.control0 = ctx->alg;
 115         cdesc->control_data.control1 = 0;
 116
 117         /*
 118          * Copy the input digest if needed, and setup the context
 119          * fields. Do this now as we need it to setup the first command
 120          * descriptor.
 121          */
 122         if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) {
 123                 if (req->xcbcmac)
 124                         memcpy(ctx->base.ctxr->data, &ctx->base.ipad, ctx->key_sz);
 125                 else
 126                         memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
 127
 128                 if (!req->finish && req->xcbcmac)
 129                         cdesc->control_data.control0 |=
 130                                 CONTEXT_CONTROL_DIGEST_XCM |
 131                                 CONTEXT_CONTROL_TYPE_HASH_OUT  |
 132                                 CONTEXT_CONTROL_NO_FINISH_HASH |
 133                                 CONTEXT_CONTROL_SIZE(req->state_sz /
 134                                                      sizeof(u32));
 135                 else
 136                         cdesc->control_data.control0 |=
 137                                 CONTEXT_CONTROL_DIGEST_XCM |
 138                                 CONTEXT_CONTROL_TYPE_HASH_OUT  |
 139                                 CONTEXT_CONTROL_SIZE(req->state_sz /
 140                                                      sizeof(u32));
 141                 return;
 142         } else if (!req->processed) {
 143                 /* First - and possibly only - block of basic hash only */
 144                 if (req->finish)
 145                         cdesc->control_data.control0 |= req->digest |
 146                                 CONTEXT_CONTROL_TYPE_HASH_OUT |
 147                                 CONTEXT_CONTROL_RESTART_HASH  |
 148                                 /* ensure its not 0! */
 149                                 CONTEXT_CONTROL_SIZE(1);
 150                 else
 151                         cdesc->control_data.control0 |= req->digest |
 152                                 CONTEXT_CONTROL_TYPE_HASH_OUT  |
 153                                 CONTEXT_CONTROL_RESTART_HASH   |
 154                                 CONTEXT_CONTROL_NO_FINISH_HASH |
 155                                 /* ensure its not 0! */
 156                                 CONTEXT_CONTROL_SIZE(1);
 157                 return;
 158         }
 159
 160         /* Hash continuation or HMAC, setup (inner) digest from state */
 161         memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
 162
 163         if (req->finish) {
 164                 /* Compute digest count for hash/HMAC finish operations */
 165                 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
 166                     req->hmac_zlen || (req->processed != req->block_sz)) {
 167                         count = req->processed / EIP197_COUNTER_BLOCK_SIZE;
 168
 169                         /* This is a hardware limitation, as the
 170                          * counter must fit into an u32. This represents
 171                          * a fairly big amount of input data, so we
 172                          * shouldn't see this.
 173                          */
 174                         if (unlikely(count & 0xffffffff00000000ULL)) {
 175                                 dev_warn(priv->dev,
 176                                          "Input data is too big\n");
 177                                 return;
 178                         }
 179                 }
 180
 181                 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
 182                     /* Special case: zero length HMAC */
 183                     req->hmac_zlen ||
 184                     /* PE HW < 4.4 cannot do HMAC continue, fake using hash */
 185                     (req->processed != req->block_sz)) {
 186                         /* Basic hash continue operation, need digest + cnt */
 187                         cdesc->control_data.control0 |=
 188                                 CONTEXT_CONTROL_SIZE((req->state_sz >> 2) + 1) |
 189                                 CONTEXT_CONTROL_TYPE_HASH_OUT |
 190                                 CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
 191                         /* For zero-len HMAC, don't finalize, already padded! */
 192                         if (req->hmac_zlen)
 193                                 cdesc->control_data.control0 |=
 194                                         CONTEXT_CONTROL_NO_FINISH_HASH;
 195                         cdesc->control_data.control1 |=
 196                                 CONTEXT_CONTROL_DIGEST_CNT;
 197                         ctx->base.ctxr->data[req->state_sz >> 2] =
 198                                 cpu_to_le32(count);
 199                         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
 200
 201                         /* Clear zero-length HMAC flag for next operation! */
 202                         req->hmac_zlen = false;
 203                 } else { /* HMAC */
 204                         /* Need outer digest for HMAC finalization */
 205                         memcpy(ctx->base.ctxr->data + (req->state_sz >> 2),
 206                                &ctx->base.opad, req->state_sz);
 207
 208                         /* Single pass HMAC - no digest count */
 209                         cdesc->control_data.control0 |=
 210                                 CONTEXT_CONTROL_SIZE(req->state_sz >> 1) |
 211                                 CONTEXT_CONTROL_TYPE_HASH_OUT |
 212                                 CONTEXT_CONTROL_DIGEST_HMAC;
 213                 }
 214         } else { /* Hash continuation, do not finish yet */
 215                 cdesc->control_data.control0 |=
 216                         CONTEXT_CONTROL_SIZE(req->state_sz >> 2) |
 217                         CONTEXT_CONTROL_DIGEST_PRECOMPUTED |
 218                         CONTEXT_CONTROL_TYPE_HASH_OUT |
 219                         CONTEXT_CONTROL_NO_FINISH_HASH;
 220         }
 221 }
 222
 223 static int safexcel_ahash_enqueue(struct ahash_request *areq);
 224
 225 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv,
 226                                       int ring,
 227                                       struct crypto_async_request *async,
 228                                       bool *should_complete, int *ret)
 229 {
 230         struct safexcel_result_desc *rdesc;
 231         struct ahash_request *areq = ahash_request_cast(async);
 232         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 233         struct safexcel_ahash_req *sreq = ahash_request_ctx(areq);
 234         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
 235         u64 cache_len;
 236
 237         *ret = 0;
 238
 239         rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 240         if (IS_ERR(rdesc)) {
 241                 dev_err(priv->dev,
 242                         "hash: result: could not retrieve the result descriptor\n");
 243                 *ret = PTR_ERR(rdesc);
 244         } else {
 245                 *ret = safexcel_rdesc_check_errors(priv, rdesc);
 246         }
 247
 248         safexcel_complete(priv, ring);
 249
 250         if (sreq->nents) {
 251                 dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE);
 252                 sreq->nents = 0;
 253         }
 254
 255         if (sreq->result_dma) {
 256                 dma_unmap_single(priv->dev, sreq->result_dma, sreq->digest_sz,
 257                                  DMA_FROM_DEVICE);
 258                 sreq->result_dma = 0;
 259         }
 260
 261         if (sreq->cache_dma) {
 262                 dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz,
 263                                  DMA_TO_DEVICE);
 264                 sreq->cache_dma = 0;
 265                 sreq->cache_sz = 0;
 266         }
 267
 268         if (sreq->finish) {
 269                 if (sreq->hmac &&
 270                     (sreq->digest != CONTEXT_CONTROL_DIGEST_HMAC)) {
 271                         /* Faking HMAC using hash - need to do outer hash */
 272                         memcpy(sreq->cache, sreq->state,
 273                                crypto_ahash_digestsize(ahash));
 274
 275                         memcpy(sreq->state, &ctx->base.opad, sreq->digest_sz);
 276
 277                         sreq->len = sreq->block_sz +
 278                                     crypto_ahash_digestsize(ahash);
 279                         sreq->processed = sreq->block_sz;
 280                         sreq->hmac = 0;
 281
 282                         if (priv->flags & EIP197_TRC_CACHE)
 283                                 ctx->base.needs_inv = true;
 284                         areq->nbytes = 0;
 285                         safexcel_ahash_enqueue(areq);
 286
 287                         *should_complete = false; /* Not done yet */
 288                         return 1;
 289                 }
 290
 291                 if (unlikely(sreq->digest == CONTEXT_CONTROL_DIGEST_XCM &&
 292                              ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_CRC32)) {
 293                         /* Undo final XOR with 0xffffffff ...*/
 294                         *(__le32 *)areq->result = ~sreq->state[0];
 295                 } else {
 296                         memcpy(areq->result, sreq->state,
 297                                crypto_ahash_digestsize(ahash));
 298                 }
 299         }
 300
 301         cache_len = safexcel_queued_len(sreq);
 302         if (cache_len)
 303                 memcpy(sreq->cache, sreq->cache_next, cache_len);
 304
 305         *should_complete = true;
 306
 307         return 1;
 308 }
 309
 310 static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
 311                                    int *commands, int *results)
 312 {
 313         struct ahash_request *areq = ahash_request_cast(async);
 314         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 315         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 316         struct safexcel_crypto_priv *priv = ctx->base.priv;
 317         struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
 318         struct safexcel_result_desc *rdesc;
 319         struct scatterlist *sg;
 320         struct safexcel_token *dmmy;
 321         int i, extra = 0, n_cdesc = 0, ret = 0, cache_len, skip = 0;
 322         u64 queued, len;
 323
 324         queued = safexcel_queued_len(req);
 325         if (queued <= HASH_CACHE_SIZE)
 326                 cache_len = queued;
 327         else
 328                 cache_len = queued - areq->nbytes;
 329
 330         if (!req->finish && !req->last_req) {
 331                 /* If this is not the last request and the queued data does not
 332                  * fit into full cache blocks, cache it for the next send call.
 333                  */
 334                 extra = queued & (HASH_CACHE_SIZE - 1);
 335
 336                 /* If this is not the last request and the queued data
 337                  * is a multiple of a block, cache the last one for now.
 338                  */
 339                 if (!extra)
 340                         extra = HASH_CACHE_SIZE;
 341
 342                 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
 343                                    req->cache_next, extra,
 344                                    areq->nbytes - extra);
 345
 346                 queued -= extra;
 347
 348                 if (!queued) {
 349                         *commands = 0;
 350                         *results = 0;
 351                         return 0;
 352                 }
 353
 354                 extra = 0;
 355         }
 356
 357         if (unlikely(req->xcbcmac && req->processed > AES_BLOCK_SIZE)) {
 358                 if (unlikely(cache_len < AES_BLOCK_SIZE)) {
 359                         /*
 360                          * Cache contains less than 1 full block, complete.
 361                          */
 362                         extra = AES_BLOCK_SIZE - cache_len;
 363                         if (queued > cache_len) {
 364                                 /* More data follows: borrow bytes */
 365                                 u64 tmp = queued - cache_len;
 366
 367                                 skip = min_t(u64, tmp, extra);
 368                                 sg_pcopy_to_buffer(areq->src,
 369                                         sg_nents(areq->src),
 370                                         req->cache + cache_len,
 371                                         skip, 0);
 372                         }
 373                         extra -= skip;
 374                         memset(req->cache + cache_len + skip, 0, extra);
 375                         if (!ctx->cbcmac && extra) {
 376                                 // 10- padding for XCBCMAC & CMAC
 377                                 req->cache[cache_len + skip] = 0x80;
 378                                 // HW will use K2 iso K3 - compensate!
 379                                 for (i = 0; i < AES_BLOCK_SIZE / 4; i++) {
 380                                         u32 *cache = (void *)req->cache;
 381                                         u32 *ipad = ctx->base.ipad.word;
 382                                         u32 x;
 383
 384                                         x = ipad[i] ^ ipad[i + 4];
 385                                         cache[i] ^= swab(x);
 386                                 }
 387                         }
 388                         cache_len = AES_BLOCK_SIZE;
 389                         queued = queued + extra;
 390                 }
 391
 392                 /* XCBC continue: XOR previous result into 1st word */
 393                 crypto_xor(req->cache, (const u8 *)req->state, AES_BLOCK_SIZE);
 394         }
 395
 396         len = queued;
 397         /* Add a command descriptor for the cached data, if any */
 398         if (cache_len) {
 399                 req->cache_dma = dma_map_single(priv->dev, req->cache,
 400                                                 cache_len, DMA_TO_DEVICE);
 401                 if (dma_mapping_error(priv->dev, req->cache_dma))
 402                         return -EINVAL;
 403
 404                 req->cache_sz = cache_len;
 405                 first_cdesc = safexcel_add_cdesc(priv, ring, 1,
 406                                                  (cache_len == len),
 407                                                  req->cache_dma, cache_len,
 408                                                  len, ctx->base.ctxr_dma,
 409                                                  &dmmy);
 410                 if (IS_ERR(first_cdesc)) {
 411                         ret = PTR_ERR(first_cdesc);
 412                         goto unmap_cache;
 413                 }
 414                 n_cdesc++;
 415
 416                 queued -= cache_len;
 417                 if (!queued)
 418                         goto send_command;
 419         }
 420
 421         /* Now handle the current ahash request buffer(s) */
 422         req->nents = dma_map_sg(priv->dev, areq->src,
 423                                 sg_nents_for_len(areq->src,
 424                                                  areq->nbytes),
 425                                 DMA_TO_DEVICE);
 426         if (!req->nents) {
 427                 ret = -ENOMEM;
 428                 goto cdesc_rollback;
 429         }
 430
 431         for_each_sg(areq->src, sg, req->nents, i) {
 432                 int sglen = sg_dma_len(sg);
 433
 434                 if (unlikely(sglen <= skip)) {
 435                         skip -= sglen;
 436                         continue;
 437                 }
 438
 439                 /* Do not overflow the request */
 440                 if ((queued + skip) <= sglen)
 441                         sglen = queued;
 442                 else
 443                         sglen -= skip;
 444
 445                 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
 446                                            !(queued - sglen),
 447                                            sg_dma_address(sg) + skip, sglen,
 448                                            len, ctx->base.ctxr_dma, &dmmy);
 449                 if (IS_ERR(cdesc)) {
 450                         ret = PTR_ERR(cdesc);
 451                         goto unmap_sg;
 452                 }
 453
 454                 if (!n_cdesc)
 455                         first_cdesc = cdesc;
 456                 n_cdesc++;
 457
 458                 queued -= sglen;
 459                 if (!queued)
 460                         break;
 461                 skip = 0;
 462         }
 463
 464 send_command:
 465         /* Setup the context options */
 466         safexcel_context_control(ctx, req, first_cdesc);
 467
 468         /* Add the token */
 469         safexcel_hash_token(first_cdesc, len, req->digest_sz, ctx->cbcmac);
 470
 471         req->result_dma = dma_map_single(priv->dev, req->state, req->digest_sz,
 472                                          DMA_FROM_DEVICE);
 473         if (dma_mapping_error(priv->dev, req->result_dma)) {
 474                 ret = -EINVAL;
 475                 goto unmap_sg;
 476         }
 477
 478         /* Add a result descriptor */
 479         rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma,
 480                                    req->digest_sz);
 481         if (IS_ERR(rdesc)) {
 482                 ret = PTR_ERR(rdesc);
 483                 goto unmap_result;
 484         }
 485
 486         safexcel_rdr_req_set(priv, ring, rdesc, &areq->base);
 487
 488         req->processed += len - extra;
 489
 490         *commands = n_cdesc;
 491         *results = 1;
 492         return 0;
 493
 494 unmap_result:
 495         dma_unmap_single(priv->dev, req->result_dma, req->digest_sz,
 496                          DMA_FROM_DEVICE);
 497 unmap_sg:
 498         if (req->nents) {
 499                 dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE);
 500                 req->nents = 0;
 501         }
 502 cdesc_rollback:
 503         for (i = 0; i < n_cdesc; i++)
 504                 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
 505 unmap_cache:
 506         if (req->cache_dma) {
 507                 dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz,
 508                                  DMA_TO_DEVICE);
 509                 req->cache_dma = 0;
 510                 req->cache_sz = 0;
 511         }
 512
 513         return ret;
 514 }
 515
 516 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
 517                                       int ring,
 518                                       struct crypto_async_request *async,
 519                                       bool *should_complete, int *ret)
 520 {
 521         struct safexcel_result_desc *rdesc;
 522         struct ahash_request *areq = ahash_request_cast(async);
 523         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 524         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
 525         int enq_ret;
 526
 527         *ret = 0;
 528
 529         rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 530         if (IS_ERR(rdesc)) {
 531                 dev_err(priv->dev,
 532                         "hash: invalidate: could not retrieve the result descriptor\n");
 533                 *ret = PTR_ERR(rdesc);
 534         } else {
 535                 *ret = safexcel_rdesc_check_errors(priv, rdesc);
 536         }
 537
 538         safexcel_complete(priv, ring);
 539
 540         if (ctx->base.exit_inv) {
 541                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
 542                               ctx->base.ctxr_dma);
 543
 544                 *should_complete = true;
 545                 return 1;
 546         }
 547
 548         ring = safexcel_select_ring(priv);
 549         ctx->base.ring = ring;
 550
 551         spin_lock_bh(&priv->ring[ring].queue_lock);
 552         enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async);
 553         spin_unlock_bh(&priv->ring[ring].queue_lock);
 554
 555         if (enq_ret != -EINPROGRESS)
 556                 *ret = enq_ret;
 557
 558         queue_work(priv->ring[ring].workqueue,
 559                    &priv->ring[ring].work_data.work);
 560
 561         *should_complete = false;
 562
 563         return 1;
 564 }
 565
 566 static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
 567                                   struct crypto_async_request *async,
 568                                   bool *should_complete, int *ret)
 569 {
 570         struct ahash_request *areq = ahash_request_cast(async);
 571         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 572         int err;
 573
 574         BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv);
 575
 576         if (req->needs_inv) {
 577                 req->needs_inv = false;
 578                 err = safexcel_handle_inv_result(priv, ring, async,
 579                                                  should_complete, ret);
 580         } else {
 581                 err = safexcel_handle_req_result(priv, ring, async,
 582                                                  should_complete, ret);
 583         }
 584
 585         return err;
 586 }
 587
 588 static int safexcel_ahash_send_inv(struct crypto_async_request *async,
 589                                    int ring, int *commands, int *results)
 590 {
 591         struct ahash_request *areq = ahash_request_cast(async);
 592         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 593         int ret;
 594
 595         ret = safexcel_invalidate_cache(async, ctx->base.priv,
 596                                         ctx->base.ctxr_dma, ring);
 597         if (unlikely(ret))
 598                 return ret;
 599
 600         *commands = 1;
 601         *results = 1;
 602
 603         return 0;
 604 }
 605
 606 static int safexcel_ahash_send(struct crypto_async_request *async,
 607                                int ring, int *commands, int *results)
 608 {
 609         struct ahash_request *areq = ahash_request_cast(async);
 610         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 611         int ret;
 612
 613         if (req->needs_inv)
 614                 ret = safexcel_ahash_send_inv(async, ring, commands, results);
 615         else
 616                 ret = safexcel_ahash_send_req(async, ring, commands, results);
 617
 618         return ret;
 619 }
 620
 621 static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
 622 {
 623         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
 624         struct safexcel_crypto_priv *priv = ctx->base.priv;
 625         EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE);
 626         struct safexcel_ahash_req *rctx = ahash_request_ctx(req);
 627         struct safexcel_inv_result result = {};
 628         int ring = ctx->base.ring;
 629
 630         memset(req, 0, EIP197_AHASH_REQ_SIZE);
 631
 632         /* create invalidation request */
 633         init_completion(&result.completion);
 634         ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 635                                    safexcel_inv_complete, &result);
 636
 637         ahash_request_set_tfm(req, __crypto_ahash_cast(tfm));
 638         ctx = crypto_tfm_ctx(req->base.tfm);
 639         ctx->base.exit_inv = true;
 640         rctx->needs_inv = true;
 641
 642         spin_lock_bh(&priv->ring[ring].queue_lock);
 643         crypto_enqueue_request(&priv->ring[ring].queue, &req->base);
 644         spin_unlock_bh(&priv->ring[ring].queue_lock);
 645
 646         queue_work(priv->ring[ring].workqueue,
 647                    &priv->ring[ring].work_data.work);
 648
 649         wait_for_completion(&result.completion);
 650
 651         if (result.error) {
 652                 dev_warn(priv->dev, "hash: completion error (%d)\n",
 653                          result.error);
 654                 return result.error;
 655         }
 656
 657         return 0;
 658 }
 659
 660 /* safexcel_ahash_cache: cache data until at least one request can be sent to
 661  * the engine, aka. when there is at least 1 block size in the pipe.
 662  */
 663 static int safexcel_ahash_cache(struct ahash_request *areq)
 664 {
 665         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 666         u64 cache_len;
 667
 668         /* cache_len: everything accepted by the driver but not sent yet,
 669          * tot sz handled by update() - last req sz - tot sz handled by send()
 670          */
 671         cache_len = safexcel_queued_len(req);
 672
 673         /*
 674          * In case there isn't enough bytes to proceed (less than a
 675          * block size), cache the data until we have enough.
 676          */
 677         if (cache_len + areq->nbytes <= HASH_CACHE_SIZE) {
 678                 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
 679                                    req->cache + cache_len,
 680                                    areq->nbytes, 0);
 681                 return 0;
 682         }
 683
 684         /* We couldn't cache all the data */
 685         return -E2BIG;
 686 }
 687
 688 static int safexcel_ahash_enqueue(struct ahash_request *areq)
 689 {
 690         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 691         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 692         struct safexcel_crypto_priv *priv = ctx->base.priv;
 693         int ret, ring;
 694
 695         req->needs_inv = false;
 696
 697         if (ctx->base.ctxr) {
 698                 if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv &&
 699                      /* invalidate for *any* non-XCBC continuation */
 700                    ((req->not_first && !req->xcbcmac) ||
 701                      /* invalidate if (i)digest changed */
 702                      memcmp(ctx->base.ctxr->data, req->state, req->state_sz) ||
 703                      /* invalidate for HMAC finish with odigest changed */
 704                      (req->finish && req->hmac &&
 705                       memcmp(ctx->base.ctxr->data + (req->state_sz>>2),
 706                              &ctx->base.opad, req->state_sz))))
 707                         /*
 708                          * We're still setting needs_inv here, even though it is
 709                          * cleared right away, because the needs_inv flag can be
 710                          * set in other functions and we want to keep the same
 711                          * logic.
 712                          */
 713                         ctx->base.needs_inv = true;
 714
 715                 if (ctx->base.needs_inv) {
 716                         ctx->base.needs_inv = false;
 717                         req->needs_inv = true;
 718                 }
 719         } else {
 720                 ctx->base.ring = safexcel_select_ring(priv);
 721                 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
 722                                                  EIP197_GFP_FLAGS(areq->base),
 723                                                  &ctx->base.ctxr_dma);
 724                 if (!ctx->base.ctxr)
 725                         return -ENOMEM;
 726         }
 727         req->not_first = true;
 728
 729         ring = ctx->base.ring;
 730
 731         spin_lock_bh(&priv->ring[ring].queue_lock);
 732         ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base);
 733         spin_unlock_bh(&priv->ring[ring].queue_lock);
 734
 735         queue_work(priv->ring[ring].workqueue,
 736                    &priv->ring[ring].work_data.work);
 737
 738         return ret;
 739 }
 740
 741 static int safexcel_ahash_update(struct ahash_request *areq)
 742 {
 743         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 744         int ret;
 745
 746         /* If the request is 0 length, do nothing */
 747         if (!areq->nbytes)
 748                 return 0;
 749
 750         /* Add request to the cache if it fits */
 751         ret = safexcel_ahash_cache(areq);
 752
 753         /* Update total request length */
 754         req->len += areq->nbytes;
 755
 756         /* If not all data could fit into the cache, go process the excess.
 757          * Also go process immediately for an HMAC IV precompute, which
 758          * will never be finished at all, but needs to be processed anyway.
 759          */
 760         if ((ret && !req->finish) || req->last_req)
 761                 return safexcel_ahash_enqueue(areq);
 762
 763         return 0;
 764 }
 765
 766 static int safexcel_ahash_final(struct ahash_request *areq)
 767 {
 768         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 769         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 770
 771         req->finish = true;
 772
 773         if (unlikely(!req->len && !areq->nbytes)) {
 774                 /*
 775                  * If we have an overall 0 length *hash* request:
 776                  * The HW cannot do 0 length hash, so we provide the correct
 777                  * result directly here.
 778                  */
 779                 if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5)
 780                         memcpy(areq->result, md5_zero_message_hash,
 781                                MD5_DIGEST_SIZE);
 782                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
 783                         memcpy(areq->result, sha1_zero_message_hash,
 784                                SHA1_DIGEST_SIZE);
 785                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224)
 786                         memcpy(areq->result, sha224_zero_message_hash,
 787                                SHA224_DIGEST_SIZE);
 788                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
 789                         memcpy(areq->result, sha256_zero_message_hash,
 790                                SHA256_DIGEST_SIZE);
 791                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384)
 792                         memcpy(areq->result, sha384_zero_message_hash,
 793                                SHA384_DIGEST_SIZE);
 794                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512)
 795                         memcpy(areq->result, sha512_zero_message_hash,
 796                                SHA512_DIGEST_SIZE);
 797                 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SM3) {
 798                         memcpy(areq->result,
 799                                EIP197_SM3_ZEROM_HASH, SM3_DIGEST_SIZE);
 800                 }
 801
 802                 return 0;
 803         } else if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM &&
 804                             ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5 &&
 805                             req->len == sizeof(u32) && !areq->nbytes)) {
 806                 /* Zero length CRC32 */
 807                 memcpy(areq->result, &ctx->base.ipad, sizeof(u32));
 808                 return 0;
 809         } else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE &&
 810                             !areq->nbytes)) {
 811                 /* Zero length CBC MAC */
 812                 memset(areq->result, 0, AES_BLOCK_SIZE);
 813                 return 0;
 814         } else if (unlikely(req->xcbcmac && req->len == AES_BLOCK_SIZE &&
 815                             !areq->nbytes)) {
 816                 /* Zero length (X)CBC/CMAC */
 817                 int i;
 818
 819                 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) {
 820                         u32 *result = (void *)areq->result;
 821
 822                         /* K3 */
 823                         result[i] = swab(ctx->base.ipad.word[i + 4]);
 824                 }
 825                 areq->result[0] ^= 0x80;                        // 10- padding
 826                 crypto_cipher_encrypt_one(ctx->kaes, areq->result, areq->result);
 827                 return 0;
 828         } else if (unlikely(req->hmac &&
 829                             (req->len == req->block_sz) &&
 830                             !areq->nbytes)) {
 831                 /*
 832                  * If we have an overall 0 length *HMAC* request:
 833                  * For HMAC, we need to finalize the inner digest
 834                  * and then perform the outer hash.
 835                  */
 836
 837                 /* generate pad block in the cache */
 838                 /* start with a hash block of all zeroes */
 839                 memset(req->cache, 0, req->block_sz);
 840                 /* set the first byte to 0x80 to 'append a 1 bit' */
 841                 req->cache[0] = 0x80;
 842                 /* add the length in bits in the last 2 bytes */
 843                 if (req->len_is_le) {
 844                         /* Little endian length word (e.g. MD5) */
 845                         req->cache[req->block_sz-8] = (req->block_sz << 3) &
 846                                                       255;
 847                         req->cache[req->block_sz-7] = (req->block_sz >> 5);
 848                 } else {
 849                         /* Big endian length word (e.g. any SHA) */
 850                         req->cache[req->block_sz-2] = (req->block_sz >> 5);
 851                         req->cache[req->block_sz-1] = (req->block_sz << 3) &
 852                                                       255;
 853                 }
 854
 855                 req->len += req->block_sz; /* plus 1 hash block */
 856
 857                 /* Set special zero-length HMAC flag */
 858                 req->hmac_zlen = true;
 859
 860                 /* Finalize HMAC */
 861                 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
 862         } else if (req->hmac) {
 863                 /* Finalize HMAC */
 864                 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
 865         }
 866
 867         return safexcel_ahash_enqueue(areq);
 868 }
 869
 870 static int safexcel_ahash_finup(struct ahash_request *areq)
 871 {
 872         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 873
 874         req->finish = true;
 875
 876         safexcel_ahash_update(areq);
 877         return safexcel_ahash_final(areq);
 878 }
 879
 880 static int safexcel_ahash_export(struct ahash_request *areq, void *out)
 881 {
 882         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 883         struct safexcel_ahash_export_state *export = out;
 884
 885         export->len = req->len;
 886         export->processed = req->processed;
 887
 888         export->digest = req->digest;
 889
 890         memcpy(export->state, req->state, req->state_sz);
 891         memcpy(export->cache, req->cache, HASH_CACHE_SIZE);
 892
 893         return 0;
 894 }
 895
 896 static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
 897 {
 898         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 899         const struct safexcel_ahash_export_state *export = in;
 900         int ret;
 901
 902         ret = crypto_ahash_init(areq);
 903         if (ret)
 904                 return ret;
 905
 906         req->len = export->len;
 907         req->processed = export->processed;
 908
 909         req->digest = export->digest;
 910
 911         memcpy(req->cache, export->cache, HASH_CACHE_SIZE);
 912         memcpy(req->state, export->state, req->state_sz);
 913
 914         return 0;
 915 }
 916
 917 static int safexcel_ahash_cra_init(struct crypto_tfm *tfm)
 918 {
 919         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
 920         struct safexcel_alg_template *tmpl =
 921                 container_of(__crypto_ahash_alg(tfm->__crt_alg),
 922                              struct safexcel_alg_template, alg.ahash);
 923
 924         ctx->base.priv = tmpl->priv;
 925         ctx->base.send = safexcel_ahash_send;
 926         ctx->base.handle_result = safexcel_handle_result;
 927         ctx->fb_do_setkey = false;
 928
 929         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 930                                  sizeof(struct safexcel_ahash_req));
 931         return 0;
 932 }
 933
 934 static int safexcel_sha1_init(struct ahash_request *areq)
 935 {
 936         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 937         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 938
 939         memset(req, 0, sizeof(*req));
 940
 941         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
 942         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
 943         req->state_sz = SHA1_DIGEST_SIZE;
 944         req->digest_sz = SHA1_DIGEST_SIZE;
 945         req->block_sz = SHA1_BLOCK_SIZE;
 946
 947         return 0;
 948 }
 949
 950 static int safexcel_sha1_digest(struct ahash_request *areq)
 951 {
 952         int ret = safexcel_sha1_init(areq);
 953
 954         if (ret)
 955                 return ret;
 956
 957         return safexcel_ahash_finup(areq);
 958 }
 959
 960 static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm)
 961 {
 962         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
 963         struct safexcel_crypto_priv *priv = ctx->base.priv;
 964         int ret;
 965
 966         /* context not allocated, skip invalidation */
 967         if (!ctx->base.ctxr)
 968                 return;
 969
 970         if (priv->flags & EIP197_TRC_CACHE) {
 971                 ret = safexcel_ahash_exit_inv(tfm);
 972                 if (ret)
 973                         dev_warn(priv->dev, "hash: invalidation error %d\n", ret);
 974         } else {
 975                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
 976                               ctx->base.ctxr_dma);
 977         }
 978 }
 979
 980 struct safexcel_alg_template safexcel_alg_sha1 = {
 981         .type = SAFEXCEL_ALG_TYPE_AHASH,
 982         .algo_mask = SAFEXCEL_ALG_SHA1,
 983         .alg.ahash = {
 984                 .init = safexcel_sha1_init,
 985                 .update = safexcel_ahash_update,
 986                 .final = safexcel_ahash_final,
 987                 .finup = safexcel_ahash_finup,
 988                 .digest = safexcel_sha1_digest,
 989                 .export = safexcel_ahash_export,
 990                 .import = safexcel_ahash_import,
 991                 .halg = {
 992                         .digestsize = SHA1_DIGEST_SIZE,
 993                         .statesize = sizeof(struct safexcel_ahash_export_state),
 994                         .base = {
 995                                 .cra_name = "sha1",
 996                                 .cra_driver_name = "safexcel-sha1",
 997                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
 998                                 .cra_flags = CRYPTO_ALG_ASYNC |
 999                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1000                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1001                                 .cra_blocksize = SHA1_BLOCK_SIZE,
1002                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1003                                 .cra_init = safexcel_ahash_cra_init,
1004                                 .cra_exit = safexcel_ahash_cra_exit,
1005                                 .cra_module = THIS_MODULE,
1006                         },
1007                 },
1008         },
1009 };
1010
1011 static int safexcel_hmac_sha1_init(struct ahash_request *areq)
1012 {
1013         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1014         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1015
1016         memset(req, 0, sizeof(*req));
1017
1018         /* Start from ipad precompute */
1019         memcpy(req->state, &ctx->base.ipad, SHA1_DIGEST_SIZE);
1020         /* Already processed the key^ipad part now! */
1021         req->len        = SHA1_BLOCK_SIZE;
1022         req->processed  = SHA1_BLOCK_SIZE;
1023
1024         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
1025         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1026         req->state_sz = SHA1_DIGEST_SIZE;
1027         req->digest_sz = SHA1_DIGEST_SIZE;
1028         req->block_sz = SHA1_BLOCK_SIZE;
1029         req->hmac = true;
1030
1031         return 0;
1032 }
1033
1034 static int safexcel_hmac_sha1_digest(struct ahash_request *areq)
1035 {
1036         int ret = safexcel_hmac_sha1_init(areq);
1037
1038         if (ret)
1039                 return ret;
1040
1041         return safexcel_ahash_finup(areq);
1042 }
1043
1044 struct safexcel_ahash_result {
1045         struct completion completion;
1046         int error;
1047 };
1048
1049 static void safexcel_ahash_complete(struct crypto_async_request *req, int error)
1050 {
1051         struct safexcel_ahash_result *result = req->data;
1052
1053         if (error == -EINPROGRESS)
1054                 return;
1055
1056         result->error = error;
1057         complete(&result->completion);
1058 }
1059
1060 static int safexcel_hmac_init_pad(struct ahash_request *areq,
1061                                   unsigned int blocksize, const u8 *key,
1062                                   unsigned int keylen, u8 *ipad, u8 *opad)
1063 {
1064         struct safexcel_ahash_result result;
1065         struct scatterlist sg;
1066         int ret, i;
1067         u8 *keydup;
1068
1069         if (keylen <= blocksize) {
1070                 memcpy(ipad, key, keylen);
1071         } else {
1072                 keydup = kmemdup(key, keylen, GFP_KERNEL);
1073                 if (!keydup)
1074                         return -ENOMEM;
1075
1076                 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
1077                                            safexcel_ahash_complete, &result);
1078                 sg_init_one(&sg, keydup, keylen);
1079                 ahash_request_set_crypt(areq, &sg, ipad, keylen);
1080                 init_completion(&result.completion);
1081
1082                 ret = crypto_ahash_digest(areq);
1083                 if (ret == -EINPROGRESS || ret == -EBUSY) {
1084                         wait_for_completion_interruptible(&result.completion);
1085                         ret = result.error;
1086                 }
1087
1088                 /* Avoid leaking */
1089                 kfree_sensitive(keydup);
1090
1091                 if (ret)
1092                         return ret;
1093
1094                 keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));
1095         }
1096
1097         memset(ipad + keylen, 0, blocksize - keylen);
1098         memcpy(opad, ipad, blocksize);
1099
1100         for (i = 0; i < blocksize; i++) {
1101                 ipad[i] ^= HMAC_IPAD_VALUE;
1102                 opad[i] ^= HMAC_OPAD_VALUE;
1103         }
1104
1105         return 0;
1106 }
1107
1108 static int safexcel_hmac_init_iv(struct ahash_request *areq,
1109                                  unsigned int blocksize, u8 *pad, void *state)
1110 {
1111         struct safexcel_ahash_result result;
1112         struct safexcel_ahash_req *req;
1113         struct scatterlist sg;
1114         int ret;
1115
1116         ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
1117                                    safexcel_ahash_complete, &result);
1118         sg_init_one(&sg, pad, blocksize);
1119         ahash_request_set_crypt(areq, &sg, pad, blocksize);
1120         init_completion(&result.completion);
1121
1122         ret = crypto_ahash_init(areq);
1123         if (ret)
1124                 return ret;
1125
1126         req = ahash_request_ctx(areq);
1127         req->hmac = true;
1128         req->last_req = true;
1129
1130         ret = crypto_ahash_update(areq);
1131         if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1132                 return ret;
1133
1134         wait_for_completion_interruptible(&result.completion);
1135         if (result.error)
1136                 return result.error;
1137
1138         return crypto_ahash_export(areq, state);
1139 }
1140
1141 static int __safexcel_hmac_setkey(const char *alg, const u8 *key,
1142                                   unsigned int keylen,
1143                                   void *istate, void *ostate)
1144 {
1145         struct ahash_request *areq;
1146         struct crypto_ahash *tfm;
1147         unsigned int blocksize;
1148         u8 *ipad, *opad;
1149         int ret;
1150
1151         tfm = crypto_alloc_ahash(alg, 0, 0);
1152         if (IS_ERR(tfm))
1153                 return PTR_ERR(tfm);
1154
1155         areq = ahash_request_alloc(tfm, GFP_KERNEL);
1156         if (!areq) {
1157                 ret = -ENOMEM;
1158                 goto free_ahash;
1159         }
1160
1161         crypto_ahash_clear_flags(tfm, ~0);
1162         blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1163
1164         ipad = kcalloc(2, blocksize, GFP_KERNEL);
1165         if (!ipad) {
1166                 ret = -ENOMEM;
1167                 goto free_request;
1168         }
1169
1170         opad = ipad + blocksize;
1171
1172         ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad);
1173         if (ret)
1174                 goto free_ipad;
1175
1176         ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate);
1177         if (ret)
1178                 goto free_ipad;
1179
1180         ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate);
1181
1182 free_ipad:
1183         kfree(ipad);
1184 free_request:
1185         ahash_request_free(areq);
1186 free_ahash:
1187         crypto_free_ahash(tfm);
1188
1189         return ret;
1190 }
1191
1192 int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key,
1193                          unsigned int keylen, const char *alg,
1194                          unsigned int state_sz)
1195 {
1196         struct safexcel_crypto_priv *priv = base->priv;
1197         struct safexcel_ahash_export_state istate, ostate;
1198         int ret;
1199
1200         ret = __safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate);
1201         if (ret)
1202                 return ret;
1203
1204         if (priv->flags & EIP197_TRC_CACHE && base->ctxr &&
1205             (memcmp(&base->ipad, istate.state, state_sz) ||
1206              memcmp(&base->opad, ostate.state, state_sz)))
1207                 base->needs_inv = true;
1208
1209         memcpy(&base->ipad, &istate.state, state_sz);
1210         memcpy(&base->opad, &ostate.state, state_sz);
1211
1212         return 0;
1213 }
1214
1215 static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key,
1216                                     unsigned int keylen, const char *alg,
1217                                     unsigned int state_sz)
1218 {
1219         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
1220
1221         return safexcel_hmac_setkey(&ctx->base, key, keylen, alg, state_sz);
1222 }
1223
1224 static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
1225                                      unsigned int keylen)
1226 {
1227         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha1",
1228                                         SHA1_DIGEST_SIZE);
1229 }
1230
1231 struct safexcel_alg_template safexcel_alg_hmac_sha1 = {
1232         .type = SAFEXCEL_ALG_TYPE_AHASH,
1233         .algo_mask = SAFEXCEL_ALG_SHA1,
1234         .alg.ahash = {
1235                 .init = safexcel_hmac_sha1_init,
1236                 .update = safexcel_ahash_update,
1237                 .final = safexcel_ahash_final,
1238                 .finup = safexcel_ahash_finup,
1239                 .digest = safexcel_hmac_sha1_digest,
1240                 .setkey = safexcel_hmac_sha1_setkey,
1241                 .export = safexcel_ahash_export,
1242                 .import = safexcel_ahash_import,
1243                 .halg = {
1244                         .digestsize = SHA1_DIGEST_SIZE,
1245                         .statesize = sizeof(struct safexcel_ahash_export_state),
1246                         .base = {
1247                                 .cra_name = "hmac(sha1)",
1248                                 .cra_driver_name = "safexcel-hmac-sha1",
1249                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1250                                 .cra_flags = CRYPTO_ALG_ASYNC |
1251                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1252                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1253                                 .cra_blocksize = SHA1_BLOCK_SIZE,
1254                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1255                                 .cra_init = safexcel_ahash_cra_init,
1256                                 .cra_exit = safexcel_ahash_cra_exit,
1257                                 .cra_module = THIS_MODULE,
1258                         },
1259                 },
1260         },
1261 };
1262
1263 static int safexcel_sha256_init(struct ahash_request *areq)
1264 {
1265         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1266         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1267
1268         memset(req, 0, sizeof(*req));
1269
1270         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1271         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1272         req->state_sz = SHA256_DIGEST_SIZE;
1273         req->digest_sz = SHA256_DIGEST_SIZE;
1274         req->block_sz = SHA256_BLOCK_SIZE;
1275
1276         return 0;
1277 }
1278
1279 static int safexcel_sha256_digest(struct ahash_request *areq)
1280 {
1281         int ret = safexcel_sha256_init(areq);
1282
1283         if (ret)
1284                 return ret;
1285
1286         return safexcel_ahash_finup(areq);
1287 }
1288
1289 struct safexcel_alg_template safexcel_alg_sha256 = {
1290         .type = SAFEXCEL_ALG_TYPE_AHASH,
1291         .algo_mask = SAFEXCEL_ALG_SHA2_256,
1292         .alg.ahash = {
1293                 .init = safexcel_sha256_init,
1294                 .update = safexcel_ahash_update,
1295                 .final = safexcel_ahash_final,
1296                 .finup = safexcel_ahash_finup,
1297                 .digest = safexcel_sha256_digest,
1298                 .export = safexcel_ahash_export,
1299                 .import = safexcel_ahash_import,
1300                 .halg = {
1301                         .digestsize = SHA256_DIGEST_SIZE,
1302                         .statesize = sizeof(struct safexcel_ahash_export_state),
1303                         .base = {
1304                                 .cra_name = "sha256",
1305                                 .cra_driver_name = "safexcel-sha256",
1306                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1307                                 .cra_flags = CRYPTO_ALG_ASYNC |
1308                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1309                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1310                                 .cra_blocksize = SHA256_BLOCK_SIZE,
1311                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1312                                 .cra_init = safexcel_ahash_cra_init,
1313                                 .cra_exit = safexcel_ahash_cra_exit,
1314                                 .cra_module = THIS_MODULE,
1315                         },
1316                 },
1317         },
1318 };
1319
1320 static int safexcel_sha224_init(struct ahash_request *areq)
1321 {
1322         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1323         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1324
1325         memset(req, 0, sizeof(*req));
1326
1327         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1328         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1329         req->state_sz = SHA256_DIGEST_SIZE;
1330         req->digest_sz = SHA256_DIGEST_SIZE;
1331         req->block_sz = SHA256_BLOCK_SIZE;
1332
1333         return 0;
1334 }
1335
1336 static int safexcel_sha224_digest(struct ahash_request *areq)
1337 {
1338         int ret = safexcel_sha224_init(areq);
1339
1340         if (ret)
1341                 return ret;
1342
1343         return safexcel_ahash_finup(areq);
1344 }
1345
1346 struct safexcel_alg_template safexcel_alg_sha224 = {
1347         .type = SAFEXCEL_ALG_TYPE_AHASH,
1348         .algo_mask = SAFEXCEL_ALG_SHA2_256,
1349         .alg.ahash = {
1350                 .init = safexcel_sha224_init,
1351                 .update = safexcel_ahash_update,
1352                 .final = safexcel_ahash_final,
1353                 .finup = safexcel_ahash_finup,
1354                 .digest = safexcel_sha224_digest,
1355                 .export = safexcel_ahash_export,
1356                 .import = safexcel_ahash_import,
1357                 .halg = {
1358                         .digestsize = SHA224_DIGEST_SIZE,
1359                         .statesize = sizeof(struct safexcel_ahash_export_state),
1360                         .base = {
1361                                 .cra_name = "sha224",
1362                                 .cra_driver_name = "safexcel-sha224",
1363                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1364                                 .cra_flags = CRYPTO_ALG_ASYNC |
1365                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1366                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1367                                 .cra_blocksize = SHA224_BLOCK_SIZE,
1368                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1369                                 .cra_init = safexcel_ahash_cra_init,
1370                                 .cra_exit = safexcel_ahash_cra_exit,
1371                                 .cra_module = THIS_MODULE,
1372                         },
1373                 },
1374         },
1375 };
1376
1377 static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key,
1378                                        unsigned int keylen)
1379 {
1380         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha224",
1381                                         SHA256_DIGEST_SIZE);
1382 }
1383
1384 static int safexcel_hmac_sha224_init(struct ahash_request *areq)
1385 {
1386         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1387         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1388
1389         memset(req, 0, sizeof(*req));
1390
1391         /* Start from ipad precompute */
1392         memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);
1393         /* Already processed the key^ipad part now! */
1394         req->len        = SHA256_BLOCK_SIZE;
1395         req->processed  = SHA256_BLOCK_SIZE;
1396
1397         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1398         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1399         req->state_sz = SHA256_DIGEST_SIZE;
1400         req->digest_sz = SHA256_DIGEST_SIZE;
1401         req->block_sz = SHA256_BLOCK_SIZE;
1402         req->hmac = true;
1403
1404         return 0;
1405 }
1406
1407 static int safexcel_hmac_sha224_digest(struct ahash_request *areq)
1408 {
1409         int ret = safexcel_hmac_sha224_init(areq);
1410
1411         if (ret)
1412                 return ret;
1413
1414         return safexcel_ahash_finup(areq);
1415 }
1416
1417 struct safexcel_alg_template safexcel_alg_hmac_sha224 = {
1418         .type = SAFEXCEL_ALG_TYPE_AHASH,
1419         .algo_mask = SAFEXCEL_ALG_SHA2_256,
1420         .alg.ahash = {
1421                 .init = safexcel_hmac_sha224_init,
1422                 .update = safexcel_ahash_update,
1423                 .final = safexcel_ahash_final,
1424                 .finup = safexcel_ahash_finup,
1425                 .digest = safexcel_hmac_sha224_digest,
1426                 .setkey = safexcel_hmac_sha224_setkey,
1427                 .export = safexcel_ahash_export,
1428                 .import = safexcel_ahash_import,
1429                 .halg = {
1430                         .digestsize = SHA224_DIGEST_SIZE,
1431                         .statesize = sizeof(struct safexcel_ahash_export_state),
1432                         .base = {
1433                                 .cra_name = "hmac(sha224)",
1434                                 .cra_driver_name = "safexcel-hmac-sha224",
1435                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1436                                 .cra_flags = CRYPTO_ALG_ASYNC |
1437                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1438                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1439                                 .cra_blocksize = SHA224_BLOCK_SIZE,
1440                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1441                                 .cra_init = safexcel_ahash_cra_init,
1442                                 .cra_exit = safexcel_ahash_cra_exit,
1443                                 .cra_module = THIS_MODULE,
1444                         },
1445                 },
1446         },
1447 };
1448
1449 static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key,
1450                                      unsigned int keylen)
1451 {
1452         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha256",
1453                                         SHA256_DIGEST_SIZE);
1454 }
1455
1456 static int safexcel_hmac_sha256_init(struct ahash_request *areq)
1457 {
1458         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1459         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1460
1461         memset(req, 0, sizeof(*req));
1462
1463         /* Start from ipad precompute */
1464         memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);
1465         /* Already processed the key^ipad part now! */
1466         req->len        = SHA256_BLOCK_SIZE;
1467         req->processed  = SHA256_BLOCK_SIZE;
1468
1469         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1470         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1471         req->state_sz = SHA256_DIGEST_SIZE;
1472         req->digest_sz = SHA256_DIGEST_SIZE;
1473         req->block_sz = SHA256_BLOCK_SIZE;
1474         req->hmac = true;
1475
1476         return 0;
1477 }
1478
1479 static int safexcel_hmac_sha256_digest(struct ahash_request *areq)
1480 {
1481         int ret = safexcel_hmac_sha256_init(areq);
1482
1483         if (ret)
1484                 return ret;
1485
1486         return safexcel_ahash_finup(areq);
1487 }
1488
1489 struct safexcel_alg_template safexcel_alg_hmac_sha256 = {
1490         .type = SAFEXCEL_ALG_TYPE_AHASH,
1491         .algo_mask = SAFEXCEL_ALG_SHA2_256,
1492         .alg.ahash = {
1493                 .init = safexcel_hmac_sha256_init,
1494                 .update = safexcel_ahash_update,
1495                 .final = safexcel_ahash_final,
1496                 .finup = safexcel_ahash_finup,
1497                 .digest = safexcel_hmac_sha256_digest,
1498                 .setkey = safexcel_hmac_sha256_setkey,
1499                 .export = safexcel_ahash_export,
1500                 .import = safexcel_ahash_import,
1501                 .halg = {
1502                         .digestsize = SHA256_DIGEST_SIZE,
1503                         .statesize = sizeof(struct safexcel_ahash_export_state),
1504                         .base = {
1505                                 .cra_name = "hmac(sha256)",
1506                                 .cra_driver_name = "safexcel-hmac-sha256",
1507                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1508                                 .cra_flags = CRYPTO_ALG_ASYNC |
1509                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1510                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1511                                 .cra_blocksize = SHA256_BLOCK_SIZE,
1512                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1513                                 .cra_init = safexcel_ahash_cra_init,
1514                                 .cra_exit = safexcel_ahash_cra_exit,
1515                                 .cra_module = THIS_MODULE,
1516                         },
1517                 },
1518         },
1519 };
1520
1521 static int safexcel_sha512_init(struct ahash_request *areq)
1522 {
1523         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1524         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1525
1526         memset(req, 0, sizeof(*req));
1527
1528         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1529         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1530         req->state_sz = SHA512_DIGEST_SIZE;
1531         req->digest_sz = SHA512_DIGEST_SIZE;
1532         req->block_sz = SHA512_BLOCK_SIZE;
1533
1534         return 0;
1535 }
1536
1537 static int safexcel_sha512_digest(struct ahash_request *areq)
1538 {
1539         int ret = safexcel_sha512_init(areq);
1540
1541         if (ret)
1542                 return ret;
1543
1544         return safexcel_ahash_finup(areq);
1545 }
1546
1547 struct safexcel_alg_template safexcel_alg_sha512 = {
1548         .type = SAFEXCEL_ALG_TYPE_AHASH,
1549         .algo_mask = SAFEXCEL_ALG_SHA2_512,
1550         .alg.ahash = {
1551                 .init = safexcel_sha512_init,
1552                 .update = safexcel_ahash_update,
1553                 .final = safexcel_ahash_final,
1554                 .finup = safexcel_ahash_finup,
1555                 .digest = safexcel_sha512_digest,
1556                 .export = safexcel_ahash_export,
1557                 .import = safexcel_ahash_import,
1558                 .halg = {
1559                         .digestsize = SHA512_DIGEST_SIZE,
1560                         .statesize = sizeof(struct safexcel_ahash_export_state),
1561                         .base = {
1562                                 .cra_name = "sha512",
1563                                 .cra_driver_name = "safexcel-sha512",
1564                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1565                                 .cra_flags = CRYPTO_ALG_ASYNC |
1566                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1567                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1568                                 .cra_blocksize = SHA512_BLOCK_SIZE,
1569                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1570                                 .cra_init = safexcel_ahash_cra_init,
1571                                 .cra_exit = safexcel_ahash_cra_exit,
1572                                 .cra_module = THIS_MODULE,
1573                         },
1574                 },
1575         },
1576 };
1577
1578 static int safexcel_sha384_init(struct ahash_request *areq)
1579 {
1580         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1581         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1582
1583         memset(req, 0, sizeof(*req));
1584
1585         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1586         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1587         req->state_sz = SHA512_DIGEST_SIZE;
1588         req->digest_sz = SHA512_DIGEST_SIZE;
1589         req->block_sz = SHA512_BLOCK_SIZE;
1590
1591         return 0;
1592 }
1593
1594 static int safexcel_sha384_digest(struct ahash_request *areq)
1595 {
1596         int ret = safexcel_sha384_init(areq);
1597
1598         if (ret)
1599                 return ret;
1600
1601         return safexcel_ahash_finup(areq);
1602 }
1603
1604 struct safexcel_alg_template safexcel_alg_sha384 = {
1605         .type = SAFEXCEL_ALG_TYPE_AHASH,
1606         .algo_mask = SAFEXCEL_ALG_SHA2_512,
1607         .alg.ahash = {
1608                 .init = safexcel_sha384_init,
1609                 .update = safexcel_ahash_update,
1610                 .final = safexcel_ahash_final,
1611                 .finup = safexcel_ahash_finup,
1612                 .digest = safexcel_sha384_digest,
1613                 .export = safexcel_ahash_export,
1614                 .import = safexcel_ahash_import,
1615                 .halg = {
1616                         .digestsize = SHA384_DIGEST_SIZE,
1617                         .statesize = sizeof(struct safexcel_ahash_export_state),
1618                         .base = {
1619                                 .cra_name = "sha384",
1620                                 .cra_driver_name = "safexcel-sha384",
1621                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1622                                 .cra_flags = CRYPTO_ALG_ASYNC |
1623                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1624                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1625                                 .cra_blocksize = SHA384_BLOCK_SIZE,
1626                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1627                                 .cra_init = safexcel_ahash_cra_init,
1628                                 .cra_exit = safexcel_ahash_cra_exit,
1629                                 .cra_module = THIS_MODULE,
1630                         },
1631                 },
1632         },
1633 };
1634
1635 static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key,
1636                                        unsigned int keylen)
1637 {
1638         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha512",
1639                                         SHA512_DIGEST_SIZE);
1640 }
1641
1642 static int safexcel_hmac_sha512_init(struct ahash_request *areq)
1643 {
1644         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1645         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1646
1647         memset(req, 0, sizeof(*req));
1648
1649         /* Start from ipad precompute */
1650         memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);
1651         /* Already processed the key^ipad part now! */
1652         req->len        = SHA512_BLOCK_SIZE;
1653         req->processed  = SHA512_BLOCK_SIZE;
1654
1655         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1656         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1657         req->state_sz = SHA512_DIGEST_SIZE;
1658         req->digest_sz = SHA512_DIGEST_SIZE;
1659         req->block_sz = SHA512_BLOCK_SIZE;
1660         req->hmac = true;
1661
1662         return 0;
1663 }
1664
1665 static int safexcel_hmac_sha512_digest(struct ahash_request *areq)
1666 {
1667         int ret = safexcel_hmac_sha512_init(areq);
1668
1669         if (ret)
1670                 return ret;
1671
1672         return safexcel_ahash_finup(areq);
1673 }
1674
1675 struct safexcel_alg_template safexcel_alg_hmac_sha512 = {
1676         .type = SAFEXCEL_ALG_TYPE_AHASH,
1677         .algo_mask = SAFEXCEL_ALG_SHA2_512,
1678         .alg.ahash = {
1679                 .init = safexcel_hmac_sha512_init,
1680                 .update = safexcel_ahash_update,
1681                 .final = safexcel_ahash_final,
1682                 .finup = safexcel_ahash_finup,
1683                 .digest = safexcel_hmac_sha512_digest,
1684                 .setkey = safexcel_hmac_sha512_setkey,
1685                 .export = safexcel_ahash_export,
1686                 .import = safexcel_ahash_import,
1687                 .halg = {
1688                         .digestsize = SHA512_DIGEST_SIZE,
1689                         .statesize = sizeof(struct safexcel_ahash_export_state),
1690                         .base = {
1691                                 .cra_name = "hmac(sha512)",
1692                                 .cra_driver_name = "safexcel-hmac-sha512",
1693                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1694                                 .cra_flags = CRYPTO_ALG_ASYNC |
1695                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1696                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1697                                 .cra_blocksize = SHA512_BLOCK_SIZE,
1698                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1699                                 .cra_init = safexcel_ahash_cra_init,
1700                                 .cra_exit = safexcel_ahash_cra_exit,
1701                                 .cra_module = THIS_MODULE,
1702                         },
1703                 },
1704         },
1705 };
1706
1707 static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key,
1708                                        unsigned int keylen)
1709 {
1710         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha384",
1711                                         SHA512_DIGEST_SIZE);
1712 }
1713
1714 static int safexcel_hmac_sha384_init(struct ahash_request *areq)
1715 {
1716         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1717         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1718
1719         memset(req, 0, sizeof(*req));
1720
1721         /* Start from ipad precompute */
1722         memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);
1723         /* Already processed the key^ipad part now! */
1724         req->len        = SHA512_BLOCK_SIZE;
1725         req->processed  = SHA512_BLOCK_SIZE;
1726
1727         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1728         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1729         req->state_sz = SHA512_DIGEST_SIZE;
1730         req->digest_sz = SHA512_DIGEST_SIZE;
1731         req->block_sz = SHA512_BLOCK_SIZE;
1732         req->hmac = true;
1733
1734         return 0;
1735 }
1736
1737 static int safexcel_hmac_sha384_digest(struct ahash_request *areq)
1738 {
1739         int ret = safexcel_hmac_sha384_init(areq);
1740
1741         if (ret)
1742                 return ret;
1743
1744         return safexcel_ahash_finup(areq);
1745 }
1746
1747 struct safexcel_alg_template safexcel_alg_hmac_sha384 = {
1748         .type = SAFEXCEL_ALG_TYPE_AHASH,
1749         .algo_mask = SAFEXCEL_ALG_SHA2_512,
1750         .alg.ahash = {
1751                 .init = safexcel_hmac_sha384_init,
1752                 .update = safexcel_ahash_update,
1753                 .final = safexcel_ahash_final,
1754                 .finup = safexcel_ahash_finup,
1755                 .digest = safexcel_hmac_sha384_digest,
1756                 .setkey = safexcel_hmac_sha384_setkey,
1757                 .export = safexcel_ahash_export,
1758                 .import = safexcel_ahash_import,
1759                 .halg = {
1760                         .digestsize = SHA384_DIGEST_SIZE,
1761                         .statesize = sizeof(struct safexcel_ahash_export_state),
1762                         .base = {
1763                                 .cra_name = "hmac(sha384)",
1764                                 .cra_driver_name = "safexcel-hmac-sha384",
1765                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1766                                 .cra_flags = CRYPTO_ALG_ASYNC |
1767                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1768                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1769                                 .cra_blocksize = SHA384_BLOCK_SIZE,
1770                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1771                                 .cra_init = safexcel_ahash_cra_init,
1772                                 .cra_exit = safexcel_ahash_cra_exit,
1773                                 .cra_module = THIS_MODULE,
1774                         },
1775                 },
1776         },
1777 };
1778
1779 static int safexcel_md5_init(struct ahash_request *areq)
1780 {
1781         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1782         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1783
1784         memset(req, 0, sizeof(*req));
1785
1786         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
1787         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1788         req->state_sz = MD5_DIGEST_SIZE;
1789         req->digest_sz = MD5_DIGEST_SIZE;
1790         req->block_sz = MD5_HMAC_BLOCK_SIZE;
1791
1792         return 0;
1793 }
1794
1795 static int safexcel_md5_digest(struct ahash_request *areq)
1796 {
1797         int ret = safexcel_md5_init(areq);
1798
1799         if (ret)
1800                 return ret;
1801
1802         return safexcel_ahash_finup(areq);
1803 }
1804
1805 struct safexcel_alg_template safexcel_alg_md5 = {
1806         .type = SAFEXCEL_ALG_TYPE_AHASH,
1807         .algo_mask = SAFEXCEL_ALG_MD5,
1808         .alg.ahash = {
1809                 .init = safexcel_md5_init,
1810                 .update = safexcel_ahash_update,
1811                 .final = safexcel_ahash_final,
1812                 .finup = safexcel_ahash_finup,
1813                 .digest = safexcel_md5_digest,
1814                 .export = safexcel_ahash_export,
1815                 .import = safexcel_ahash_import,
1816                 .halg = {
1817                         .digestsize = MD5_DIGEST_SIZE,
1818                         .statesize = sizeof(struct safexcel_ahash_export_state),
1819                         .base = {
1820                                 .cra_name = "md5",
1821                                 .cra_driver_name = "safexcel-md5",
1822                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1823                                 .cra_flags = CRYPTO_ALG_ASYNC |
1824                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1825                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1826                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1827                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1828                                 .cra_init = safexcel_ahash_cra_init,
1829                                 .cra_exit = safexcel_ahash_cra_exit,
1830                                 .cra_module = THIS_MODULE,
1831                         },
1832                 },
1833         },
1834 };
1835
1836 static int safexcel_hmac_md5_init(struct ahash_request *areq)
1837 {
1838         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1839         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1840
1841         memset(req, 0, sizeof(*req));
1842
1843         /* Start from ipad precompute */
1844         memcpy(req->state, &ctx->base.ipad, MD5_DIGEST_SIZE);
1845         /* Already processed the key^ipad part now! */
1846         req->len        = MD5_HMAC_BLOCK_SIZE;
1847         req->processed  = MD5_HMAC_BLOCK_SIZE;
1848
1849         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
1850         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1851         req->state_sz = MD5_DIGEST_SIZE;
1852         req->digest_sz = MD5_DIGEST_SIZE;
1853         req->block_sz = MD5_HMAC_BLOCK_SIZE;
1854         req->len_is_le = true; /* MD5 is little endian! ... */
1855         req->hmac = true;
1856
1857         return 0;
1858 }
1859
1860 static int safexcel_hmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key,
1861                                      unsigned int keylen)
1862 {
1863         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-md5",
1864                                         MD5_DIGEST_SIZE);
1865 }
1866
1867 static int safexcel_hmac_md5_digest(struct ahash_request *areq)
1868 {
1869         int ret = safexcel_hmac_md5_init(areq);
1870
1871         if (ret)
1872                 return ret;
1873
1874         return safexcel_ahash_finup(areq);
1875 }
1876
1877 struct safexcel_alg_template safexcel_alg_hmac_md5 = {
1878         .type = SAFEXCEL_ALG_TYPE_AHASH,
1879         .algo_mask = SAFEXCEL_ALG_MD5,
1880         .alg.ahash = {
1881                 .init = safexcel_hmac_md5_init,
1882                 .update = safexcel_ahash_update,
1883                 .final = safexcel_ahash_final,
1884                 .finup = safexcel_ahash_finup,
1885                 .digest = safexcel_hmac_md5_digest,
1886                 .setkey = safexcel_hmac_md5_setkey,
1887                 .export = safexcel_ahash_export,
1888                 .import = safexcel_ahash_import,
1889                 .halg = {
1890                         .digestsize = MD5_DIGEST_SIZE,
1891                         .statesize = sizeof(struct safexcel_ahash_export_state),
1892                         .base = {
1893                                 .cra_name = "hmac(md5)",
1894                                 .cra_driver_name = "safexcel-hmac-md5",
1895                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1896                                 .cra_flags = CRYPTO_ALG_ASYNC |
1897                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1898                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1899                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1900                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1901                                 .cra_init = safexcel_ahash_cra_init,
1902                                 .cra_exit = safexcel_ahash_cra_exit,
1903                                 .cra_module = THIS_MODULE,
1904                         },
1905                 },
1906         },
1907 };
1908
1909 static int safexcel_crc32_cra_init(struct crypto_tfm *tfm)
1910 {
1911         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
1912         int ret = safexcel_ahash_cra_init(tfm);
1913
1914         /* Default 'key' is all zeroes */
1915         memset(&ctx->base.ipad, 0, sizeof(u32));
1916         return ret;
1917 }
1918
1919 static int safexcel_crc32_init(struct ahash_request *areq)
1920 {
1921         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1922         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1923
1924         memset(req, 0, sizeof(*req));
1925
1926         /* Start from loaded key */
1927         req->state[0]   = cpu_to_le32(~ctx->base.ipad.word[0]);
1928         /* Set processed to non-zero to enable invalidation detection */
1929         req->len        = sizeof(u32);
1930         req->processed  = sizeof(u32);
1931
1932         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_CRC32;
1933         req->digest = CONTEXT_CONTROL_DIGEST_XCM;
1934         req->state_sz = sizeof(u32);
1935         req->digest_sz = sizeof(u32);
1936         req->block_sz = sizeof(u32);
1937
1938         return 0;
1939 }
1940
1941 static int safexcel_crc32_setkey(struct crypto_ahash *tfm, const u8 *key,
1942                                  unsigned int keylen)
1943 {
1944         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1945
1946         if (keylen != sizeof(u32))
1947                 return -EINVAL;
1948
1949         memcpy(&ctx->base.ipad, key, sizeof(u32));
1950         return 0;
1951 }
1952
1953 static int safexcel_crc32_digest(struct ahash_request *areq)
1954 {
1955         return safexcel_crc32_init(areq) ?: safexcel_ahash_finup(areq);
1956 }
1957
1958 struct safexcel_alg_template safexcel_alg_crc32 = {
1959         .type = SAFEXCEL_ALG_TYPE_AHASH,
1960         .algo_mask = 0,
1961         .alg.ahash = {
1962                 .init = safexcel_crc32_init,
1963                 .update = safexcel_ahash_update,
1964                 .final = safexcel_ahash_final,
1965                 .finup = safexcel_ahash_finup,
1966                 .digest = safexcel_crc32_digest,
1967                 .setkey = safexcel_crc32_setkey,
1968                 .export = safexcel_ahash_export,
1969                 .import = safexcel_ahash_import,
1970                 .halg = {
1971                         .digestsize = sizeof(u32),
1972                         .statesize = sizeof(struct safexcel_ahash_export_state),
1973                         .base = {
1974                                 .cra_name = "crc32",
1975                                 .cra_driver_name = "safexcel-crc32",
1976                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1977                                 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY |
1978                                              CRYPTO_ALG_ASYNC |
1979                                              CRYPTO_ALG_ALLOCATES_MEMORY |
1980                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
1981                                 .cra_blocksize = 1,
1982                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1983                                 .cra_init = safexcel_crc32_cra_init,
1984                                 .cra_exit = safexcel_ahash_cra_exit,
1985                                 .cra_module = THIS_MODULE,
1986                         },
1987                 },
1988         },
1989 };
1990
1991 static int safexcel_cbcmac_init(struct ahash_request *areq)
1992 {
1993         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1994         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1995
1996         memset(req, 0, sizeof(*req));
1997
1998         /* Start from loaded keys */
1999         memcpy(req->state, &ctx->base.ipad, ctx->key_sz);
2000         /* Set processed to non-zero to enable invalidation detection */
2001         req->len        = AES_BLOCK_SIZE;
2002         req->processed  = AES_BLOCK_SIZE;
2003
2004         req->digest   = CONTEXT_CONTROL_DIGEST_XCM;
2005         req->state_sz = ctx->key_sz;
2006         req->digest_sz = AES_BLOCK_SIZE;
2007         req->block_sz = AES_BLOCK_SIZE;
2008         req->xcbcmac  = true;
2009
2010         return 0;
2011 }
2012
2013 static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2014                                  unsigned int len)
2015 {
2016         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2017         struct crypto_aes_ctx aes;
2018         int ret, i;
2019
2020         ret = aes_expandkey(&aes, key, len);
2021         if (ret)
2022                 return ret;
2023
2024         memset(&ctx->base.ipad, 0, 2 * AES_BLOCK_SIZE);
2025         for (i = 0; i < len / sizeof(u32); i++)
2026                 ctx->base.ipad.be[i + 8] = cpu_to_be32(aes.key_enc[i]);
2027
2028         if (len == AES_KEYSIZE_192) {
2029                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2030                 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2031         } else if (len == AES_KEYSIZE_256) {
2032                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2033                 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2034         } else {
2035                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2036                 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2037         }
2038         ctx->cbcmac  = true;
2039
2040         memzero_explicit(&aes, sizeof(aes));
2041         return 0;
2042 }
2043
2044 static int safexcel_cbcmac_digest(struct ahash_request *areq)
2045 {
2046         return safexcel_cbcmac_init(areq) ?: safexcel_ahash_finup(areq);
2047 }
2048
2049 struct safexcel_alg_template safexcel_alg_cbcmac = {
2050         .type = SAFEXCEL_ALG_TYPE_AHASH,
2051         .algo_mask = 0,
2052         .alg.ahash = {
2053                 .init = safexcel_cbcmac_init,
2054                 .update = safexcel_ahash_update,
2055                 .final = safexcel_ahash_final,
2056                 .finup = safexcel_ahash_finup,
2057                 .digest = safexcel_cbcmac_digest,
2058                 .setkey = safexcel_cbcmac_setkey,
2059                 .export = safexcel_ahash_export,
2060                 .import = safexcel_ahash_import,
2061                 .halg = {
2062                         .digestsize = AES_BLOCK_SIZE,
2063                         .statesize = sizeof(struct safexcel_ahash_export_state),
2064                         .base = {
2065                                 .cra_name = "cbcmac(aes)",
2066                                 .cra_driver_name = "safexcel-cbcmac-aes",
2067                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2068                                 .cra_flags = CRYPTO_ALG_ASYNC |
2069                                              CRYPTO_ALG_ALLOCATES_MEMORY |
2070                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
2071                                 .cra_blocksize = 1,
2072                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2073                                 .cra_init = safexcel_ahash_cra_init,
2074                                 .cra_exit = safexcel_ahash_cra_exit,
2075                                 .cra_module = THIS_MODULE,
2076                         },
2077                 },
2078         },
2079 };
2080
2081 static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2082                                  unsigned int len)
2083 {
2084         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2085         struct crypto_aes_ctx aes;
2086         u32 key_tmp[3 * AES_BLOCK_SIZE / sizeof(u32)];
2087         int ret, i;
2088
2089         ret = aes_expandkey(&aes, key, len);
2090         if (ret)
2091                 return ret;
2092
2093         /* precompute the XCBC key material */
2094         crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
2095         crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) &
2096                                 CRYPTO_TFM_REQ_MASK);
2097         ret = crypto_cipher_setkey(ctx->kaes, key, len);
2098         if (ret)
2099                 return ret;
2100
2101         crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
2102                 "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1");
2103         crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp,
2104                 "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2");
2105         crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + AES_BLOCK_SIZE,
2106                 "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3");
2107         for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++)
2108                 ctx->base.ipad.word[i] = swab(key_tmp[i]);
2109
2110         crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
2111         crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) &
2112                                 CRYPTO_TFM_REQ_MASK);
2113         ret = crypto_cipher_setkey(ctx->kaes,
2114                                    (u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
2115                                    AES_MIN_KEY_SIZE);
2116         if (ret)
2117                 return ret;
2118
2119         ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2120         ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2121         ctx->cbcmac = false;
2122
2123         memzero_explicit(&aes, sizeof(aes));
2124         return 0;
2125 }
2126
2127 static int safexcel_xcbcmac_cra_init(struct crypto_tfm *tfm)
2128 {
2129         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2130
2131         safexcel_ahash_cra_init(tfm);
2132         ctx->kaes = crypto_alloc_cipher("aes", 0, 0);
2133         return PTR_ERR_OR_ZERO(ctx->kaes);
2134 }
2135
2136 static void safexcel_xcbcmac_cra_exit(struct crypto_tfm *tfm)
2137 {
2138         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2139
2140         crypto_free_cipher(ctx->kaes);
2141         safexcel_ahash_cra_exit(tfm);
2142 }
2143
2144 struct safexcel_alg_template safexcel_alg_xcbcmac = {
2145         .type = SAFEXCEL_ALG_TYPE_AHASH,
2146         .algo_mask = 0,
2147         .alg.ahash = {
2148                 .init = safexcel_cbcmac_init,
2149                 .update = safexcel_ahash_update,
2150                 .final = safexcel_ahash_final,
2151                 .finup = safexcel_ahash_finup,
2152                 .digest = safexcel_cbcmac_digest,
2153                 .setkey = safexcel_xcbcmac_setkey,
2154                 .export = safexcel_ahash_export,
2155                 .import = safexcel_ahash_import,
2156                 .halg = {
2157                         .digestsize = AES_BLOCK_SIZE,
2158                         .statesize = sizeof(struct safexcel_ahash_export_state),
2159                         .base = {
2160                                 .cra_name = "xcbc(aes)",
2161                                 .cra_driver_name = "safexcel-xcbc-aes",
2162                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2163                                 .cra_flags = CRYPTO_ALG_ASYNC |
2164                                              CRYPTO_ALG_ALLOCATES_MEMORY |
2165                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
2166                                 .cra_blocksize = AES_BLOCK_SIZE,
2167                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2168                                 .cra_init = safexcel_xcbcmac_cra_init,
2169                                 .cra_exit = safexcel_xcbcmac_cra_exit,
2170                                 .cra_module = THIS_MODULE,
2171                         },
2172                 },
2173         },
2174 };
2175
2176 static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2177                                 unsigned int len)
2178 {
2179         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2180         struct crypto_aes_ctx aes;
2181         __be64 consts[4];
2182         u64 _const[2];
2183         u8 msb_mask, gfmask;
2184         int ret, i;
2185
2186         ret = aes_expandkey(&aes, key, len);
2187         if (ret)
2188                 return ret;
2189
2190         for (i = 0; i < len / sizeof(u32); i++)
2191                 ctx->base.ipad.word[i + 8] = swab(aes.key_enc[i]);
2192
2193         /* precompute the CMAC key material */
2194         crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
2195         crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) &
2196                                 CRYPTO_TFM_REQ_MASK);
2197         ret = crypto_cipher_setkey(ctx->kaes, key, len);
2198         if (ret)
2199                 return ret;
2200
2201         /* code below borrowed from crypto/cmac.c */
2202         /* encrypt the zero block */
2203         memset(consts, 0, AES_BLOCK_SIZE);
2204         crypto_cipher_encrypt_one(ctx->kaes, (u8 *)consts, (u8 *)consts);
2205
2206         gfmask = 0x87;
2207         _const[0] = be64_to_cpu(consts[1]);
2208         _const[1] = be64_to_cpu(consts[0]);
2209
2210         /* gf(2^128) multiply zero-ciphertext with u and u^2 */
2211         for (i = 0; i < 4; i += 2) {
2212                 msb_mask = ((s64)_const[1] >> 63) & gfmask;
2213                 _const[1] = (_const[1] << 1) | (_const[0] >> 63);
2214                 _const[0] = (_const[0] << 1) ^ msb_mask;
2215
2216                 consts[i + 0] = cpu_to_be64(_const[1]);
2217                 consts[i + 1] = cpu_to_be64(_const[0]);
2218         }
2219         /* end of code borrowed from crypto/cmac.c */
2220
2221         for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++)
2222                 ctx->base.ipad.be[i] = cpu_to_be32(((u32 *)consts)[i]);
2223
2224         if (len == AES_KEYSIZE_192) {
2225                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2226                 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2227         } else if (len == AES_KEYSIZE_256) {
2228                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2229                 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2230         } else {
2231                 ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2232                 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2233         }
2234         ctx->cbcmac = false;
2235
2236         memzero_explicit(&aes, sizeof(aes));
2237         return 0;
2238 }
2239
2240 struct safexcel_alg_template safexcel_alg_cmac = {
2241         .type = SAFEXCEL_ALG_TYPE_AHASH,
2242         .algo_mask = 0,
2243         .alg.ahash = {
2244                 .init = safexcel_cbcmac_init,
2245                 .update = safexcel_ahash_update,
2246                 .final = safexcel_ahash_final,
2247                 .finup = safexcel_ahash_finup,
2248                 .digest = safexcel_cbcmac_digest,
2249                 .setkey = safexcel_cmac_setkey,
2250                 .export = safexcel_ahash_export,
2251                 .import = safexcel_ahash_import,
2252                 .halg = {
2253                         .digestsize = AES_BLOCK_SIZE,
2254                         .statesize = sizeof(struct safexcel_ahash_export_state),
2255                         .base = {
2256                                 .cra_name = "cmac(aes)",
2257                                 .cra_driver_name = "safexcel-cmac-aes",
2258                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2259                                 .cra_flags = CRYPTO_ALG_ASYNC |
2260                                              CRYPTO_ALG_ALLOCATES_MEMORY |
2261                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
2262                                 .cra_blocksize = AES_BLOCK_SIZE,
2263                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2264                                 .cra_init = safexcel_xcbcmac_cra_init,
2265                                 .cra_exit = safexcel_xcbcmac_cra_exit,
2266                                 .cra_module = THIS_MODULE,
2267                         },
2268                 },
2269         },
2270 };
2271
2272 static int safexcel_sm3_init(struct ahash_request *areq)
2273 {
2274         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
2275         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2276
2277         memset(req, 0, sizeof(*req));
2278
2279         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
2280         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
2281         req->state_sz = SM3_DIGEST_SIZE;
2282         req->digest_sz = SM3_DIGEST_SIZE;
2283         req->block_sz = SM3_BLOCK_SIZE;
2284
2285         return 0;
2286 }
2287
2288 static int safexcel_sm3_digest(struct ahash_request *areq)
2289 {
2290         int ret = safexcel_sm3_init(areq);
2291
2292         if (ret)
2293                 return ret;
2294
2295         return safexcel_ahash_finup(areq);
2296 }
2297
2298 struct safexcel_alg_template safexcel_alg_sm3 = {
2299         .type = SAFEXCEL_ALG_TYPE_AHASH,
2300         .algo_mask = SAFEXCEL_ALG_SM3,
2301         .alg.ahash = {
2302                 .init = safexcel_sm3_init,
2303                 .update = safexcel_ahash_update,
2304                 .final = safexcel_ahash_final,
2305                 .finup = safexcel_ahash_finup,
2306                 .digest = safexcel_sm3_digest,
2307                 .export = safexcel_ahash_export,
2308                 .import = safexcel_ahash_import,
2309                 .halg = {
2310                         .digestsize = SM3_DIGEST_SIZE,
2311                         .statesize = sizeof(struct safexcel_ahash_export_state),
2312                         .base = {
2313                                 .cra_name = "sm3",
2314                                 .cra_driver_name = "safexcel-sm3",
2315                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2316                                 .cra_flags = CRYPTO_ALG_ASYNC |
2317                                              CRYPTO_ALG_ALLOCATES_MEMORY |
2318                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
2319                                 .cra_blocksize = SM3_BLOCK_SIZE,
2320                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2321                                 .cra_init = safexcel_ahash_cra_init,
2322                                 .cra_exit = safexcel_ahash_cra_exit,
2323                                 .cra_module = THIS_MODULE,
2324                         },
2325                 },
2326         },
2327 };
2328
2329 static int safexcel_hmac_sm3_setkey(struct crypto_ahash *tfm, const u8 *key,
2330                                     unsigned int keylen)
2331 {
2332         return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sm3",
2333                                         SM3_DIGEST_SIZE);
2334 }
2335
2336 static int safexcel_hmac_sm3_init(struct ahash_request *areq)
2337 {
2338         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
2339         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2340
2341         memset(req, 0, sizeof(*req));
2342
2343         /* Start from ipad precompute */
2344         memcpy(req->state, &ctx->base.ipad, SM3_DIGEST_SIZE);
2345         /* Already processed the key^ipad part now! */
2346         req->len        = SM3_BLOCK_SIZE;
2347         req->processed  = SM3_BLOCK_SIZE;
2348
2349         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
2350         req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
2351         req->state_sz = SM3_DIGEST_SIZE;
2352         req->digest_sz = SM3_DIGEST_SIZE;
2353         req->block_sz = SM3_BLOCK_SIZE;
2354         req->hmac = true;
2355
2356         return 0;
2357 }
2358
2359 static int safexcel_hmac_sm3_digest(struct ahash_request *areq)
2360 {
2361         int ret = safexcel_hmac_sm3_init(areq);
2362
2363         if (ret)
2364                 return ret;
2365
2366         return safexcel_ahash_finup(areq);
2367 }
2368
2369 struct safexcel_alg_template safexcel_alg_hmac_sm3 = {
2370         .type = SAFEXCEL_ALG_TYPE_AHASH,
2371         .algo_mask = SAFEXCEL_ALG_SM3,
2372         .alg.ahash = {
2373                 .init = safexcel_hmac_sm3_init,
2374                 .update = safexcel_ahash_update,
2375                 .final = safexcel_ahash_final,
2376                 .finup = safexcel_ahash_finup,
2377                 .digest = safexcel_hmac_sm3_digest,
2378                 .setkey = safexcel_hmac_sm3_setkey,
2379                 .export = safexcel_ahash_export,
2380                 .import = safexcel_ahash_import,
2381                 .halg = {
2382                         .digestsize = SM3_DIGEST_SIZE,
2383                         .statesize = sizeof(struct safexcel_ahash_export_state),
2384                         .base = {
2385                                 .cra_name = "hmac(sm3)",
2386                                 .cra_driver_name = "safexcel-hmac-sm3",
2387                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2388                                 .cra_flags = CRYPTO_ALG_ASYNC |
2389                                              CRYPTO_ALG_ALLOCATES_MEMORY |
2390                                              CRYPTO_ALG_KERN_DRIVER_ONLY,
2391                                 .cra_blocksize = SM3_BLOCK_SIZE,
2392                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2393                                 .cra_init = safexcel_ahash_cra_init,
2394                                 .cra_exit = safexcel_ahash_cra_exit,
2395                                 .cra_module = THIS_MODULE,
2396                         },
2397                 },
2398         },
2399 };
2400
2401 static int safexcel_sha3_224_init(struct ahash_request *areq)
2402 {
2403         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2404         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2405         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2406
2407         memset(req, 0, sizeof(*req));
2408
2409         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224;
2410         req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2411         req->state_sz = SHA3_224_DIGEST_SIZE;
2412         req->digest_sz = SHA3_224_DIGEST_SIZE;
2413         req->block_sz = SHA3_224_BLOCK_SIZE;
2414         ctx->do_fallback = false;
2415         ctx->fb_init_done = false;
2416         return 0;
2417 }
2418
2419 static int safexcel_sha3_fbcheck(struct ahash_request *req)
2420 {
2421         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2422         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2423         struct ahash_request *subreq = ahash_request_ctx(req);
2424         int ret = 0;
2425
2426         if (ctx->do_fallback) {
2427                 ahash_request_set_tfm(subreq, ctx->fback);
2428                 ahash_request_set_callback(subreq, req->base.flags,
2429                                            req->base.complete, req->base.data);
2430                 ahash_request_set_crypt(subreq, req->src, req->result,
2431                                         req->nbytes);
2432                 if (!ctx->fb_init_done) {
2433                         if (ctx->fb_do_setkey) {
2434                                 /* Set fallback cipher HMAC key */
2435                                 u8 key[SHA3_224_BLOCK_SIZE];
2436
2437                                 memcpy(key, &ctx->base.ipad,
2438                                        crypto_ahash_blocksize(ctx->fback) / 2);
2439                                 memcpy(key +
2440                                        crypto_ahash_blocksize(ctx->fback) / 2,
2441                                        &ctx->base.opad,
2442                                        crypto_ahash_blocksize(ctx->fback) / 2);
2443                                 ret = crypto_ahash_setkey(ctx->fback, key,
2444                                         crypto_ahash_blocksize(ctx->fback));
2445                                 memzero_explicit(key,
2446                                         crypto_ahash_blocksize(ctx->fback));
2447                                 ctx->fb_do_setkey = false;
2448                         }
2449                         ret = ret ?: crypto_ahash_init(subreq);
2450                         ctx->fb_init_done = true;
2451                 }
2452         }
2453         return ret;
2454 }
2455
2456 static int safexcel_sha3_update(struct ahash_request *req)
2457 {
2458         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2459         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2460         struct ahash_request *subreq = ahash_request_ctx(req);
2461
2462         ctx->do_fallback = true;
2463         return safexcel_sha3_fbcheck(req) ?: crypto_ahash_update(subreq);
2464 }
2465
2466 static int safexcel_sha3_final(struct ahash_request *req)
2467 {
2468         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2469         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2470         struct ahash_request *subreq = ahash_request_ctx(req);
2471
2472         ctx->do_fallback = true;
2473         return safexcel_sha3_fbcheck(req) ?: crypto_ahash_final(subreq);
2474 }
2475
2476 static int safexcel_sha3_finup(struct ahash_request *req)
2477 {
2478         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2479         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2480         struct ahash_request *subreq = ahash_request_ctx(req);
2481
2482         ctx->do_fallback |= !req->nbytes;
2483         if (ctx->do_fallback)
2484                 /* Update or ex/import happened or len 0, cannot use the HW */
2485                 return safexcel_sha3_fbcheck(req) ?:
2486                        crypto_ahash_finup(subreq);
2487         else
2488                 return safexcel_ahash_finup(req);
2489 }
2490
2491 static int safexcel_sha3_digest_fallback(struct ahash_request *req)
2492 {
2493         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2494         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2495         struct ahash_request *subreq = ahash_request_ctx(req);
2496
2497         ctx->do_fallback = true;
2498         ctx->fb_init_done = false;
2499         return safexcel_sha3_fbcheck(req) ?: crypto_ahash_finup(subreq);
2500 }
2501
2502 static int safexcel_sha3_224_digest(struct ahash_request *req)
2503 {
2504         if (req->nbytes)
2505                 return safexcel_sha3_224_init(req) ?: safexcel_ahash_finup(req);
2506
2507         /* HW cannot do zero length hash, use fallback instead */
2508         return safexcel_sha3_digest_fallback(req);
2509 }
2510
2511 static int safexcel_sha3_export(struct ahash_request *req, void *out)
2512 {
2513         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2514         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2515         struct ahash_request *subreq = ahash_request_ctx(req);
2516
2517         ctx->do_fallback = true;
2518         return safexcel_sha3_fbcheck(req) ?: crypto_ahash_export(subreq, out);
2519 }
2520
2521 static int safexcel_sha3_import(struct ahash_request *req, const void *in)
2522 {
2523         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2524         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2525         struct ahash_request *subreq = ahash_request_ctx(req);
2526
2527         ctx->do_fallback = true;
2528         return safexcel_sha3_fbcheck(req) ?: crypto_ahash_import(subreq, in);
2529         // return safexcel_ahash_import(req, in);
2530 }
2531
2532 static int safexcel_sha3_cra_init(struct crypto_tfm *tfm)
2533 {
2534         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
2535         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2536
2537         safexcel_ahash_cra_init(tfm);
2538
2539         /* Allocate fallback implementation */
2540         ctx->fback = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0,
2541                                         CRYPTO_ALG_ASYNC |
2542                                         CRYPTO_ALG_NEED_FALLBACK);
2543         if (IS_ERR(ctx->fback))
2544                 return PTR_ERR(ctx->fback);
2545
2546         /* Update statesize from fallback algorithm! */
2547         crypto_hash_alg_common(ahash)->statesize =
2548                 crypto_ahash_statesize(ctx->fback);
2549         crypto_ahash_set_reqsize(ahash, max(sizeof(struct safexcel_ahash_req),
2550                                             sizeof(struct ahash_request) +
2551                                             crypto_ahash_reqsize(ctx->fback)));
2552         return 0;
2553 }
2554
2555 static void safexcel_sha3_cra_exit(struct crypto_tfm *tfm)
2556 {
2557         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2558
2559         crypto_free_ahash(ctx->fback);
2560         safexcel_ahash_cra_exit(tfm);
2561 }
2562
2563 struct safexcel_alg_template safexcel_alg_sha3_224 = {
2564         .type = SAFEXCEL_ALG_TYPE_AHASH,
2565         .algo_mask = SAFEXCEL_ALG_SHA3,
2566         .alg.ahash = {
2567                 .init = safexcel_sha3_224_init,
2568                 .update = safexcel_sha3_update,
2569                 .final = safexcel_sha3_final,
2570                 .finup = safexcel_sha3_finup,
2571                 .digest = safexcel_sha3_224_digest,
2572                 .export = safexcel_sha3_export,
2573                 .import = safexcel_sha3_import,
2574                 .halg = {
2575                         .digestsize = SHA3_224_DIGEST_SIZE,
2576                         .statesize = sizeof(struct safexcel_ahash_export_state),
2577                         .base = {
2578                                 .cra_name = "sha3-224",
2579                                 .cra_driver_name = "safexcel-sha3-224",
2580                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2581                                 .cra_flags = CRYPTO_ALG_ASYNC |
2582                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2583                                              CRYPTO_ALG_NEED_FALLBACK,
2584                                 .cra_blocksize = SHA3_224_BLOCK_SIZE,
2585                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2586                                 .cra_init = safexcel_sha3_cra_init,
2587                                 .cra_exit = safexcel_sha3_cra_exit,
2588                                 .cra_module = THIS_MODULE,
2589                         },
2590                 },
2591         },
2592 };
2593
2594 static int safexcel_sha3_256_init(struct ahash_request *areq)
2595 {
2596         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2597         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2598         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2599
2600         memset(req, 0, sizeof(*req));
2601
2602         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256;
2603         req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2604         req->state_sz = SHA3_256_DIGEST_SIZE;
2605         req->digest_sz = SHA3_256_DIGEST_SIZE;
2606         req->block_sz = SHA3_256_BLOCK_SIZE;
2607         ctx->do_fallback = false;
2608         ctx->fb_init_done = false;
2609         return 0;
2610 }
2611
2612 static int safexcel_sha3_256_digest(struct ahash_request *req)
2613 {
2614         if (req->nbytes)
2615                 return safexcel_sha3_256_init(req) ?: safexcel_ahash_finup(req);
2616
2617         /* HW cannot do zero length hash, use fallback instead */
2618         return safexcel_sha3_digest_fallback(req);
2619 }
2620
2621 struct safexcel_alg_template safexcel_alg_sha3_256 = {
2622         .type = SAFEXCEL_ALG_TYPE_AHASH,
2623         .algo_mask = SAFEXCEL_ALG_SHA3,
2624         .alg.ahash = {
2625                 .init = safexcel_sha3_256_init,
2626                 .update = safexcel_sha3_update,
2627                 .final = safexcel_sha3_final,
2628                 .finup = safexcel_sha3_finup,
2629                 .digest = safexcel_sha3_256_digest,
2630                 .export = safexcel_sha3_export,
2631                 .import = safexcel_sha3_import,
2632                 .halg = {
2633                         .digestsize = SHA3_256_DIGEST_SIZE,
2634                         .statesize = sizeof(struct safexcel_ahash_export_state),
2635                         .base = {
2636                                 .cra_name = "sha3-256",
2637                                 .cra_driver_name = "safexcel-sha3-256",
2638                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2639                                 .cra_flags = CRYPTO_ALG_ASYNC |
2640                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2641                                              CRYPTO_ALG_NEED_FALLBACK,
2642                                 .cra_blocksize = SHA3_256_BLOCK_SIZE,
2643                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2644                                 .cra_init = safexcel_sha3_cra_init,
2645                                 .cra_exit = safexcel_sha3_cra_exit,
2646                                 .cra_module = THIS_MODULE,
2647                         },
2648                 },
2649         },
2650 };
2651
2652 static int safexcel_sha3_384_init(struct ahash_request *areq)
2653 {
2654         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2655         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2656         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2657
2658         memset(req, 0, sizeof(*req));
2659
2660         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384;
2661         req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2662         req->state_sz = SHA3_384_DIGEST_SIZE;
2663         req->digest_sz = SHA3_384_DIGEST_SIZE;
2664         req->block_sz = SHA3_384_BLOCK_SIZE;
2665         ctx->do_fallback = false;
2666         ctx->fb_init_done = false;
2667         return 0;
2668 }
2669
2670 static int safexcel_sha3_384_digest(struct ahash_request *req)
2671 {
2672         if (req->nbytes)
2673                 return safexcel_sha3_384_init(req) ?: safexcel_ahash_finup(req);
2674
2675         /* HW cannot do zero length hash, use fallback instead */
2676         return safexcel_sha3_digest_fallback(req);
2677 }
2678
2679 struct safexcel_alg_template safexcel_alg_sha3_384 = {
2680         .type = SAFEXCEL_ALG_TYPE_AHASH,
2681         .algo_mask = SAFEXCEL_ALG_SHA3,
2682         .alg.ahash = {
2683                 .init = safexcel_sha3_384_init,
2684                 .update = safexcel_sha3_update,
2685                 .final = safexcel_sha3_final,
2686                 .finup = safexcel_sha3_finup,
2687                 .digest = safexcel_sha3_384_digest,
2688                 .export = safexcel_sha3_export,
2689                 .import = safexcel_sha3_import,
2690                 .halg = {
2691                         .digestsize = SHA3_384_DIGEST_SIZE,
2692                         .statesize = sizeof(struct safexcel_ahash_export_state),
2693                         .base = {
2694                                 .cra_name = "sha3-384",
2695                                 .cra_driver_name = "safexcel-sha3-384",
2696                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2697                                 .cra_flags = CRYPTO_ALG_ASYNC |
2698                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2699                                              CRYPTO_ALG_NEED_FALLBACK,
2700                                 .cra_blocksize = SHA3_384_BLOCK_SIZE,
2701                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2702                                 .cra_init = safexcel_sha3_cra_init,
2703                                 .cra_exit = safexcel_sha3_cra_exit,
2704                                 .cra_module = THIS_MODULE,
2705                         },
2706                 },
2707         },
2708 };
2709
2710 static int safexcel_sha3_512_init(struct ahash_request *areq)
2711 {
2712         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2713         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2714         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2715
2716         memset(req, 0, sizeof(*req));
2717
2718         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512;
2719         req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2720         req->state_sz = SHA3_512_DIGEST_SIZE;
2721         req->digest_sz = SHA3_512_DIGEST_SIZE;
2722         req->block_sz = SHA3_512_BLOCK_SIZE;
2723         ctx->do_fallback = false;
2724         ctx->fb_init_done = false;
2725         return 0;
2726 }
2727
2728 static int safexcel_sha3_512_digest(struct ahash_request *req)
2729 {
2730         if (req->nbytes)
2731                 return safexcel_sha3_512_init(req) ?: safexcel_ahash_finup(req);
2732
2733         /* HW cannot do zero length hash, use fallback instead */
2734         return safexcel_sha3_digest_fallback(req);
2735 }
2736
2737 struct safexcel_alg_template safexcel_alg_sha3_512 = {
2738         .type = SAFEXCEL_ALG_TYPE_AHASH,
2739         .algo_mask = SAFEXCEL_ALG_SHA3,
2740         .alg.ahash = {
2741                 .init = safexcel_sha3_512_init,
2742                 .update = safexcel_sha3_update,
2743                 .final = safexcel_sha3_final,
2744                 .finup = safexcel_sha3_finup,
2745                 .digest = safexcel_sha3_512_digest,
2746                 .export = safexcel_sha3_export,
2747                 .import = safexcel_sha3_import,
2748                 .halg = {
2749                         .digestsize = SHA3_512_DIGEST_SIZE,
2750                         .statesize = sizeof(struct safexcel_ahash_export_state),
2751                         .base = {
2752                                 .cra_name = "sha3-512",
2753                                 .cra_driver_name = "safexcel-sha3-512",
2754                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2755                                 .cra_flags = CRYPTO_ALG_ASYNC |
2756                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2757                                              CRYPTO_ALG_NEED_FALLBACK,
2758                                 .cra_blocksize = SHA3_512_BLOCK_SIZE,
2759                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2760                                 .cra_init = safexcel_sha3_cra_init,
2761                                 .cra_exit = safexcel_sha3_cra_exit,
2762                                 .cra_module = THIS_MODULE,
2763                         },
2764                 },
2765         },
2766 };
2767
2768 static int safexcel_hmac_sha3_cra_init(struct crypto_tfm *tfm, const char *alg)
2769 {
2770         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2771         int ret;
2772
2773         ret = safexcel_sha3_cra_init(tfm);
2774         if (ret)
2775                 return ret;
2776
2777         /* Allocate precalc basic digest implementation */
2778         ctx->shpre = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
2779         if (IS_ERR(ctx->shpre))
2780                 return PTR_ERR(ctx->shpre);
2781
2782         ctx->shdesc = kmalloc(sizeof(*ctx->shdesc) +
2783                               crypto_shash_descsize(ctx->shpre), GFP_KERNEL);
2784         if (!ctx->shdesc) {
2785                 crypto_free_shash(ctx->shpre);
2786                 return -ENOMEM;
2787         }
2788         ctx->shdesc->tfm = ctx->shpre;
2789         return 0;
2790 }
2791
2792 static void safexcel_hmac_sha3_cra_exit(struct crypto_tfm *tfm)
2793 {
2794         struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2795
2796         crypto_free_ahash(ctx->fback);
2797         crypto_free_shash(ctx->shpre);
2798         kfree(ctx->shdesc);
2799         safexcel_ahash_cra_exit(tfm);
2800 }
2801
2802 static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key,
2803                                      unsigned int keylen)
2804 {
2805         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2806         int ret = 0;
2807
2808         if (keylen > crypto_ahash_blocksize(tfm)) {
2809                 /*
2810                  * If the key is larger than the blocksize, then hash it
2811                  * first using our fallback cipher
2812                  */
2813                 ret = crypto_shash_digest(ctx->shdesc, key, keylen,
2814                                           ctx->base.ipad.byte);
2815                 keylen = crypto_shash_digestsize(ctx->shpre);
2816
2817                 /*
2818                  * If the digest is larger than half the blocksize, we need to
2819                  * move the rest to opad due to the way our HMAC infra works.
2820                  */
2821                 if (keylen > crypto_ahash_blocksize(tfm) / 2)
2822                         /* Buffers overlap, need to use memmove iso memcpy! */
2823                         memmove(&ctx->base.opad,
2824                                 ctx->base.ipad.byte +
2825                                         crypto_ahash_blocksize(tfm) / 2,
2826                                 keylen - crypto_ahash_blocksize(tfm) / 2);
2827         } else {
2828                 /*
2829                  * Copy the key to our ipad & opad buffers
2830                  * Note that ipad and opad each contain one half of the key,
2831                  * to match the existing HMAC driver infrastructure.
2832                  */
2833                 if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
2834                         memcpy(&ctx->base.ipad, key, keylen);
2835                 } else {
2836                         memcpy(&ctx->base.ipad, key,
2837                                crypto_ahash_blocksize(tfm) / 2);
2838                         memcpy(&ctx->base.opad,
2839                                key + crypto_ahash_blocksize(tfm) / 2,
2840                                keylen - crypto_ahash_blocksize(tfm) / 2);
2841                 }
2842         }
2843
2844         /* Pad key with zeroes */
2845         if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
2846                 memset(ctx->base.ipad.byte + keylen, 0,
2847                        crypto_ahash_blocksize(tfm) / 2 - keylen);
2848                 memset(&ctx->base.opad, 0, crypto_ahash_blocksize(tfm) / 2);
2849         } else {
2850                 memset(ctx->base.opad.byte + keylen -
2851                        crypto_ahash_blocksize(tfm) / 2, 0,
2852                        crypto_ahash_blocksize(tfm) - keylen);
2853         }
2854
2855         /* If doing fallback, still need to set the new key! */
2856         ctx->fb_do_setkey = true;
2857         return ret;
2858 }
2859
2860 static int safexcel_hmac_sha3_224_init(struct ahash_request *areq)
2861 {
2862         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2863         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2864         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2865
2866         memset(req, 0, sizeof(*req));
2867
2868         /* Copy (half of) the key */
2869         memcpy(req->state, &ctx->base.ipad, SHA3_224_BLOCK_SIZE / 2);
2870         /* Start of HMAC should have len == processed == blocksize */
2871         req->len        = SHA3_224_BLOCK_SIZE;
2872         req->processed  = SHA3_224_BLOCK_SIZE;
2873         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224;
2874         req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2875         req->state_sz = SHA3_224_BLOCK_SIZE / 2;
2876         req->digest_sz = SHA3_224_DIGEST_SIZE;
2877         req->block_sz = SHA3_224_BLOCK_SIZE;
2878         req->hmac = true;
2879         ctx->do_fallback = false;
2880         ctx->fb_init_done = false;
2881         return 0;
2882 }
2883
2884 static int safexcel_hmac_sha3_224_digest(struct ahash_request *req)
2885 {
2886         if (req->nbytes)
2887                 return safexcel_hmac_sha3_224_init(req) ?:
2888                        safexcel_ahash_finup(req);
2889
2890         /* HW cannot do zero length HMAC, use fallback instead */
2891         return safexcel_sha3_digest_fallback(req);
2892 }
2893
2894 static int safexcel_hmac_sha3_224_cra_init(struct crypto_tfm *tfm)
2895 {
2896         return safexcel_hmac_sha3_cra_init(tfm, "sha3-224");
2897 }
2898
2899 struct safexcel_alg_template safexcel_alg_hmac_sha3_224 = {
2900         .type = SAFEXCEL_ALG_TYPE_AHASH,
2901         .algo_mask = SAFEXCEL_ALG_SHA3,
2902         .alg.ahash = {
2903                 .init = safexcel_hmac_sha3_224_init,
2904                 .update = safexcel_sha3_update,
2905                 .final = safexcel_sha3_final,
2906                 .finup = safexcel_sha3_finup,
2907                 .digest = safexcel_hmac_sha3_224_digest,
2908                 .setkey = safexcel_hmac_sha3_setkey,
2909                 .export = safexcel_sha3_export,
2910                 .import = safexcel_sha3_import,
2911                 .halg = {
2912                         .digestsize = SHA3_224_DIGEST_SIZE,
2913                         .statesize = sizeof(struct safexcel_ahash_export_state),
2914                         .base = {
2915                                 .cra_name = "hmac(sha3-224)",
2916                                 .cra_driver_name = "safexcel-hmac-sha3-224",
2917                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2918                                 .cra_flags = CRYPTO_ALG_ASYNC |
2919                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2920                                              CRYPTO_ALG_NEED_FALLBACK,
2921                                 .cra_blocksize = SHA3_224_BLOCK_SIZE,
2922                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2923                                 .cra_init = safexcel_hmac_sha3_224_cra_init,
2924                                 .cra_exit = safexcel_hmac_sha3_cra_exit,
2925                                 .cra_module = THIS_MODULE,
2926                         },
2927                 },
2928         },
2929 };
2930
2931 static int safexcel_hmac_sha3_256_init(struct ahash_request *areq)
2932 {
2933         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2934         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2935         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2936
2937         memset(req, 0, sizeof(*req));
2938
2939         /* Copy (half of) the key */
2940         memcpy(req->state, &ctx->base.ipad, SHA3_256_BLOCK_SIZE / 2);
2941         /* Start of HMAC should have len == processed == blocksize */
2942         req->len        = SHA3_256_BLOCK_SIZE;
2943         req->processed  = SHA3_256_BLOCK_SIZE;
2944         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256;
2945         req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2946         req->state_sz = SHA3_256_BLOCK_SIZE / 2;
2947         req->digest_sz = SHA3_256_DIGEST_SIZE;
2948         req->block_sz = SHA3_256_BLOCK_SIZE;
2949         req->hmac = true;
2950         ctx->do_fallback = false;
2951         ctx->fb_init_done = false;
2952         return 0;
2953 }
2954
2955 static int safexcel_hmac_sha3_256_digest(struct ahash_request *req)
2956 {
2957         if (req->nbytes)
2958                 return safexcel_hmac_sha3_256_init(req) ?:
2959                        safexcel_ahash_finup(req);
2960
2961         /* HW cannot do zero length HMAC, use fallback instead */
2962         return safexcel_sha3_digest_fallback(req);
2963 }
2964
2965 static int safexcel_hmac_sha3_256_cra_init(struct crypto_tfm *tfm)
2966 {
2967         return safexcel_hmac_sha3_cra_init(tfm, "sha3-256");
2968 }
2969
2970 struct safexcel_alg_template safexcel_alg_hmac_sha3_256 = {
2971         .type = SAFEXCEL_ALG_TYPE_AHASH,
2972         .algo_mask = SAFEXCEL_ALG_SHA3,
2973         .alg.ahash = {
2974                 .init = safexcel_hmac_sha3_256_init,
2975                 .update = safexcel_sha3_update,
2976                 .final = safexcel_sha3_final,
2977                 .finup = safexcel_sha3_finup,
2978                 .digest = safexcel_hmac_sha3_256_digest,
2979                 .setkey = safexcel_hmac_sha3_setkey,
2980                 .export = safexcel_sha3_export,
2981                 .import = safexcel_sha3_import,
2982                 .halg = {
2983                         .digestsize = SHA3_256_DIGEST_SIZE,
2984                         .statesize = sizeof(struct safexcel_ahash_export_state),
2985                         .base = {
2986                                 .cra_name = "hmac(sha3-256)",
2987                                 .cra_driver_name = "safexcel-hmac-sha3-256",
2988                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2989                                 .cra_flags = CRYPTO_ALG_ASYNC |
2990                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
2991                                              CRYPTO_ALG_NEED_FALLBACK,
2992                                 .cra_blocksize = SHA3_256_BLOCK_SIZE,
2993                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2994                                 .cra_init = safexcel_hmac_sha3_256_cra_init,
2995                                 .cra_exit = safexcel_hmac_sha3_cra_exit,
2996                                 .cra_module = THIS_MODULE,
2997                         },
2998                 },
2999         },
3000 };
3001
3002 static int safexcel_hmac_sha3_384_init(struct ahash_request *areq)
3003 {
3004         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
3005         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
3006         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
3007
3008         memset(req, 0, sizeof(*req));
3009
3010         /* Copy (half of) the key */
3011         memcpy(req->state, &ctx->base.ipad, SHA3_384_BLOCK_SIZE / 2);
3012         /* Start of HMAC should have len == processed == blocksize */
3013         req->len        = SHA3_384_BLOCK_SIZE;
3014         req->processed  = SHA3_384_BLOCK_SIZE;
3015         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384;
3016         req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
3017         req->state_sz = SHA3_384_BLOCK_SIZE / 2;
3018         req->digest_sz = SHA3_384_DIGEST_SIZE;
3019         req->block_sz = SHA3_384_BLOCK_SIZE;
3020         req->hmac = true;
3021         ctx->do_fallback = false;
3022         ctx->fb_init_done = false;
3023         return 0;
3024 }
3025
3026 static int safexcel_hmac_sha3_384_digest(struct ahash_request *req)
3027 {
3028         if (req->nbytes)
3029                 return safexcel_hmac_sha3_384_init(req) ?:
3030                        safexcel_ahash_finup(req);
3031
3032         /* HW cannot do zero length HMAC, use fallback instead */
3033         return safexcel_sha3_digest_fallback(req);
3034 }
3035
3036 static int safexcel_hmac_sha3_384_cra_init(struct crypto_tfm *tfm)
3037 {
3038         return safexcel_hmac_sha3_cra_init(tfm, "sha3-384");
3039 }
3040
3041 struct safexcel_alg_template safexcel_alg_hmac_sha3_384 = {
3042         .type = SAFEXCEL_ALG_TYPE_AHASH,
3043         .algo_mask = SAFEXCEL_ALG_SHA3,
3044         .alg.ahash = {
3045                 .init = safexcel_hmac_sha3_384_init,
3046                 .update = safexcel_sha3_update,
3047                 .final = safexcel_sha3_final,
3048                 .finup = safexcel_sha3_finup,
3049                 .digest = safexcel_hmac_sha3_384_digest,
3050                 .setkey = safexcel_hmac_sha3_setkey,
3051                 .export = safexcel_sha3_export,
3052                 .import = safexcel_sha3_import,
3053                 .halg = {
3054                         .digestsize = SHA3_384_DIGEST_SIZE,
3055                         .statesize = sizeof(struct safexcel_ahash_export_state),
3056                         .base = {
3057                                 .cra_name = "hmac(sha3-384)",
3058                                 .cra_driver_name = "safexcel-hmac-sha3-384",
3059                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
3060                                 .cra_flags = CRYPTO_ALG_ASYNC |
3061                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
3062                                              CRYPTO_ALG_NEED_FALLBACK,
3063                                 .cra_blocksize = SHA3_384_BLOCK_SIZE,
3064                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
3065                                 .cra_init = safexcel_hmac_sha3_384_cra_init,
3066                                 .cra_exit = safexcel_hmac_sha3_cra_exit,
3067                                 .cra_module = THIS_MODULE,
3068                         },
3069                 },
3070         },
3071 };
3072
3073 static int safexcel_hmac_sha3_512_init(struct ahash_request *areq)
3074 {
3075         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
3076         struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
3077         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
3078
3079         memset(req, 0, sizeof(*req));
3080
3081         /* Copy (half of) the key */
3082         memcpy(req->state, &ctx->base.ipad, SHA3_512_BLOCK_SIZE / 2);
3083         /* Start of HMAC should have len == processed == blocksize */
3084         req->len        = SHA3_512_BLOCK_SIZE;
3085         req->processed  = SHA3_512_BLOCK_SIZE;
3086         ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512;
3087         req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
3088         req->state_sz = SHA3_512_BLOCK_SIZE / 2;
3089         req->digest_sz = SHA3_512_DIGEST_SIZE;
3090         req->block_sz = SHA3_512_BLOCK_SIZE;
3091         req->hmac = true;
3092         ctx->do_fallback = false;
3093         ctx->fb_init_done = false;
3094         return 0;
3095 }
3096
3097 static int safexcel_hmac_sha3_512_digest(struct ahash_request *req)
3098 {
3099         if (req->nbytes)
3100                 return safexcel_hmac_sha3_512_init(req) ?:
3101                        safexcel_ahash_finup(req);
3102
3103         /* HW cannot do zero length HMAC, use fallback instead */
3104         return safexcel_sha3_digest_fallback(req);
3105 }
3106
3107 static int safexcel_hmac_sha3_512_cra_init(struct crypto_tfm *tfm)
3108 {
3109         return safexcel_hmac_sha3_cra_init(tfm, "sha3-512");
3110 }
3111 struct safexcel_alg_template safexcel_alg_hmac_sha3_512 = {
3112         .type = SAFEXCEL_ALG_TYPE_AHASH,
3113         .algo_mask = SAFEXCEL_ALG_SHA3,
3114         .alg.ahash = {
3115                 .init = safexcel_hmac_sha3_512_init,
3116                 .update = safexcel_sha3_update,
3117                 .final = safexcel_sha3_final,
3118                 .finup = safexcel_sha3_finup,
3119                 .digest = safexcel_hmac_sha3_512_digest,
3120                 .setkey = safexcel_hmac_sha3_setkey,
3121                 .export = safexcel_sha3_export,
3122                 .import = safexcel_sha3_import,
3123                 .halg = {
3124                         .digestsize = SHA3_512_DIGEST_SIZE,
3125                         .statesize = sizeof(struct safexcel_ahash_export_state),
3126                         .base = {
3127                                 .cra_name = "hmac(sha3-512)",
3128                                 .cra_driver_name = "safexcel-hmac-sha3-512",
3129                                 .cra_priority = SAFEXCEL_CRA_PRIORITY,
3130                                 .cra_flags = CRYPTO_ALG_ASYNC |
3131                                              CRYPTO_ALG_KERN_DRIVER_ONLY |
3132                                              CRYPTO_ALG_NEED_FALLBACK,
3133                                 .cra_blocksize = SHA3_512_BLOCK_SIZE,
3134                                 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
3135                                 .cra_init = safexcel_hmac_sha3_512_cra_init,
3136                                 .cra_exit = safexcel_hmac_sha3_cra_exit,
3137                                 .cra_module = THIS_MODULE,
3138                         },
3139                 },
3140         },
3141 };