drivers/crypto/inside-secure/safexcel_cipher.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2017 Marvell
   4  *
   5  * Antoine Tenart <[email protected]>
   6  */
   7
   8 #include <asm/unaligned.h>
   9 #include <linux/device.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/dmapool.h>
  12 #include <crypto/aead.h>
  13 #include <crypto/aes.h>
  14 #include <crypto/authenc.h>
  15 #include <crypto/chacha.h>
  16 #include <crypto/ctr.h>
  17 #include <crypto/internal/des.h>
  18 #include <crypto/gcm.h>
  19 #include <crypto/ghash.h>
  20 #include <crypto/poly1305.h>
  21 #include <crypto/sha.h>
  22 #include <crypto/sm3.h>
  23 #include <crypto/sm4.h>
  24 #include <crypto/xts.h>
  25 #include <crypto/skcipher.h>
  26 #include <crypto/internal/aead.h>
  27 #include <crypto/internal/skcipher.h>
  28
  29 #include "safexcel.h"
  30
  31 enum safexcel_cipher_direction {
  32         SAFEXCEL_ENCRYPT,
  33         SAFEXCEL_DECRYPT,
  34 };
  35
  36 enum safexcel_cipher_alg {
  37         SAFEXCEL_DES,
  38         SAFEXCEL_3DES,
  39         SAFEXCEL_AES,
  40         SAFEXCEL_CHACHA20,
  41         SAFEXCEL_SM4,
  42 };
  43
  44 struct safexcel_cipher_ctx {
  45         struct safexcel_context base;
  46         struct safexcel_crypto_priv *priv;
  47
  48         u32 mode;
  49         enum safexcel_cipher_alg alg;
  50         u8 aead; /* !=0=AEAD, 2=IPSec ESP AEAD, 3=IPsec ESP GMAC */
  51         u8 xcm;  /* 0=authenc, 1=GCM, 2 reserved for CCM */
  52         u8 aadskip;
  53         u8 blocksz;
  54         u32 ivmask;
  55         u32 ctrinit;
  56
  57         __le32 key[16];
  58         u32 nonce;
  59         unsigned int key_len, xts;
  60
  61         /* All the below is AEAD specific */
  62         u32 hash_alg;
  63         u32 state_sz;
  64         __be32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)];
  65         __be32 opad[SHA512_DIGEST_SIZE / sizeof(u32)];
  66
  67         struct crypto_cipher *hkaes;
  68         struct crypto_aead *fback;
  69 };
  70
  71 struct safexcel_cipher_req {
  72         enum safexcel_cipher_direction direction;
  73         /* Number of result descriptors associated to the request */
  74         unsigned int rdescs;
  75         bool needs_inv;
  76         int  nr_src, nr_dst;
  77 };
  78
  79 static int safexcel_skcipher_iv(struct safexcel_cipher_ctx *ctx, u8 *iv,
  80                                 struct safexcel_command_desc *cdesc)
  81 {
  82         if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) {
  83                 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
  84                 /* 32 bit nonce */
  85                 cdesc->control_data.token[0] = ctx->nonce;
  86                 /* 64 bit IV part */
  87                 memcpy(&cdesc->control_data.token[1], iv, 8);
  88                 /* 32 bit counter, start at 0 or 1 (big endian!) */
  89                 cdesc->control_data.token[3] =
  90                         (__force u32)cpu_to_be32(ctx->ctrinit);
  91                 return 4;
  92         }
  93         if (ctx->alg == SAFEXCEL_CHACHA20) {
  94                 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
  95                 /* 96 bit nonce part */
  96                 memcpy(&cdesc->control_data.token[0], &iv[4], 12);
  97                 /* 32 bit counter */
  98                 cdesc->control_data.token[3] = *(u32 *)iv;
  99                 return 4;
 100         }
 101
 102         cdesc->control_data.options |= ctx->ivmask;
 103         memcpy(cdesc->control_data.token, iv, ctx->blocksz);
 104         return ctx->blocksz / sizeof(u32);
 105 }
 106
 107 static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
 108                                     struct safexcel_command_desc *cdesc,
 109                                     struct safexcel_token *atoken,
 110                                     u32 length)
 111 {
 112         struct safexcel_token *token;
 113         int ivlen;
 114
 115         ivlen = safexcel_skcipher_iv(ctx, iv, cdesc);
 116         if (ivlen == 4) {
 117                 /* No space in cdesc, instruction moves to atoken */
 118                 cdesc->additional_cdata_size = 1;
 119                 token = atoken;
 120         } else {
 121                 /* Everything fits in cdesc */
 122                 token = (struct safexcel_token *)(cdesc->control_data.token + 2);
 123                 /* Need to pad with NOP */
 124                 eip197_noop_token(&token[1]);
 125         }
 126
 127         token->opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 128         token->packet_length = length;
 129         token->stat = EIP197_TOKEN_STAT_LAST_PACKET |
 130                       EIP197_TOKEN_STAT_LAST_HASH;
 131         token->instructions = EIP197_TOKEN_INS_LAST |
 132                               EIP197_TOKEN_INS_TYPE_CRYPTO |
 133                               EIP197_TOKEN_INS_TYPE_OUTPUT;
 134 }
 135
 136 static void safexcel_aead_iv(struct safexcel_cipher_ctx *ctx, u8 *iv,
 137                              struct safexcel_command_desc *cdesc)
 138 {
 139         if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD ||
 140             ctx->aead & EIP197_AEAD_TYPE_IPSEC_ESP) { /* _ESP and _ESP_GMAC */
 141                 /* 32 bit nonce */
 142                 cdesc->control_data.token[0] = ctx->nonce;
 143                 /* 64 bit IV part */
 144                 memcpy(&cdesc->control_data.token[1], iv, 8);
 145                 /* 32 bit counter, start at 0 or 1 (big endian!) */
 146                 cdesc->control_data.token[3] =
 147                         (__force u32)cpu_to_be32(ctx->ctrinit);
 148                 return;
 149         }
 150         if (ctx->xcm == EIP197_XCM_MODE_GCM || ctx->alg == SAFEXCEL_CHACHA20) {
 151                 /* 96 bit IV part */
 152                 memcpy(&cdesc->control_data.token[0], iv, 12);
 153                 /* 32 bit counter, start at 0 or 1 (big endian!) */
 154                 cdesc->control_data.token[3] =
 155                         (__force u32)cpu_to_be32(ctx->ctrinit);
 156                 return;
 157         }
 158         /* CBC */
 159         memcpy(cdesc->control_data.token, iv, ctx->blocksz);
 160 }
 161
 162 static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
 163                                 struct safexcel_command_desc *cdesc,
 164                                 struct safexcel_token *atoken,
 165                                 enum safexcel_cipher_direction direction,
 166                                 u32 cryptlen, u32 assoclen, u32 digestsize)
 167 {
 168         struct safexcel_token *aadref;
 169         int atoksize = 2; /* Start with minimum size */
 170         int assocadj = assoclen - ctx->aadskip, aadalign;
 171
 172         /* Always 4 dwords of embedded IV  for AEAD modes */
 173         cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
 174
 175         if (direction == SAFEXCEL_DECRYPT)
 176                 cryptlen -= digestsize;
 177
 178         if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM)) {
 179                 /* Construct IV block B0 for the CBC-MAC */
 180                 u8 *final_iv = (u8 *)cdesc->control_data.token;
 181                 u8 *cbcmaciv = (u8 *)&atoken[1];
 182                 __le32 *aadlen = (__le32 *)&atoken[5];
 183
 184                 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) {
 185                         /* Length + nonce */
 186                         cdesc->control_data.token[0] = ctx->nonce;
 187                         /* Fixup flags byte */
 188                         *(__le32 *)cbcmaciv =
 189                                 cpu_to_le32(ctx->nonce |
 190                                             ((assocadj > 0) << 6) |
 191                                             ((digestsize - 2) << 2));
 192                         /* 64 bit IV part */
 193                         memcpy(&cdesc->control_data.token[1], iv, 8);
 194                         memcpy(cbcmaciv + 4, iv, 8);
 195                         /* Start counter at 0 */
 196                         cdesc->control_data.token[3] = 0;
 197                         /* Message length */
 198                         *(__be32 *)(cbcmaciv + 12) = cpu_to_be32(cryptlen);
 199                 } else {
 200                         /* Variable length IV part */
 201                         memcpy(final_iv, iv, 15 - iv[0]);
 202                         memcpy(cbcmaciv, iv, 15 - iv[0]);
 203                         /* Start variable length counter at 0 */
 204                         memset(final_iv + 15 - iv[0], 0, iv[0] + 1);
 205                         memset(cbcmaciv + 15 - iv[0], 0, iv[0] - 1);
 206                         /* fixup flags byte */
 207                         cbcmaciv[0] |= ((assocadj > 0) << 6) |
 208                                        ((digestsize - 2) << 2);
 209                         /* insert lower 2 bytes of message length */
 210                         cbcmaciv[14] = cryptlen >> 8;
 211                         cbcmaciv[15] = cryptlen & 255;
 212                 }
 213
 214                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 215                 atoken->packet_length = AES_BLOCK_SIZE +
 216                                         ((assocadj > 0) << 1);
 217                 atoken->stat = 0;
 218                 atoken->instructions = EIP197_TOKEN_INS_ORIGIN_TOKEN |
 219                                        EIP197_TOKEN_INS_TYPE_HASH;
 220
 221                 if (likely(assocadj)) {
 222                         *aadlen = cpu_to_le32((assocadj >> 8) |
 223                                               (assocadj & 255) << 8);
 224                         atoken += 6;
 225                         atoksize += 7;
 226                 } else {
 227                         atoken += 5;
 228                         atoksize += 6;
 229                 }
 230
 231                 /* Process AAD data */
 232                 aadref = atoken;
 233                 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 234                 atoken->packet_length = assocadj;
 235                 atoken->stat = 0;
 236                 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH;
 237                 atoken++;
 238
 239                 /* For CCM only, align AAD data towards hash engine */
 240                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 241                 aadalign = (assocadj + 2) & 15;
 242                 atoken->packet_length = assocadj && aadalign ?
 243                                                 16 - aadalign :
 244                                                 0;
 245                 if (likely(cryptlen)) {
 246                         atoken->stat = 0;
 247                         atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH;
 248                 } else {
 249                         atoken->stat = EIP197_TOKEN_STAT_LAST_HASH;
 250                         atoken->instructions = EIP197_TOKEN_INS_LAST |
 251                                                EIP197_TOKEN_INS_TYPE_HASH;
 252                 }
 253         } else {
 254                 safexcel_aead_iv(ctx, iv, cdesc);
 255
 256                 /* Process AAD data */
 257                 aadref = atoken;
 258                 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 259                 atoken->packet_length = assocadj;
 260                 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH;
 261                 atoken->instructions = EIP197_TOKEN_INS_LAST |
 262                                        EIP197_TOKEN_INS_TYPE_HASH;
 263         }
 264         atoken++;
 265
 266         if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) {
 267                 /* For ESP mode (and not GMAC), skip over the IV */
 268                 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 269                 atoken->packet_length = EIP197_AEAD_IPSEC_IV_SIZE;
 270                 atoken->stat = 0;
 271                 atoken->instructions = 0;
 272                 atoken++;
 273                 atoksize++;
 274         } else if (unlikely(ctx->alg == SAFEXCEL_CHACHA20 &&
 275                             direction == SAFEXCEL_DECRYPT)) {
 276                 /* Poly-chacha decryption needs a dummy NOP here ... */
 277                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 278                 atoken->packet_length = 16; /* According to Op Manual */
 279                 atoken->stat = 0;
 280                 atoken->instructions = 0;
 281                 atoken++;
 282                 atoksize++;
 283         }
 284
 285         if  (ctx->xcm) {
 286                 /* For GCM and CCM, obtain enc(Y0) */
 287                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES;
 288                 atoken->packet_length = 0;
 289                 atoken->stat = 0;
 290                 atoken->instructions = AES_BLOCK_SIZE;
 291                 atoken++;
 292
 293                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 294                 atoken->packet_length = AES_BLOCK_SIZE;
 295                 atoken->stat = 0;
 296                 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
 297                                        EIP197_TOKEN_INS_TYPE_CRYPTO;
 298                 atoken++;
 299                 atoksize += 2;
 300         }
 301
 302         if (likely(cryptlen || ctx->alg == SAFEXCEL_CHACHA20)) {
 303                 /* Fixup stat field for AAD direction instruction */
 304                 aadref->stat = 0;
 305
 306                 /* Process crypto data */
 307                 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 308                 atoken->packet_length = cryptlen;
 309
 310                 if (unlikely(ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) {
 311                         /* Fixup instruction field for AAD dir instruction */
 312                         aadref->instructions = EIP197_TOKEN_INS_TYPE_HASH;
 313
 314                         /* Do not send to crypt engine in case of GMAC */
 315                         atoken->instructions = EIP197_TOKEN_INS_LAST |
 316                                                EIP197_TOKEN_INS_TYPE_HASH |
 317                                                EIP197_TOKEN_INS_TYPE_OUTPUT;
 318                 } else {
 319                         atoken->instructions = EIP197_TOKEN_INS_LAST |
 320                                                EIP197_TOKEN_INS_TYPE_CRYPTO |
 321                                                EIP197_TOKEN_INS_TYPE_HASH |
 322                                                EIP197_TOKEN_INS_TYPE_OUTPUT;
 323                 }
 324
 325                 cryptlen &= 15;
 326                 if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM && cryptlen)) {
 327                         atoken->stat = 0;
 328                         /* For CCM only, pad crypto data to the hash engine */
 329                         atoken++;
 330                         atoksize++;
 331                         atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 332                         atoken->packet_length = 16 - cryptlen;
 333                         atoken->stat = EIP197_TOKEN_STAT_LAST_HASH;
 334                         atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH;
 335                 } else {
 336                         atoken->stat = EIP197_TOKEN_STAT_LAST_HASH;
 337                 }
 338                 atoken++;
 339                 atoksize++;
 340         }
 341
 342         if (direction == SAFEXCEL_ENCRYPT) {
 343                 /* Append ICV */
 344                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 345                 atoken->packet_length = digestsize;
 346                 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH |
 347                                EIP197_TOKEN_STAT_LAST_PACKET;
 348                 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
 349                                        EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
 350         } else {
 351                 /* Extract ICV */
 352                 atoken->opcode = EIP197_TOKEN_OPCODE_RETRIEVE;
 353                 atoken->packet_length = digestsize;
 354                 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH |
 355                                EIP197_TOKEN_STAT_LAST_PACKET;
 356                 atoken->instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
 357                 atoken++;
 358                 atoksize++;
 359
 360                 /* Verify ICV */
 361                 atoken->opcode = EIP197_TOKEN_OPCODE_VERIFY;
 362                 atoken->packet_length = digestsize |
 363                                         EIP197_TOKEN_HASH_RESULT_VERIFY;
 364                 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH |
 365                                EIP197_TOKEN_STAT_LAST_PACKET;
 366                 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT;
 367         }
 368
 369         /* Fixup length of the token in the command descriptor */
 370         cdesc->additional_cdata_size = atoksize;
 371 }
 372
 373 static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm,
 374                                         const u8 *key, unsigned int len)
 375 {
 376         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
 377         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 378         struct safexcel_crypto_priv *priv = ctx->priv;
 379         struct crypto_aes_ctx aes;
 380         int ret, i;
 381
 382         ret = aes_expandkey(&aes, key, len);
 383         if (ret)
 384                 return ret;
 385
 386         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
 387                 for (i = 0; i < len / sizeof(u32); i++) {
 388                         if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) {
 389                                 ctx->base.needs_inv = true;
 390                                 break;
 391                         }
 392                 }
 393         }
 394
 395         for (i = 0; i < len / sizeof(u32); i++)
 396                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
 397
 398         ctx->key_len = len;
 399
 400         memzero_explicit(&aes, sizeof(aes));
 401         return 0;
 402 }
 403
 404 static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key,
 405                                 unsigned int len)
 406 {
 407         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
 408         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 409         struct safexcel_ahash_export_state istate, ostate;
 410         struct safexcel_crypto_priv *priv = ctx->priv;
 411         struct crypto_authenc_keys keys;
 412         struct crypto_aes_ctx aes;
 413         int err = -EINVAL, i;
 414
 415         if (unlikely(crypto_authenc_extractkeys(&keys, key, len)))
 416                 goto badkey;
 417
 418         if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) {
 419                 /* Must have at least space for the nonce here */
 420                 if (unlikely(keys.enckeylen < CTR_RFC3686_NONCE_SIZE))
 421                         goto badkey;
 422                 /* last 4 bytes of key are the nonce! */
 423                 ctx->nonce = *(u32 *)(keys.enckey + keys.enckeylen -
 424                                       CTR_RFC3686_NONCE_SIZE);
 425                 /* exclude the nonce here */
 426                 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
 427         }
 428
 429         /* Encryption key */
 430         switch (ctx->alg) {
 431         case SAFEXCEL_DES:
 432                 err = verify_aead_des_key(ctfm, keys.enckey, keys.enckeylen);
 433                 if (unlikely(err))
 434                         goto badkey;
 435                 break;
 436         case SAFEXCEL_3DES:
 437                 err = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen);
 438                 if (unlikely(err))
 439                         goto badkey;
 440                 break;
 441         case SAFEXCEL_AES:
 442                 err = aes_expandkey(&aes, keys.enckey, keys.enckeylen);
 443                 if (unlikely(err))
 444                         goto badkey;
 445                 break;
 446         case SAFEXCEL_SM4:
 447                 if (unlikely(keys.enckeylen != SM4_KEY_SIZE))
 448                         goto badkey;
 449                 break;
 450         default:
 451                 dev_err(priv->dev, "aead: unsupported cipher algorithm\n");
 452                 goto badkey;
 453         }
 454
 455         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
 456                 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) {
 457                         if (le32_to_cpu(ctx->key[i]) !=
 458                             ((u32 *)keys.enckey)[i]) {
 459                                 ctx->base.needs_inv = true;
 460                                 break;
 461                         }
 462                 }
 463         }
 464
 465         /* Auth key */
 466         switch (ctx->hash_alg) {
 467         case CONTEXT_CONTROL_CRYPTO_ALG_SHA1:
 468                 if (safexcel_hmac_setkey("safexcel-sha1", keys.authkey,
 469                                          keys.authkeylen, &istate, &ostate))
 470                         goto badkey;
 471                 break;
 472         case CONTEXT_CONTROL_CRYPTO_ALG_SHA224:
 473                 if (safexcel_hmac_setkey("safexcel-sha224", keys.authkey,
 474                                          keys.authkeylen, &istate, &ostate))
 475                         goto badkey;
 476                 break;
 477         case CONTEXT_CONTROL_CRYPTO_ALG_SHA256:
 478                 if (safexcel_hmac_setkey("safexcel-sha256", keys.authkey,
 479                                          keys.authkeylen, &istate, &ostate))
 480                         goto badkey;
 481                 break;
 482         case CONTEXT_CONTROL_CRYPTO_ALG_SHA384:
 483                 if (safexcel_hmac_setkey("safexcel-sha384", keys.authkey,
 484                                          keys.authkeylen, &istate, &ostate))
 485                         goto badkey;
 486                 break;
 487         case CONTEXT_CONTROL_CRYPTO_ALG_SHA512:
 488                 if (safexcel_hmac_setkey("safexcel-sha512", keys.authkey,
 489                                          keys.authkeylen, &istate, &ostate))
 490                         goto badkey;
 491                 break;
 492         case CONTEXT_CONTROL_CRYPTO_ALG_SM3:
 493                 if (safexcel_hmac_setkey("safexcel-sm3", keys.authkey,
 494                                          keys.authkeylen, &istate, &ostate))
 495                         goto badkey;
 496                 break;
 497         default:
 498                 dev_err(priv->dev, "aead: unsupported hash algorithm\n");
 499                 goto badkey;
 500         }
 501
 502         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma &&
 503             (memcmp(ctx->ipad, istate.state, ctx->state_sz) ||
 504              memcmp(ctx->opad, ostate.state, ctx->state_sz)))
 505                 ctx->base.needs_inv = true;
 506
 507         /* Now copy the keys into the context */
 508         for (i = 0; i < keys.enckeylen / sizeof(u32); i++)
 509                 ctx->key[i] = cpu_to_le32(((u32 *)keys.enckey)[i]);
 510         ctx->key_len = keys.enckeylen;
 511
 512         memcpy(ctx->ipad, &istate.state, ctx->state_sz);
 513         memcpy(ctx->opad, &ostate.state, ctx->state_sz);
 514
 515         memzero_explicit(&keys, sizeof(keys));
 516         return 0;
 517
 518 badkey:
 519         memzero_explicit(&keys, sizeof(keys));
 520         return err;
 521 }
 522
 523 static int safexcel_context_control(struct safexcel_cipher_ctx *ctx,
 524                                     struct crypto_async_request *async,
 525                                     struct safexcel_cipher_req *sreq,
 526                                     struct safexcel_command_desc *cdesc)
 527 {
 528         struct safexcel_crypto_priv *priv = ctx->priv;
 529         int ctrl_size = ctx->key_len / sizeof(u32);
 530
 531         cdesc->control_data.control1 = ctx->mode;
 532
 533         if (ctx->aead) {
 534                 /* Take in account the ipad+opad digests */
 535                 if (ctx->xcm) {
 536                         ctrl_size += ctx->state_sz / sizeof(u32);
 537                         cdesc->control_data.control0 =
 538                                 CONTEXT_CONTROL_KEY_EN |
 539                                 CONTEXT_CONTROL_DIGEST_XCM |
 540                                 ctx->hash_alg |
 541                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 542                 } else if (ctx->alg == SAFEXCEL_CHACHA20) {
 543                         /* Chacha20-Poly1305 */
 544                         cdesc->control_data.control0 =
 545                                 CONTEXT_CONTROL_KEY_EN |
 546                                 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 |
 547                                 (sreq->direction == SAFEXCEL_ENCRYPT ?
 548                                         CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT :
 549                                         CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN) |
 550                                 ctx->hash_alg |
 551                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 552                         return 0;
 553                 } else {
 554                         ctrl_size += ctx->state_sz / sizeof(u32) * 2;
 555                         cdesc->control_data.control0 =
 556                                 CONTEXT_CONTROL_KEY_EN |
 557                                 CONTEXT_CONTROL_DIGEST_HMAC |
 558                                 ctx->hash_alg |
 559                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 560                 }
 561
 562                 if (sreq->direction == SAFEXCEL_ENCRYPT &&
 563                     (ctx->xcm == EIP197_XCM_MODE_CCM ||
 564                      ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC))
 565                         cdesc->control_data.control0 |=
 566                                 CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT;
 567                 else if (sreq->direction == SAFEXCEL_ENCRYPT)
 568                         cdesc->control_data.control0 |=
 569                                 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT;
 570                 else if (ctx->xcm == EIP197_XCM_MODE_CCM)
 571                         cdesc->control_data.control0 |=
 572                                 CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN;
 573                 else
 574                         cdesc->control_data.control0 |=
 575                                 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN;
 576         } else {
 577                 if (sreq->direction == SAFEXCEL_ENCRYPT)
 578                         cdesc->control_data.control0 =
 579                                 CONTEXT_CONTROL_TYPE_CRYPTO_OUT |
 580                                 CONTEXT_CONTROL_KEY_EN |
 581                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 582                 else
 583                         cdesc->control_data.control0 =
 584                                 CONTEXT_CONTROL_TYPE_CRYPTO_IN |
 585                                 CONTEXT_CONTROL_KEY_EN |
 586                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 587         }
 588
 589         if (ctx->alg == SAFEXCEL_DES) {
 590                 cdesc->control_data.control0 |=
 591                         CONTEXT_CONTROL_CRYPTO_ALG_DES;
 592         } else if (ctx->alg == SAFEXCEL_3DES) {
 593                 cdesc->control_data.control0 |=
 594                         CONTEXT_CONTROL_CRYPTO_ALG_3DES;
 595         } else if (ctx->alg == SAFEXCEL_AES) {
 596                 switch (ctx->key_len >> ctx->xts) {
 597                 case AES_KEYSIZE_128:
 598                         cdesc->control_data.control0 |=
 599                                 CONTEXT_CONTROL_CRYPTO_ALG_AES128;
 600                         break;
 601                 case AES_KEYSIZE_192:
 602                         cdesc->control_data.control0 |=
 603                                 CONTEXT_CONTROL_CRYPTO_ALG_AES192;
 604                         break;
 605                 case AES_KEYSIZE_256:
 606                         cdesc->control_data.control0 |=
 607                                 CONTEXT_CONTROL_CRYPTO_ALG_AES256;
 608                         break;
 609                 default:
 610                         dev_err(priv->dev, "aes keysize not supported: %u\n",
 611                                 ctx->key_len >> ctx->xts);
 612                         return -EINVAL;
 613                 }
 614         } else if (ctx->alg == SAFEXCEL_CHACHA20) {
 615                 cdesc->control_data.control0 |=
 616                         CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20;
 617         } else if (ctx->alg == SAFEXCEL_SM4) {
 618                 cdesc->control_data.control0 |=
 619                         CONTEXT_CONTROL_CRYPTO_ALG_SM4;
 620         }
 621
 622         return 0;
 623 }
 624
 625 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring,
 626                                       struct crypto_async_request *async,
 627                                       struct scatterlist *src,
 628                                       struct scatterlist *dst,
 629                                       unsigned int cryptlen,
 630                                       struct safexcel_cipher_req *sreq,
 631                                       bool *should_complete, int *ret)
 632 {
 633         struct skcipher_request *areq = skcipher_request_cast(async);
 634         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq);
 635         struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(skcipher);
 636         struct safexcel_result_desc *rdesc;
 637         int ndesc = 0;
 638
 639         *ret = 0;
 640
 641         if (unlikely(!sreq->rdescs))
 642                 return 0;
 643
 644         while (sreq->rdescs--) {
 645                 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 646                 if (IS_ERR(rdesc)) {
 647                         dev_err(priv->dev,
 648                                 "cipher: result: could not retrieve the result descriptor\n");
 649                         *ret = PTR_ERR(rdesc);
 650                         break;
 651                 }
 652
 653                 if (likely(!*ret))
 654                         *ret = safexcel_rdesc_check_errors(priv, rdesc);
 655
 656                 ndesc++;
 657         }
 658
 659         safexcel_complete(priv, ring);
 660
 661         if (src == dst) {
 662                 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL);
 663         } else {
 664                 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE);
 665                 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE);
 666         }
 667
 668         /*
 669          * Update IV in req from last crypto output word for CBC modes
 670          */
 671         if ((!ctx->aead) && (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) &&
 672             (sreq->direction == SAFEXCEL_ENCRYPT)) {
 673                 /* For encrypt take the last output word */
 674                 sg_pcopy_to_buffer(dst, sreq->nr_dst, areq->iv,
 675                                    crypto_skcipher_ivsize(skcipher),
 676                                    (cryptlen -
 677                                     crypto_skcipher_ivsize(skcipher)));
 678         }
 679
 680         *should_complete = true;
 681
 682         return ndesc;
 683 }
 684
 685 static int safexcel_send_req(struct crypto_async_request *base, int ring,
 686                              struct safexcel_cipher_req *sreq,
 687                              struct scatterlist *src, struct scatterlist *dst,
 688                              unsigned int cryptlen, unsigned int assoclen,
 689                              unsigned int digestsize, u8 *iv, int *commands,
 690                              int *results)
 691 {
 692         struct skcipher_request *areq = skcipher_request_cast(base);
 693         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq);
 694         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 695         struct safexcel_crypto_priv *priv = ctx->priv;
 696         struct safexcel_command_desc *cdesc;
 697         struct safexcel_command_desc *first_cdesc = NULL;
 698         struct safexcel_result_desc *rdesc, *first_rdesc = NULL;
 699         struct scatterlist *sg;
 700         unsigned int totlen;
 701         unsigned int totlen_src = cryptlen + assoclen;
 702         unsigned int totlen_dst = totlen_src;
 703         struct safexcel_token *atoken;
 704         int n_cdesc = 0, n_rdesc = 0;
 705         int queued, i, ret = 0;
 706         bool first = true;
 707
 708         sreq->nr_src = sg_nents_for_len(src, totlen_src);
 709
 710         if (ctx->aead) {
 711                 /*
 712                  * AEAD has auth tag appended to output for encrypt and
 713                  * removed from the output for decrypt!
 714                  */
 715                 if (sreq->direction == SAFEXCEL_DECRYPT)
 716                         totlen_dst -= digestsize;
 717                 else
 718                         totlen_dst += digestsize;
 719
 720                 memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32),
 721                        ctx->ipad, ctx->state_sz);
 722                 if (!ctx->xcm)
 723                         memcpy(ctx->base.ctxr->data + (ctx->key_len +
 724                                ctx->state_sz) / sizeof(u32), ctx->opad,
 725                                ctx->state_sz);
 726         } else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) &&
 727                    (sreq->direction == SAFEXCEL_DECRYPT)) {
 728                 /*
 729                  * Save IV from last crypto input word for CBC modes in decrypt
 730                  * direction. Need to do this first in case of inplace operation
 731                  * as it will be overwritten.
 732                  */
 733                 sg_pcopy_to_buffer(src, sreq->nr_src, areq->iv,
 734                                    crypto_skcipher_ivsize(skcipher),
 735                                    (totlen_src -
 736                                     crypto_skcipher_ivsize(skcipher)));
 737         }
 738
 739         sreq->nr_dst = sg_nents_for_len(dst, totlen_dst);
 740
 741         /*
 742          * Remember actual input length, source buffer length may be
 743          * updated in case of inline operation below.
 744          */
 745         totlen = totlen_src;
 746         queued = totlen_src;
 747
 748         if (src == dst) {
 749                 sreq->nr_src = max(sreq->nr_src, sreq->nr_dst);
 750                 sreq->nr_dst = sreq->nr_src;
 751                 if (unlikely((totlen_src || totlen_dst) &&
 752                     (sreq->nr_src <= 0))) {
 753                         dev_err(priv->dev, "In-place buffer not large enough (need %d bytes)!",
 754                                 max(totlen_src, totlen_dst));
 755                         return -EINVAL;
 756                 }
 757                 dma_map_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL);
 758         } else {
 759                 if (unlikely(totlen_src && (sreq->nr_src <= 0))) {
 760                         dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!",
 761                                 totlen_src);
 762                         return -EINVAL;
 763                 }
 764                 dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE);
 765
 766                 if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) {
 767                         dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!",
 768                                 totlen_dst);
 769                         dma_unmap_sg(priv->dev, src, sreq->nr_src,
 770                                      DMA_TO_DEVICE);
 771                         return -EINVAL;
 772                 }
 773                 dma_map_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE);
 774         }
 775
 776         memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len);
 777
 778         if (!totlen) {
 779                 /*
 780                  * The EIP97 cannot deal with zero length input packets!
 781                  * So stuff a dummy command descriptor indicating a 1 byte
 782                  * (dummy) input packet, using the context record as source.
 783                  */
 784                 first_cdesc = safexcel_add_cdesc(priv, ring,
 785                                                  1, 1, ctx->base.ctxr_dma,
 786                                                  1, 1, ctx->base.ctxr_dma,
 787                                                  &atoken);
 788                 if (IS_ERR(first_cdesc)) {
 789                         /* No space left in the command descriptor ring */
 790                         ret = PTR_ERR(first_cdesc);
 791                         goto cdesc_rollback;
 792                 }
 793                 n_cdesc = 1;
 794                 goto skip_cdesc;
 795         }
 796
 797         /* command descriptors */
 798         for_each_sg(src, sg, sreq->nr_src, i) {
 799                 int len = sg_dma_len(sg);
 800
 801                 /* Do not overflow the request */
 802                 if (queued < len)
 803                         len = queued;
 804
 805                 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
 806                                            !(queued - len),
 807                                            sg_dma_address(sg), len, totlen,
 808                                            ctx->base.ctxr_dma, &atoken);
 809                 if (IS_ERR(cdesc)) {
 810                         /* No space left in the command descriptor ring */
 811                         ret = PTR_ERR(cdesc);
 812                         goto cdesc_rollback;
 813                 }
 814
 815                 if (!n_cdesc)
 816                         first_cdesc = cdesc;
 817
 818                 n_cdesc++;
 819                 queued -= len;
 820                 if (!queued)
 821                         break;
 822         }
 823 skip_cdesc:
 824         /* Add context control words and token to first command descriptor */
 825         safexcel_context_control(ctx, base, sreq, first_cdesc);
 826         if (ctx->aead)
 827                 safexcel_aead_token(ctx, iv, first_cdesc, atoken,
 828                                     sreq->direction, cryptlen,
 829                                     assoclen, digestsize);
 830         else
 831                 safexcel_skcipher_token(ctx, iv, first_cdesc, atoken,
 832                                         cryptlen);
 833
 834         /* result descriptors */
 835         for_each_sg(dst, sg, sreq->nr_dst, i) {
 836                 bool last = (i == sreq->nr_dst - 1);
 837                 u32 len = sg_dma_len(sg);
 838
 839                 /* only allow the part of the buffer we know we need */
 840                 if (len > totlen_dst)
 841                         len = totlen_dst;
 842                 if (unlikely(!len))
 843                         break;
 844                 totlen_dst -= len;
 845
 846                 /* skip over AAD space in buffer - not written */
 847                 if (assoclen) {
 848                         if (assoclen >= len) {
 849                                 assoclen -= len;
 850                                 continue;
 851                         }
 852                         rdesc = safexcel_add_rdesc(priv, ring, first, last,
 853                                                    sg_dma_address(sg) +
 854                                                    assoclen,
 855                                                    len - assoclen);
 856                         assoclen = 0;
 857                 } else {
 858                         rdesc = safexcel_add_rdesc(priv, ring, first, last,
 859                                                    sg_dma_address(sg),
 860                                                    len);
 861                 }
 862                 if (IS_ERR(rdesc)) {
 863                         /* No space left in the result descriptor ring */
 864                         ret = PTR_ERR(rdesc);
 865                         goto rdesc_rollback;
 866                 }
 867                 if (first) {
 868                         first_rdesc = rdesc;
 869                         first = false;
 870                 }
 871                 n_rdesc++;
 872         }
 873
 874         if (unlikely(first)) {
 875                 /*
 876                  * Special case: AEAD decrypt with only AAD data.
 877                  * In this case there is NO output data from the engine,
 878                  * but the engine still needs a result descriptor!
 879                  * Create a dummy one just for catching the result token.
 880                  */
 881                 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
 882                 if (IS_ERR(rdesc)) {
 883                         /* No space left in the result descriptor ring */
 884                         ret = PTR_ERR(rdesc);
 885                         goto rdesc_rollback;
 886                 }
 887                 first_rdesc = rdesc;
 888                 n_rdesc = 1;
 889         }
 890
 891         safexcel_rdr_req_set(priv, ring, first_rdesc, base);
 892
 893         *commands = n_cdesc;
 894         *results = n_rdesc;
 895         return 0;
 896
 897 rdesc_rollback:
 898         for (i = 0; i < n_rdesc; i++)
 899                 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr);
 900 cdesc_rollback:
 901         for (i = 0; i < n_cdesc; i++)
 902                 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
 903
 904         if (src == dst) {
 905                 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL);
 906         } else {
 907                 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE);
 908                 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE);
 909         }
 910
 911         return ret;
 912 }
 913
 914 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
 915                                       int ring,
 916                                       struct crypto_async_request *base,
 917                                       struct safexcel_cipher_req *sreq,
 918                                       bool *should_complete, int *ret)
 919 {
 920         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 921         struct safexcel_result_desc *rdesc;
 922         int ndesc = 0, enq_ret;
 923
 924         *ret = 0;
 925
 926         if (unlikely(!sreq->rdescs))
 927                 return 0;
 928
 929         while (sreq->rdescs--) {
 930                 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 931                 if (IS_ERR(rdesc)) {
 932                         dev_err(priv->dev,
 933                                 "cipher: invalidate: could not retrieve the result descriptor\n");
 934                         *ret = PTR_ERR(rdesc);
 935                         break;
 936                 }
 937
 938                 if (likely(!*ret))
 939                         *ret = safexcel_rdesc_check_errors(priv, rdesc);
 940
 941                 ndesc++;
 942         }
 943
 944         safexcel_complete(priv, ring);
 945
 946         if (ctx->base.exit_inv) {
 947                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
 948                               ctx->base.ctxr_dma);
 949
 950                 *should_complete = true;
 951
 952                 return ndesc;
 953         }
 954
 955         ring = safexcel_select_ring(priv);
 956         ctx->base.ring = ring;
 957
 958         spin_lock_bh(&priv->ring[ring].queue_lock);
 959         enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, base);
 960         spin_unlock_bh(&priv->ring[ring].queue_lock);
 961
 962         if (enq_ret != -EINPROGRESS)
 963                 *ret = enq_ret;
 964
 965         queue_work(priv->ring[ring].workqueue,
 966                    &priv->ring[ring].work_data.work);
 967
 968         *should_complete = false;
 969
 970         return ndesc;
 971 }
 972
 973 static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv,
 974                                            int ring,
 975                                            struct crypto_async_request *async,
 976                                            bool *should_complete, int *ret)
 977 {
 978         struct skcipher_request *req = skcipher_request_cast(async);
 979         struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
 980         int err;
 981
 982         if (sreq->needs_inv) {
 983                 sreq->needs_inv = false;
 984                 err = safexcel_handle_inv_result(priv, ring, async, sreq,
 985                                                  should_complete, ret);
 986         } else {
 987                 err = safexcel_handle_req_result(priv, ring, async, req->src,
 988                                                  req->dst, req->cryptlen, sreq,
 989                                                  should_complete, ret);
 990         }
 991
 992         return err;
 993 }
 994
 995 static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv,
 996                                        int ring,
 997                                        struct crypto_async_request *async,
 998                                        bool *should_complete, int *ret)
 999 {
1000         struct aead_request *req = aead_request_cast(async);
1001         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1002         struct safexcel_cipher_req *sreq = aead_request_ctx(req);
1003         int err;
1004
1005         if (sreq->needs_inv) {
1006                 sreq->needs_inv = false;
1007                 err = safexcel_handle_inv_result(priv, ring, async, sreq,
1008                                                  should_complete, ret);
1009         } else {
1010                 err = safexcel_handle_req_result(priv, ring, async, req->src,
1011                                                  req->dst,
1012                                                  req->cryptlen + crypto_aead_authsize(tfm),
1013                                                  sreq, should_complete, ret);
1014         }
1015
1016         return err;
1017 }
1018
1019 static int safexcel_cipher_send_inv(struct crypto_async_request *base,
1020                                     int ring, int *commands, int *results)
1021 {
1022         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
1023         struct safexcel_crypto_priv *priv = ctx->priv;
1024         int ret;
1025
1026         ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring);
1027         if (unlikely(ret))
1028                 return ret;
1029
1030         *commands = 1;
1031         *results = 1;
1032
1033         return 0;
1034 }
1035
1036 static int safexcel_skcipher_send(struct crypto_async_request *async, int ring,
1037                                   int *commands, int *results)
1038 {
1039         struct skcipher_request *req = skcipher_request_cast(async);
1040         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
1041         struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
1042         struct safexcel_crypto_priv *priv = ctx->priv;
1043         int ret;
1044
1045         BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
1046
1047         if (sreq->needs_inv) {
1048                 ret = safexcel_cipher_send_inv(async, ring, commands, results);
1049         } else {
1050                 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1051                 u8 input_iv[AES_BLOCK_SIZE];
1052
1053                 /*
1054                  * Save input IV in case of CBC decrypt mode
1055                  * Will be overwritten with output IV prior to use!
1056                  */
1057                 memcpy(input_iv, req->iv, crypto_skcipher_ivsize(skcipher));
1058
1059                 ret = safexcel_send_req(async, ring, sreq, req->src,
1060                                         req->dst, req->cryptlen, 0, 0, input_iv,
1061                                         commands, results);
1062         }
1063
1064         sreq->rdescs = *results;
1065         return ret;
1066 }
1067
1068 static int safexcel_aead_send(struct crypto_async_request *async, int ring,
1069                               int *commands, int *results)
1070 {
1071         struct aead_request *req = aead_request_cast(async);
1072         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1073         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
1074         struct safexcel_cipher_req *sreq = aead_request_ctx(req);
1075         struct safexcel_crypto_priv *priv = ctx->priv;
1076         int ret;
1077
1078         BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
1079
1080         if (sreq->needs_inv)
1081                 ret = safexcel_cipher_send_inv(async, ring, commands, results);
1082         else
1083                 ret = safexcel_send_req(async, ring, sreq, req->src, req->dst,
1084                                         req->cryptlen, req->assoclen,
1085                                         crypto_aead_authsize(tfm), req->iv,
1086                                         commands, results);
1087         sreq->rdescs = *results;
1088         return ret;
1089 }
1090
1091 static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm,
1092                                     struct crypto_async_request *base,
1093                                     struct safexcel_cipher_req *sreq,
1094                                     struct safexcel_inv_result *result)
1095 {
1096         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1097         struct safexcel_crypto_priv *priv = ctx->priv;
1098         int ring = ctx->base.ring;
1099
1100         init_completion(&result->completion);
1101
1102         ctx = crypto_tfm_ctx(base->tfm);
1103         ctx->base.exit_inv = true;
1104         sreq->needs_inv = true;
1105
1106         spin_lock_bh(&priv->ring[ring].queue_lock);
1107         crypto_enqueue_request(&priv->ring[ring].queue, base);
1108         spin_unlock_bh(&priv->ring[ring].queue_lock);
1109
1110         queue_work(priv->ring[ring].workqueue,
1111                    &priv->ring[ring].work_data.work);
1112
1113         wait_for_completion(&result->completion);
1114
1115         if (result->error) {
1116                 dev_warn(priv->dev,
1117                         "cipher: sync: invalidate: completion error %d\n",
1118                          result->error);
1119                 return result->error;
1120         }
1121
1122         return 0;
1123 }
1124
1125 static int safexcel_skcipher_exit_inv(struct crypto_tfm *tfm)
1126 {
1127         EIP197_REQUEST_ON_STACK(req, skcipher, EIP197_SKCIPHER_REQ_SIZE);
1128         struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
1129         struct safexcel_inv_result result = {};
1130
1131         memset(req, 0, sizeof(struct skcipher_request));
1132
1133         skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1134                                       safexcel_inv_complete, &result);
1135         skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm));
1136
1137         return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
1138 }
1139
1140 static int safexcel_aead_exit_inv(struct crypto_tfm *tfm)
1141 {
1142         EIP197_REQUEST_ON_STACK(req, aead, EIP197_AEAD_REQ_SIZE);
1143         struct safexcel_cipher_req *sreq = aead_request_ctx(req);
1144         struct safexcel_inv_result result = {};
1145
1146         memset(req, 0, sizeof(struct aead_request));
1147
1148         aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1149                                   safexcel_inv_complete, &result);
1150         aead_request_set_tfm(req, __crypto_aead_cast(tfm));
1151
1152         return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
1153 }
1154
1155 static int safexcel_queue_req(struct crypto_async_request *base,
1156                         struct safexcel_cipher_req *sreq,
1157                         enum safexcel_cipher_direction dir)
1158 {
1159         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
1160         struct safexcel_crypto_priv *priv = ctx->priv;
1161         int ret, ring;
1162
1163         sreq->needs_inv = false;
1164         sreq->direction = dir;
1165
1166         if (ctx->base.ctxr) {
1167                 if (priv->flags & EIP197_TRC_CACHE && ctx->base.needs_inv) {
1168                         sreq->needs_inv = true;
1169                         ctx->base.needs_inv = false;
1170                 }
1171         } else {
1172                 ctx->base.ring = safexcel_select_ring(priv);
1173                 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
1174                                                  EIP197_GFP_FLAGS(*base),
1175                                                  &ctx->base.ctxr_dma);
1176                 if (!ctx->base.ctxr)
1177                         return -ENOMEM;
1178         }
1179
1180         ring = ctx->base.ring;
1181
1182         spin_lock_bh(&priv->ring[ring].queue_lock);
1183         ret = crypto_enqueue_request(&priv->ring[ring].queue, base);
1184         spin_unlock_bh(&priv->ring[ring].queue_lock);
1185
1186         queue_work(priv->ring[ring].workqueue,
1187                    &priv->ring[ring].work_data.work);
1188
1189         return ret;
1190 }
1191
1192 static int safexcel_encrypt(struct skcipher_request *req)
1193 {
1194         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1195                         SAFEXCEL_ENCRYPT);
1196 }
1197
1198 static int safexcel_decrypt(struct skcipher_request *req)
1199 {
1200         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1201                         SAFEXCEL_DECRYPT);
1202 }
1203
1204 static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm)
1205 {
1206         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1207         struct safexcel_alg_template *tmpl =
1208                 container_of(tfm->__crt_alg, struct safexcel_alg_template,
1209                              alg.skcipher.base);
1210
1211         crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm),
1212                                     sizeof(struct safexcel_cipher_req));
1213
1214         ctx->priv = tmpl->priv;
1215
1216         ctx->base.send = safexcel_skcipher_send;
1217         ctx->base.handle_result = safexcel_skcipher_handle_result;
1218         ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD;
1219         ctx->ctrinit = 1;
1220         return 0;
1221 }
1222
1223 static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm)
1224 {
1225         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1226
1227         memzero_explicit(ctx->key, sizeof(ctx->key));
1228
1229         /* context not allocated, skip invalidation */
1230         if (!ctx->base.ctxr)
1231                 return -ENOMEM;
1232
1233         memzero_explicit(ctx->base.ctxr->data, sizeof(ctx->base.ctxr->data));
1234         return 0;
1235 }
1236
1237 static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm)
1238 {
1239         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1240         struct safexcel_crypto_priv *priv = ctx->priv;
1241         int ret;
1242
1243         if (safexcel_cipher_cra_exit(tfm))
1244                 return;
1245
1246         if (priv->flags & EIP197_TRC_CACHE) {
1247                 ret = safexcel_skcipher_exit_inv(tfm);
1248                 if (ret)
1249                         dev_warn(priv->dev, "skcipher: invalidation error %d\n",
1250                                  ret);
1251         } else {
1252                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
1253                               ctx->base.ctxr_dma);
1254         }
1255 }
1256
1257 static void safexcel_aead_cra_exit(struct crypto_tfm *tfm)
1258 {
1259         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1260         struct safexcel_crypto_priv *priv = ctx->priv;
1261         int ret;
1262
1263         if (safexcel_cipher_cra_exit(tfm))
1264                 return;
1265
1266         if (priv->flags & EIP197_TRC_CACHE) {
1267                 ret = safexcel_aead_exit_inv(tfm);
1268                 if (ret)
1269                         dev_warn(priv->dev, "aead: invalidation error %d\n",
1270                                  ret);
1271         } else {
1272                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
1273                               ctx->base.ctxr_dma);
1274         }
1275 }
1276
1277 static int safexcel_skcipher_aes_ecb_cra_init(struct crypto_tfm *tfm)
1278 {
1279         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1280
1281         safexcel_skcipher_cra_init(tfm);
1282         ctx->alg  = SAFEXCEL_AES;
1283         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
1284         ctx->blocksz = 0;
1285         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1286         return 0;
1287 }
1288
1289 struct safexcel_alg_template safexcel_alg_ecb_aes = {
1290         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1291         .algo_mask = SAFEXCEL_ALG_AES,
1292         .alg.skcipher = {
1293                 .setkey = safexcel_skcipher_aes_setkey,
1294                 .encrypt = safexcel_encrypt,
1295                 .decrypt = safexcel_decrypt,
1296                 .min_keysize = AES_MIN_KEY_SIZE,
1297                 .max_keysize = AES_MAX_KEY_SIZE,
1298                 .base = {
1299                         .cra_name = "ecb(aes)",
1300                         .cra_driver_name = "safexcel-ecb-aes",
1301                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1302                         .cra_flags = CRYPTO_ALG_ASYNC |
1303                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1304                         .cra_blocksize = AES_BLOCK_SIZE,
1305                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1306                         .cra_alignmask = 0,
1307                         .cra_init = safexcel_skcipher_aes_ecb_cra_init,
1308                         .cra_exit = safexcel_skcipher_cra_exit,
1309                         .cra_module = THIS_MODULE,
1310                 },
1311         },
1312 };
1313
1314 static int safexcel_skcipher_aes_cbc_cra_init(struct crypto_tfm *tfm)
1315 {
1316         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1317
1318         safexcel_skcipher_cra_init(tfm);
1319         ctx->alg  = SAFEXCEL_AES;
1320         ctx->blocksz = AES_BLOCK_SIZE;
1321         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
1322         return 0;
1323 }
1324
1325 struct safexcel_alg_template safexcel_alg_cbc_aes = {
1326         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1327         .algo_mask = SAFEXCEL_ALG_AES,
1328         .alg.skcipher = {
1329                 .setkey = safexcel_skcipher_aes_setkey,
1330                 .encrypt = safexcel_encrypt,
1331                 .decrypt = safexcel_decrypt,
1332                 .min_keysize = AES_MIN_KEY_SIZE,
1333                 .max_keysize = AES_MAX_KEY_SIZE,
1334                 .ivsize = AES_BLOCK_SIZE,
1335                 .base = {
1336                         .cra_name = "cbc(aes)",
1337                         .cra_driver_name = "safexcel-cbc-aes",
1338                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1339                         .cra_flags = CRYPTO_ALG_ASYNC |
1340                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1341                         .cra_blocksize = AES_BLOCK_SIZE,
1342                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1343                         .cra_alignmask = 0,
1344                         .cra_init = safexcel_skcipher_aes_cbc_cra_init,
1345                         .cra_exit = safexcel_skcipher_cra_exit,
1346                         .cra_module = THIS_MODULE,
1347                 },
1348         },
1349 };
1350
1351 static int safexcel_skcipher_aes_cfb_cra_init(struct crypto_tfm *tfm)
1352 {
1353         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1354
1355         safexcel_skcipher_cra_init(tfm);
1356         ctx->alg  = SAFEXCEL_AES;
1357         ctx->blocksz = AES_BLOCK_SIZE;
1358         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB;
1359         return 0;
1360 }
1361
1362 struct safexcel_alg_template safexcel_alg_cfb_aes = {
1363         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1364         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB,
1365         .alg.skcipher = {
1366                 .setkey = safexcel_skcipher_aes_setkey,
1367                 .encrypt = safexcel_encrypt,
1368                 .decrypt = safexcel_decrypt,
1369                 .min_keysize = AES_MIN_KEY_SIZE,
1370                 .max_keysize = AES_MAX_KEY_SIZE,
1371                 .ivsize = AES_BLOCK_SIZE,
1372                 .base = {
1373                         .cra_name = "cfb(aes)",
1374                         .cra_driver_name = "safexcel-cfb-aes",
1375                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1376                         .cra_flags = CRYPTO_ALG_ASYNC |
1377                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1378                         .cra_blocksize = 1,
1379                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1380                         .cra_alignmask = 0,
1381                         .cra_init = safexcel_skcipher_aes_cfb_cra_init,
1382                         .cra_exit = safexcel_skcipher_cra_exit,
1383                         .cra_module = THIS_MODULE,
1384                 },
1385         },
1386 };
1387
1388 static int safexcel_skcipher_aes_ofb_cra_init(struct crypto_tfm *tfm)
1389 {
1390         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1391
1392         safexcel_skcipher_cra_init(tfm);
1393         ctx->alg  = SAFEXCEL_AES;
1394         ctx->blocksz = AES_BLOCK_SIZE;
1395         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB;
1396         return 0;
1397 }
1398
1399 struct safexcel_alg_template safexcel_alg_ofb_aes = {
1400         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1401         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB,
1402         .alg.skcipher = {
1403                 .setkey = safexcel_skcipher_aes_setkey,
1404                 .encrypt = safexcel_encrypt,
1405                 .decrypt = safexcel_decrypt,
1406                 .min_keysize = AES_MIN_KEY_SIZE,
1407                 .max_keysize = AES_MAX_KEY_SIZE,
1408                 .ivsize = AES_BLOCK_SIZE,
1409                 .base = {
1410                         .cra_name = "ofb(aes)",
1411                         .cra_driver_name = "safexcel-ofb-aes",
1412                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1413                         .cra_flags = CRYPTO_ALG_ASYNC |
1414                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1415                         .cra_blocksize = 1,
1416                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1417                         .cra_alignmask = 0,
1418                         .cra_init = safexcel_skcipher_aes_ofb_cra_init,
1419                         .cra_exit = safexcel_skcipher_cra_exit,
1420                         .cra_module = THIS_MODULE,
1421                 },
1422         },
1423 };
1424
1425 static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm,
1426                                            const u8 *key, unsigned int len)
1427 {
1428         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
1429         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1430         struct safexcel_crypto_priv *priv = ctx->priv;
1431         struct crypto_aes_ctx aes;
1432         int ret, i;
1433         unsigned int keylen;
1434
1435         /* last 4 bytes of key are the nonce! */
1436         ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE);
1437         /* exclude the nonce here */
1438         keylen = len - CTR_RFC3686_NONCE_SIZE;
1439         ret = aes_expandkey(&aes, key, keylen);
1440         if (ret)
1441                 return ret;
1442
1443         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
1444                 for (i = 0; i < keylen / sizeof(u32); i++) {
1445                         if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) {
1446                                 ctx->base.needs_inv = true;
1447                                 break;
1448                         }
1449                 }
1450         }
1451
1452         for (i = 0; i < keylen / sizeof(u32); i++)
1453                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
1454
1455         ctx->key_len = keylen;
1456
1457         memzero_explicit(&aes, sizeof(aes));
1458         return 0;
1459 }
1460
1461 static int safexcel_skcipher_aes_ctr_cra_init(struct crypto_tfm *tfm)
1462 {
1463         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1464
1465         safexcel_skcipher_cra_init(tfm);
1466         ctx->alg  = SAFEXCEL_AES;
1467         ctx->blocksz = AES_BLOCK_SIZE;
1468         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD;
1469         return 0;
1470 }
1471
1472 struct safexcel_alg_template safexcel_alg_ctr_aes = {
1473         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1474         .algo_mask = SAFEXCEL_ALG_AES,
1475         .alg.skcipher = {
1476                 .setkey = safexcel_skcipher_aesctr_setkey,
1477                 .encrypt = safexcel_encrypt,
1478                 .decrypt = safexcel_decrypt,
1479                 /* Add nonce size */
1480                 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
1481                 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
1482                 .ivsize = CTR_RFC3686_IV_SIZE,
1483                 .base = {
1484                         .cra_name = "rfc3686(ctr(aes))",
1485                         .cra_driver_name = "safexcel-ctr-aes",
1486                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1487                         .cra_flags = CRYPTO_ALG_ASYNC |
1488                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1489                         .cra_blocksize = 1,
1490                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1491                         .cra_alignmask = 0,
1492                         .cra_init = safexcel_skcipher_aes_ctr_cra_init,
1493                         .cra_exit = safexcel_skcipher_cra_exit,
1494                         .cra_module = THIS_MODULE,
1495                 },
1496         },
1497 };
1498
1499 static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key,
1500                                unsigned int len)
1501 {
1502         struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm);
1503         struct safexcel_crypto_priv *priv = ctx->priv;
1504         int ret;
1505
1506         ret = verify_skcipher_des_key(ctfm, key);
1507         if (ret)
1508                 return ret;
1509
1510         /* if context exits and key changed, need to invalidate it */
1511         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma)
1512                 if (memcmp(ctx->key, key, len))
1513                         ctx->base.needs_inv = true;
1514
1515         memcpy(ctx->key, key, len);
1516         ctx->key_len = len;
1517
1518         return 0;
1519 }
1520
1521 static int safexcel_skcipher_des_cbc_cra_init(struct crypto_tfm *tfm)
1522 {
1523         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1524
1525         safexcel_skcipher_cra_init(tfm);
1526         ctx->alg  = SAFEXCEL_DES;
1527         ctx->blocksz = DES_BLOCK_SIZE;
1528         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1529         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
1530         return 0;
1531 }
1532
1533 struct safexcel_alg_template safexcel_alg_cbc_des = {
1534         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1535         .algo_mask = SAFEXCEL_ALG_DES,
1536         .alg.skcipher = {
1537                 .setkey = safexcel_des_setkey,
1538                 .encrypt = safexcel_encrypt,
1539                 .decrypt = safexcel_decrypt,
1540                 .min_keysize = DES_KEY_SIZE,
1541                 .max_keysize = DES_KEY_SIZE,
1542                 .ivsize = DES_BLOCK_SIZE,
1543                 .base = {
1544                         .cra_name = "cbc(des)",
1545                         .cra_driver_name = "safexcel-cbc-des",
1546                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1547                         .cra_flags = CRYPTO_ALG_ASYNC |
1548                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1549                         .cra_blocksize = DES_BLOCK_SIZE,
1550                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1551                         .cra_alignmask = 0,
1552                         .cra_init = safexcel_skcipher_des_cbc_cra_init,
1553                         .cra_exit = safexcel_skcipher_cra_exit,
1554                         .cra_module = THIS_MODULE,
1555                 },
1556         },
1557 };
1558
1559 static int safexcel_skcipher_des_ecb_cra_init(struct crypto_tfm *tfm)
1560 {
1561         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1562
1563         safexcel_skcipher_cra_init(tfm);
1564         ctx->alg  = SAFEXCEL_DES;
1565         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
1566         ctx->blocksz = 0;
1567         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1568         return 0;
1569 }
1570
1571 struct safexcel_alg_template safexcel_alg_ecb_des = {
1572         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1573         .algo_mask = SAFEXCEL_ALG_DES,
1574         .alg.skcipher = {
1575                 .setkey = safexcel_des_setkey,
1576                 .encrypt = safexcel_encrypt,
1577                 .decrypt = safexcel_decrypt,
1578                 .min_keysize = DES_KEY_SIZE,
1579                 .max_keysize = DES_KEY_SIZE,
1580                 .base = {
1581                         .cra_name = "ecb(des)",
1582                         .cra_driver_name = "safexcel-ecb-des",
1583                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1584                         .cra_flags = CRYPTO_ALG_ASYNC |
1585                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1586                         .cra_blocksize = DES_BLOCK_SIZE,
1587                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1588                         .cra_alignmask = 0,
1589                         .cra_init = safexcel_skcipher_des_ecb_cra_init,
1590                         .cra_exit = safexcel_skcipher_cra_exit,
1591                         .cra_module = THIS_MODULE,
1592                 },
1593         },
1594 };
1595
1596 static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm,
1597                                    const u8 *key, unsigned int len)
1598 {
1599         struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm);
1600         struct safexcel_crypto_priv *priv = ctx->priv;
1601         int err;
1602
1603         err = verify_skcipher_des3_key(ctfm, key);
1604         if (err)
1605                 return err;
1606
1607         /* if context exits and key changed, need to invalidate it */
1608         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma)
1609                 if (memcmp(ctx->key, key, len))
1610                         ctx->base.needs_inv = true;
1611
1612         memcpy(ctx->key, key, len);
1613         ctx->key_len = len;
1614
1615         return 0;
1616 }
1617
1618 static int safexcel_skcipher_des3_cbc_cra_init(struct crypto_tfm *tfm)
1619 {
1620         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1621
1622         safexcel_skcipher_cra_init(tfm);
1623         ctx->alg  = SAFEXCEL_3DES;
1624         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
1625         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1626         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
1627         return 0;
1628 }
1629
1630 struct safexcel_alg_template safexcel_alg_cbc_des3_ede = {
1631         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1632         .algo_mask = SAFEXCEL_ALG_DES,
1633         .alg.skcipher = {
1634                 .setkey = safexcel_des3_ede_setkey,
1635                 .encrypt = safexcel_encrypt,
1636                 .decrypt = safexcel_decrypt,
1637                 .min_keysize = DES3_EDE_KEY_SIZE,
1638                 .max_keysize = DES3_EDE_KEY_SIZE,
1639                 .ivsize = DES3_EDE_BLOCK_SIZE,
1640                 .base = {
1641                         .cra_name = "cbc(des3_ede)",
1642                         .cra_driver_name = "safexcel-cbc-des3_ede",
1643                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1644                         .cra_flags = CRYPTO_ALG_ASYNC |
1645                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1646                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1647                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1648                         .cra_alignmask = 0,
1649                         .cra_init = safexcel_skcipher_des3_cbc_cra_init,
1650                         .cra_exit = safexcel_skcipher_cra_exit,
1651                         .cra_module = THIS_MODULE,
1652                 },
1653         },
1654 };
1655
1656 static int safexcel_skcipher_des3_ecb_cra_init(struct crypto_tfm *tfm)
1657 {
1658         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1659
1660         safexcel_skcipher_cra_init(tfm);
1661         ctx->alg  = SAFEXCEL_3DES;
1662         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
1663         ctx->blocksz = 0;
1664         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1665         return 0;
1666 }
1667
1668 struct safexcel_alg_template safexcel_alg_ecb_des3_ede = {
1669         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1670         .algo_mask = SAFEXCEL_ALG_DES,
1671         .alg.skcipher = {
1672                 .setkey = safexcel_des3_ede_setkey,
1673                 .encrypt = safexcel_encrypt,
1674                 .decrypt = safexcel_decrypt,
1675                 .min_keysize = DES3_EDE_KEY_SIZE,
1676                 .max_keysize = DES3_EDE_KEY_SIZE,
1677                 .base = {
1678                         .cra_name = "ecb(des3_ede)",
1679                         .cra_driver_name = "safexcel-ecb-des3_ede",
1680                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1681                         .cra_flags = CRYPTO_ALG_ASYNC |
1682                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1683                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1684                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1685                         .cra_alignmask = 0,
1686                         .cra_init = safexcel_skcipher_des3_ecb_cra_init,
1687                         .cra_exit = safexcel_skcipher_cra_exit,
1688                         .cra_module = THIS_MODULE,
1689                 },
1690         },
1691 };
1692
1693 static int safexcel_aead_encrypt(struct aead_request *req)
1694 {
1695         struct safexcel_cipher_req *creq = aead_request_ctx(req);
1696
1697         return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT);
1698 }
1699
1700 static int safexcel_aead_decrypt(struct aead_request *req)
1701 {
1702         struct safexcel_cipher_req *creq = aead_request_ctx(req);
1703
1704         return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT);
1705 }
1706
1707 static int safexcel_aead_cra_init(struct crypto_tfm *tfm)
1708 {
1709         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1710         struct safexcel_alg_template *tmpl =
1711                 container_of(tfm->__crt_alg, struct safexcel_alg_template,
1712                              alg.aead.base);
1713
1714         crypto_aead_set_reqsize(__crypto_aead_cast(tfm),
1715                                 sizeof(struct safexcel_cipher_req));
1716
1717         ctx->priv = tmpl->priv;
1718
1719         ctx->alg  = SAFEXCEL_AES; /* default */
1720         ctx->blocksz = AES_BLOCK_SIZE;
1721         ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD;
1722         ctx->ctrinit = 1;
1723         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; /* default */
1724         ctx->aead = true;
1725         ctx->base.send = safexcel_aead_send;
1726         ctx->base.handle_result = safexcel_aead_handle_result;
1727         return 0;
1728 }
1729
1730 static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm)
1731 {
1732         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1733
1734         safexcel_aead_cra_init(tfm);
1735         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
1736         ctx->state_sz = SHA1_DIGEST_SIZE;
1737         return 0;
1738 }
1739
1740 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = {
1741         .type = SAFEXCEL_ALG_TYPE_AEAD,
1742         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1,
1743         .alg.aead = {
1744                 .setkey = safexcel_aead_setkey,
1745                 .encrypt = safexcel_aead_encrypt,
1746                 .decrypt = safexcel_aead_decrypt,
1747                 .ivsize = AES_BLOCK_SIZE,
1748                 .maxauthsize = SHA1_DIGEST_SIZE,
1749                 .base = {
1750                         .cra_name = "authenc(hmac(sha1),cbc(aes))",
1751                         .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes",
1752                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1753                         .cra_flags = CRYPTO_ALG_ASYNC |
1754                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1755                         .cra_blocksize = AES_BLOCK_SIZE,
1756                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1757                         .cra_alignmask = 0,
1758                         .cra_init = safexcel_aead_sha1_cra_init,
1759                         .cra_exit = safexcel_aead_cra_exit,
1760                         .cra_module = THIS_MODULE,
1761                 },
1762         },
1763 };
1764
1765 static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm)
1766 {
1767         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1768
1769         safexcel_aead_cra_init(tfm);
1770         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1771         ctx->state_sz = SHA256_DIGEST_SIZE;
1772         return 0;
1773 }
1774
1775 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = {
1776         .type = SAFEXCEL_ALG_TYPE_AEAD,
1777         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
1778         .alg.aead = {
1779                 .setkey = safexcel_aead_setkey,
1780                 .encrypt = safexcel_aead_encrypt,
1781                 .decrypt = safexcel_aead_decrypt,
1782                 .ivsize = AES_BLOCK_SIZE,
1783                 .maxauthsize = SHA256_DIGEST_SIZE,
1784                 .base = {
1785                         .cra_name = "authenc(hmac(sha256),cbc(aes))",
1786                         .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes",
1787                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1788                         .cra_flags = CRYPTO_ALG_ASYNC |
1789                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1790                         .cra_blocksize = AES_BLOCK_SIZE,
1791                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1792                         .cra_alignmask = 0,
1793                         .cra_init = safexcel_aead_sha256_cra_init,
1794                         .cra_exit = safexcel_aead_cra_exit,
1795                         .cra_module = THIS_MODULE,
1796                 },
1797         },
1798 };
1799
1800 static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm)
1801 {
1802         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1803
1804         safexcel_aead_cra_init(tfm);
1805         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1806         ctx->state_sz = SHA256_DIGEST_SIZE;
1807         return 0;
1808 }
1809
1810 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = {
1811         .type = SAFEXCEL_ALG_TYPE_AEAD,
1812         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
1813         .alg.aead = {
1814                 .setkey = safexcel_aead_setkey,
1815                 .encrypt = safexcel_aead_encrypt,
1816                 .decrypt = safexcel_aead_decrypt,
1817                 .ivsize = AES_BLOCK_SIZE,
1818                 .maxauthsize = SHA224_DIGEST_SIZE,
1819                 .base = {
1820                         .cra_name = "authenc(hmac(sha224),cbc(aes))",
1821                         .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes",
1822                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1823                         .cra_flags = CRYPTO_ALG_ASYNC |
1824                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1825                         .cra_blocksize = AES_BLOCK_SIZE,
1826                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1827                         .cra_alignmask = 0,
1828                         .cra_init = safexcel_aead_sha224_cra_init,
1829                         .cra_exit = safexcel_aead_cra_exit,
1830                         .cra_module = THIS_MODULE,
1831                 },
1832         },
1833 };
1834
1835 static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm)
1836 {
1837         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1838
1839         safexcel_aead_cra_init(tfm);
1840         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1841         ctx->state_sz = SHA512_DIGEST_SIZE;
1842         return 0;
1843 }
1844
1845 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = {
1846         .type = SAFEXCEL_ALG_TYPE_AEAD,
1847         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
1848         .alg.aead = {
1849                 .setkey = safexcel_aead_setkey,
1850                 .encrypt = safexcel_aead_encrypt,
1851                 .decrypt = safexcel_aead_decrypt,
1852                 .ivsize = AES_BLOCK_SIZE,
1853                 .maxauthsize = SHA512_DIGEST_SIZE,
1854                 .base = {
1855                         .cra_name = "authenc(hmac(sha512),cbc(aes))",
1856                         .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes",
1857                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1858                         .cra_flags = CRYPTO_ALG_ASYNC |
1859                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1860                         .cra_blocksize = AES_BLOCK_SIZE,
1861                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1862                         .cra_alignmask = 0,
1863                         .cra_init = safexcel_aead_sha512_cra_init,
1864                         .cra_exit = safexcel_aead_cra_exit,
1865                         .cra_module = THIS_MODULE,
1866                 },
1867         },
1868 };
1869
1870 static int safexcel_aead_sha384_cra_init(struct crypto_tfm *tfm)
1871 {
1872         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1873
1874         safexcel_aead_cra_init(tfm);
1875         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1876         ctx->state_sz = SHA512_DIGEST_SIZE;
1877         return 0;
1878 }
1879
1880 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = {
1881         .type = SAFEXCEL_ALG_TYPE_AEAD,
1882         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
1883         .alg.aead = {
1884                 .setkey = safexcel_aead_setkey,
1885                 .encrypt = safexcel_aead_encrypt,
1886                 .decrypt = safexcel_aead_decrypt,
1887                 .ivsize = AES_BLOCK_SIZE,
1888                 .maxauthsize = SHA384_DIGEST_SIZE,
1889                 .base = {
1890                         .cra_name = "authenc(hmac(sha384),cbc(aes))",
1891                         .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-aes",
1892                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1893                         .cra_flags = CRYPTO_ALG_ASYNC |
1894                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1895                         .cra_blocksize = AES_BLOCK_SIZE,
1896                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1897                         .cra_alignmask = 0,
1898                         .cra_init = safexcel_aead_sha384_cra_init,
1899                         .cra_exit = safexcel_aead_cra_exit,
1900                         .cra_module = THIS_MODULE,
1901                 },
1902         },
1903 };
1904
1905 static int safexcel_aead_sha1_des3_cra_init(struct crypto_tfm *tfm)
1906 {
1907         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1908
1909         safexcel_aead_sha1_cra_init(tfm);
1910         ctx->alg = SAFEXCEL_3DES; /* override default */
1911         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
1912         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1913         return 0;
1914 }
1915
1916 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede = {
1917         .type = SAFEXCEL_ALG_TYPE_AEAD,
1918         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1,
1919         .alg.aead = {
1920                 .setkey = safexcel_aead_setkey,
1921                 .encrypt = safexcel_aead_encrypt,
1922                 .decrypt = safexcel_aead_decrypt,
1923                 .ivsize = DES3_EDE_BLOCK_SIZE,
1924                 .maxauthsize = SHA1_DIGEST_SIZE,
1925                 .base = {
1926                         .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
1927                         .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des3_ede",
1928                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1929                         .cra_flags = CRYPTO_ALG_ASYNC |
1930                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1931                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1932                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1933                         .cra_alignmask = 0,
1934                         .cra_init = safexcel_aead_sha1_des3_cra_init,
1935                         .cra_exit = safexcel_aead_cra_exit,
1936                         .cra_module = THIS_MODULE,
1937                 },
1938         },
1939 };
1940
1941 static int safexcel_aead_sha256_des3_cra_init(struct crypto_tfm *tfm)
1942 {
1943         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1944
1945         safexcel_aead_sha256_cra_init(tfm);
1946         ctx->alg = SAFEXCEL_3DES; /* override default */
1947         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
1948         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1949         return 0;
1950 }
1951
1952 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ede = {
1953         .type = SAFEXCEL_ALG_TYPE_AEAD,
1954         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256,
1955         .alg.aead = {
1956                 .setkey = safexcel_aead_setkey,
1957                 .encrypt = safexcel_aead_encrypt,
1958                 .decrypt = safexcel_aead_decrypt,
1959                 .ivsize = DES3_EDE_BLOCK_SIZE,
1960                 .maxauthsize = SHA256_DIGEST_SIZE,
1961                 .base = {
1962                         .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
1963                         .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des3_ede",
1964                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1965                         .cra_flags = CRYPTO_ALG_ASYNC |
1966                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1967                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1968                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1969                         .cra_alignmask = 0,
1970                         .cra_init = safexcel_aead_sha256_des3_cra_init,
1971                         .cra_exit = safexcel_aead_cra_exit,
1972                         .cra_module = THIS_MODULE,
1973                 },
1974         },
1975 };
1976
1977 static int safexcel_aead_sha224_des3_cra_init(struct crypto_tfm *tfm)
1978 {
1979         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1980
1981         safexcel_aead_sha224_cra_init(tfm);
1982         ctx->alg = SAFEXCEL_3DES; /* override default */
1983         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
1984         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1985         return 0;
1986 }
1987
1988 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede = {
1989         .type = SAFEXCEL_ALG_TYPE_AEAD,
1990         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256,
1991         .alg.aead = {
1992                 .setkey = safexcel_aead_setkey,
1993                 .encrypt = safexcel_aead_encrypt,
1994                 .decrypt = safexcel_aead_decrypt,
1995                 .ivsize = DES3_EDE_BLOCK_SIZE,
1996                 .maxauthsize = SHA224_DIGEST_SIZE,
1997                 .base = {
1998                         .cra_name = "authenc(hmac(sha224),cbc(des3_ede))",
1999                         .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des3_ede",
2000                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2001                         .cra_flags = CRYPTO_ALG_ASYNC |
2002                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2003                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2004                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2005                         .cra_alignmask = 0,
2006                         .cra_init = safexcel_aead_sha224_des3_cra_init,
2007                         .cra_exit = safexcel_aead_cra_exit,
2008                         .cra_module = THIS_MODULE,
2009                 },
2010         },
2011 };
2012
2013 static int safexcel_aead_sha512_des3_cra_init(struct crypto_tfm *tfm)
2014 {
2015         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2016
2017         safexcel_aead_sha512_cra_init(tfm);
2018         ctx->alg = SAFEXCEL_3DES; /* override default */
2019         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
2020         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2021         return 0;
2022 }
2023
2024 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede = {
2025         .type = SAFEXCEL_ALG_TYPE_AEAD,
2026         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512,
2027         .alg.aead = {
2028                 .setkey = safexcel_aead_setkey,
2029                 .encrypt = safexcel_aead_encrypt,
2030                 .decrypt = safexcel_aead_decrypt,
2031                 .ivsize = DES3_EDE_BLOCK_SIZE,
2032                 .maxauthsize = SHA512_DIGEST_SIZE,
2033                 .base = {
2034                         .cra_name = "authenc(hmac(sha512),cbc(des3_ede))",
2035                         .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des3_ede",
2036                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2037                         .cra_flags = CRYPTO_ALG_ASYNC |
2038                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2039                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2040                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2041                         .cra_alignmask = 0,
2042                         .cra_init = safexcel_aead_sha512_des3_cra_init,
2043                         .cra_exit = safexcel_aead_cra_exit,
2044                         .cra_module = THIS_MODULE,
2045                 },
2046         },
2047 };
2048
2049 static int safexcel_aead_sha384_des3_cra_init(struct crypto_tfm *tfm)
2050 {
2051         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2052
2053         safexcel_aead_sha384_cra_init(tfm);
2054         ctx->alg = SAFEXCEL_3DES; /* override default */
2055         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
2056         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2057         return 0;
2058 }
2059
2060 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede = {
2061         .type = SAFEXCEL_ALG_TYPE_AEAD,
2062         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512,
2063         .alg.aead = {
2064                 .setkey = safexcel_aead_setkey,
2065                 .encrypt = safexcel_aead_encrypt,
2066                 .decrypt = safexcel_aead_decrypt,
2067                 .ivsize = DES3_EDE_BLOCK_SIZE,
2068                 .maxauthsize = SHA384_DIGEST_SIZE,
2069                 .base = {
2070                         .cra_name = "authenc(hmac(sha384),cbc(des3_ede))",
2071                         .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des3_ede",
2072                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2073                         .cra_flags = CRYPTO_ALG_ASYNC |
2074                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2075                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2076                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2077                         .cra_alignmask = 0,
2078                         .cra_init = safexcel_aead_sha384_des3_cra_init,
2079                         .cra_exit = safexcel_aead_cra_exit,
2080                         .cra_module = THIS_MODULE,
2081                 },
2082         },
2083 };
2084
2085 static int safexcel_aead_sha1_des_cra_init(struct crypto_tfm *tfm)
2086 {
2087         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2088
2089         safexcel_aead_sha1_cra_init(tfm);
2090         ctx->alg = SAFEXCEL_DES; /* override default */
2091         ctx->blocksz = DES_BLOCK_SIZE;
2092         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2093         return 0;
2094 }
2095
2096 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des = {
2097         .type = SAFEXCEL_ALG_TYPE_AEAD,
2098         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1,
2099         .alg.aead = {
2100                 .setkey = safexcel_aead_setkey,
2101                 .encrypt = safexcel_aead_encrypt,
2102                 .decrypt = safexcel_aead_decrypt,
2103                 .ivsize = DES_BLOCK_SIZE,
2104                 .maxauthsize = SHA1_DIGEST_SIZE,
2105                 .base = {
2106                         .cra_name = "authenc(hmac(sha1),cbc(des))",
2107                         .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des",
2108                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2109                         .cra_flags = CRYPTO_ALG_ASYNC |
2110                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2111                         .cra_blocksize = DES_BLOCK_SIZE,
2112                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2113                         .cra_alignmask = 0,
2114                         .cra_init = safexcel_aead_sha1_des_cra_init,
2115                         .cra_exit = safexcel_aead_cra_exit,
2116                         .cra_module = THIS_MODULE,
2117                 },
2118         },
2119 };
2120
2121 static int safexcel_aead_sha256_des_cra_init(struct crypto_tfm *tfm)
2122 {
2123         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2124
2125         safexcel_aead_sha256_cra_init(tfm);
2126         ctx->alg = SAFEXCEL_DES; /* override default */
2127         ctx->blocksz = DES_BLOCK_SIZE;
2128         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2129         return 0;
2130 }
2131
2132 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des = {
2133         .type = SAFEXCEL_ALG_TYPE_AEAD,
2134         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256,
2135         .alg.aead = {
2136                 .setkey = safexcel_aead_setkey,
2137                 .encrypt = safexcel_aead_encrypt,
2138                 .decrypt = safexcel_aead_decrypt,
2139                 .ivsize = DES_BLOCK_SIZE,
2140                 .maxauthsize = SHA256_DIGEST_SIZE,
2141                 .base = {
2142                         .cra_name = "authenc(hmac(sha256),cbc(des))",
2143                         .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des",
2144                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2145                         .cra_flags = CRYPTO_ALG_ASYNC |
2146                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2147                         .cra_blocksize = DES_BLOCK_SIZE,
2148                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2149                         .cra_alignmask = 0,
2150                         .cra_init = safexcel_aead_sha256_des_cra_init,
2151                         .cra_exit = safexcel_aead_cra_exit,
2152                         .cra_module = THIS_MODULE,
2153                 },
2154         },
2155 };
2156
2157 static int safexcel_aead_sha224_des_cra_init(struct crypto_tfm *tfm)
2158 {
2159         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2160
2161         safexcel_aead_sha224_cra_init(tfm);
2162         ctx->alg = SAFEXCEL_DES; /* override default */
2163         ctx->blocksz = DES_BLOCK_SIZE;
2164         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2165         return 0;
2166 }
2167
2168 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des = {
2169         .type = SAFEXCEL_ALG_TYPE_AEAD,
2170         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256,
2171         .alg.aead = {
2172                 .setkey = safexcel_aead_setkey,
2173                 .encrypt = safexcel_aead_encrypt,
2174                 .decrypt = safexcel_aead_decrypt,
2175                 .ivsize = DES_BLOCK_SIZE,
2176                 .maxauthsize = SHA224_DIGEST_SIZE,
2177                 .base = {
2178                         .cra_name = "authenc(hmac(sha224),cbc(des))",
2179                         .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des",
2180                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2181                         .cra_flags = CRYPTO_ALG_ASYNC |
2182                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2183                         .cra_blocksize = DES_BLOCK_SIZE,
2184                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2185                         .cra_alignmask = 0,
2186                         .cra_init = safexcel_aead_sha224_des_cra_init,
2187                         .cra_exit = safexcel_aead_cra_exit,
2188                         .cra_module = THIS_MODULE,
2189                 },
2190         },
2191 };
2192
2193 static int safexcel_aead_sha512_des_cra_init(struct crypto_tfm *tfm)
2194 {
2195         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2196
2197         safexcel_aead_sha512_cra_init(tfm);
2198         ctx->alg = SAFEXCEL_DES; /* override default */
2199         ctx->blocksz = DES_BLOCK_SIZE;
2200         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2201         return 0;
2202 }
2203
2204 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des = {
2205         .type = SAFEXCEL_ALG_TYPE_AEAD,
2206         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512,
2207         .alg.aead = {
2208                 .setkey = safexcel_aead_setkey,
2209                 .encrypt = safexcel_aead_encrypt,
2210                 .decrypt = safexcel_aead_decrypt,
2211                 .ivsize = DES_BLOCK_SIZE,
2212                 .maxauthsize = SHA512_DIGEST_SIZE,
2213                 .base = {
2214                         .cra_name = "authenc(hmac(sha512),cbc(des))",
2215                         .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des",
2216                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2217                         .cra_flags = CRYPTO_ALG_ASYNC |
2218                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2219                         .cra_blocksize = DES_BLOCK_SIZE,
2220                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2221                         .cra_alignmask = 0,
2222                         .cra_init = safexcel_aead_sha512_des_cra_init,
2223                         .cra_exit = safexcel_aead_cra_exit,
2224                         .cra_module = THIS_MODULE,
2225                 },
2226         },
2227 };
2228
2229 static int safexcel_aead_sha384_des_cra_init(struct crypto_tfm *tfm)
2230 {
2231         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2232
2233         safexcel_aead_sha384_cra_init(tfm);
2234         ctx->alg = SAFEXCEL_DES; /* override default */
2235         ctx->blocksz = DES_BLOCK_SIZE;
2236         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2237         return 0;
2238 }
2239
2240 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des = {
2241         .type = SAFEXCEL_ALG_TYPE_AEAD,
2242         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512,
2243         .alg.aead = {
2244                 .setkey = safexcel_aead_setkey,
2245                 .encrypt = safexcel_aead_encrypt,
2246                 .decrypt = safexcel_aead_decrypt,
2247                 .ivsize = DES_BLOCK_SIZE,
2248                 .maxauthsize = SHA384_DIGEST_SIZE,
2249                 .base = {
2250                         .cra_name = "authenc(hmac(sha384),cbc(des))",
2251                         .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des",
2252                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2253                         .cra_flags = CRYPTO_ALG_ASYNC |
2254                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2255                         .cra_blocksize = DES_BLOCK_SIZE,
2256                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2257                         .cra_alignmask = 0,
2258                         .cra_init = safexcel_aead_sha384_des_cra_init,
2259                         .cra_exit = safexcel_aead_cra_exit,
2260                         .cra_module = THIS_MODULE,
2261                 },
2262         },
2263 };
2264
2265 static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm)
2266 {
2267         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2268
2269         safexcel_aead_sha1_cra_init(tfm);
2270         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
2271         return 0;
2272 }
2273
2274 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_aes = {
2275         .type = SAFEXCEL_ALG_TYPE_AEAD,
2276         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1,
2277         .alg.aead = {
2278                 .setkey = safexcel_aead_setkey,
2279                 .encrypt = safexcel_aead_encrypt,
2280                 .decrypt = safexcel_aead_decrypt,
2281                 .ivsize = CTR_RFC3686_IV_SIZE,
2282                 .maxauthsize = SHA1_DIGEST_SIZE,
2283                 .base = {
2284                         .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
2285                         .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-aes",
2286                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2287                         .cra_flags = CRYPTO_ALG_ASYNC |
2288                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2289                         .cra_blocksize = 1,
2290                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2291                         .cra_alignmask = 0,
2292                         .cra_init = safexcel_aead_sha1_ctr_cra_init,
2293                         .cra_exit = safexcel_aead_cra_exit,
2294                         .cra_module = THIS_MODULE,
2295                 },
2296         },
2297 };
2298
2299 static int safexcel_aead_sha256_ctr_cra_init(struct crypto_tfm *tfm)
2300 {
2301         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2302
2303         safexcel_aead_sha256_cra_init(tfm);
2304         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
2305         return 0;
2306 }
2307
2308 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_ctr_aes = {
2309         .type = SAFEXCEL_ALG_TYPE_AEAD,
2310         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
2311         .alg.aead = {
2312                 .setkey = safexcel_aead_setkey,
2313                 .encrypt = safexcel_aead_encrypt,
2314                 .decrypt = safexcel_aead_decrypt,
2315                 .ivsize = CTR_RFC3686_IV_SIZE,
2316                 .maxauthsize = SHA256_DIGEST_SIZE,
2317                 .base = {
2318                         .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
2319                         .cra_driver_name = "safexcel-authenc-hmac-sha256-ctr-aes",
2320                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2321                         .cra_flags = CRYPTO_ALG_ASYNC |
2322                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2323                         .cra_blocksize = 1,
2324                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2325                         .cra_alignmask = 0,
2326                         .cra_init = safexcel_aead_sha256_ctr_cra_init,
2327                         .cra_exit = safexcel_aead_cra_exit,
2328                         .cra_module = THIS_MODULE,
2329                 },
2330         },
2331 };
2332
2333 static int safexcel_aead_sha224_ctr_cra_init(struct crypto_tfm *tfm)
2334 {
2335         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2336
2337         safexcel_aead_sha224_cra_init(tfm);
2338         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
2339         return 0;
2340 }
2341
2342 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_ctr_aes = {
2343         .type = SAFEXCEL_ALG_TYPE_AEAD,
2344         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
2345         .alg.aead = {
2346                 .setkey = safexcel_aead_setkey,
2347                 .encrypt = safexcel_aead_encrypt,
2348                 .decrypt = safexcel_aead_decrypt,
2349                 .ivsize = CTR_RFC3686_IV_SIZE,
2350                 .maxauthsize = SHA224_DIGEST_SIZE,
2351                 .base = {
2352                         .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
2353                         .cra_driver_name = "safexcel-authenc-hmac-sha224-ctr-aes",
2354                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2355                         .cra_flags = CRYPTO_ALG_ASYNC |
2356                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2357                         .cra_blocksize = 1,
2358                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2359                         .cra_alignmask = 0,
2360                         .cra_init = safexcel_aead_sha224_ctr_cra_init,
2361                         .cra_exit = safexcel_aead_cra_exit,
2362                         .cra_module = THIS_MODULE,
2363                 },
2364         },
2365 };
2366
2367 static int safexcel_aead_sha512_ctr_cra_init(struct crypto_tfm *tfm)
2368 {
2369         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2370
2371         safexcel_aead_sha512_cra_init(tfm);
2372         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
2373         return 0;
2374 }
2375
2376 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes = {
2377         .type = SAFEXCEL_ALG_TYPE_AEAD,
2378         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
2379         .alg.aead = {
2380                 .setkey = safexcel_aead_setkey,
2381                 .encrypt = safexcel_aead_encrypt,
2382                 .decrypt = safexcel_aead_decrypt,
2383                 .ivsize = CTR_RFC3686_IV_SIZE,
2384                 .maxauthsize = SHA512_DIGEST_SIZE,
2385                 .base = {
2386                         .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
2387                         .cra_driver_name = "safexcel-authenc-hmac-sha512-ctr-aes",
2388                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2389                         .cra_flags = CRYPTO_ALG_ASYNC |
2390                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2391                         .cra_blocksize = 1,
2392                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2393                         .cra_alignmask = 0,
2394                         .cra_init = safexcel_aead_sha512_ctr_cra_init,
2395                         .cra_exit = safexcel_aead_cra_exit,
2396                         .cra_module = THIS_MODULE,
2397                 },
2398         },
2399 };
2400
2401 static int safexcel_aead_sha384_ctr_cra_init(struct crypto_tfm *tfm)
2402 {
2403         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2404
2405         safexcel_aead_sha384_cra_init(tfm);
2406         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
2407         return 0;
2408 }
2409
2410 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_ctr_aes = {
2411         .type = SAFEXCEL_ALG_TYPE_AEAD,
2412         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
2413         .alg.aead = {
2414                 .setkey = safexcel_aead_setkey,
2415                 .encrypt = safexcel_aead_encrypt,
2416                 .decrypt = safexcel_aead_decrypt,
2417                 .ivsize = CTR_RFC3686_IV_SIZE,
2418                 .maxauthsize = SHA384_DIGEST_SIZE,
2419                 .base = {
2420                         .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
2421                         .cra_driver_name = "safexcel-authenc-hmac-sha384-ctr-aes",
2422                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2423                         .cra_flags = CRYPTO_ALG_ASYNC |
2424                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2425                         .cra_blocksize = 1,
2426                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2427                         .cra_alignmask = 0,
2428                         .cra_init = safexcel_aead_sha384_ctr_cra_init,
2429                         .cra_exit = safexcel_aead_cra_exit,
2430                         .cra_module = THIS_MODULE,
2431                 },
2432         },
2433 };
2434
2435 static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm,
2436                                            const u8 *key, unsigned int len)
2437 {
2438         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
2439         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2440         struct safexcel_crypto_priv *priv = ctx->priv;
2441         struct crypto_aes_ctx aes;
2442         int ret, i;
2443         unsigned int keylen;
2444
2445         /* Check for illegal XTS keys */
2446         ret = xts_verify_key(ctfm, key, len);
2447         if (ret)
2448                 return ret;
2449
2450         /* Only half of the key data is cipher key */
2451         keylen = (len >> 1);
2452         ret = aes_expandkey(&aes, key, keylen);
2453         if (ret)
2454                 return ret;
2455
2456         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2457                 for (i = 0; i < keylen / sizeof(u32); i++) {
2458                         if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) {
2459                                 ctx->base.needs_inv = true;
2460                                 break;
2461                         }
2462                 }
2463         }
2464
2465         for (i = 0; i < keylen / sizeof(u32); i++)
2466                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
2467
2468         /* The other half is the tweak key */
2469         ret = aes_expandkey(&aes, (u8 *)(key + keylen), keylen);
2470         if (ret)
2471                 return ret;
2472
2473         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2474                 for (i = 0; i < keylen / sizeof(u32); i++) {
2475                         if (le32_to_cpu(ctx->key[i + keylen / sizeof(u32)]) !=
2476                             aes.key_enc[i]) {
2477                                 ctx->base.needs_inv = true;
2478                                 break;
2479                         }
2480                 }
2481         }
2482
2483         for (i = 0; i < keylen / sizeof(u32); i++)
2484                 ctx->key[i + keylen / sizeof(u32)] =
2485                         cpu_to_le32(aes.key_enc[i]);
2486
2487         ctx->key_len = keylen << 1;
2488
2489         memzero_explicit(&aes, sizeof(aes));
2490         return 0;
2491 }
2492
2493 static int safexcel_skcipher_aes_xts_cra_init(struct crypto_tfm *tfm)
2494 {
2495         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2496
2497         safexcel_skcipher_cra_init(tfm);
2498         ctx->alg  = SAFEXCEL_AES;
2499         ctx->blocksz = AES_BLOCK_SIZE;
2500         ctx->xts  = 1;
2501         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XTS;
2502         return 0;
2503 }
2504
2505 static int safexcel_encrypt_xts(struct skcipher_request *req)
2506 {
2507         if (req->cryptlen < XTS_BLOCK_SIZE)
2508                 return -EINVAL;
2509         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
2510                                   SAFEXCEL_ENCRYPT);
2511 }
2512
2513 static int safexcel_decrypt_xts(struct skcipher_request *req)
2514 {
2515         if (req->cryptlen < XTS_BLOCK_SIZE)
2516                 return -EINVAL;
2517         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
2518                                   SAFEXCEL_DECRYPT);
2519 }
2520
2521 struct safexcel_alg_template safexcel_alg_xts_aes = {
2522         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
2523         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XTS,
2524         .alg.skcipher = {
2525                 .setkey = safexcel_skcipher_aesxts_setkey,
2526                 .encrypt = safexcel_encrypt_xts,
2527                 .decrypt = safexcel_decrypt_xts,
2528                 /* XTS actually uses 2 AES keys glued together */
2529                 .min_keysize = AES_MIN_KEY_SIZE * 2,
2530                 .max_keysize = AES_MAX_KEY_SIZE * 2,
2531                 .ivsize = XTS_BLOCK_SIZE,
2532                 .base = {
2533                         .cra_name = "xts(aes)",
2534                         .cra_driver_name = "safexcel-xts-aes",
2535                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2536                         .cra_flags = CRYPTO_ALG_ASYNC |
2537                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2538                         .cra_blocksize = XTS_BLOCK_SIZE,
2539                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2540                         .cra_alignmask = 0,
2541                         .cra_init = safexcel_skcipher_aes_xts_cra_init,
2542                         .cra_exit = safexcel_skcipher_cra_exit,
2543                         .cra_module = THIS_MODULE,
2544                 },
2545         },
2546 };
2547
2548 static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key,
2549                                     unsigned int len)
2550 {
2551         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
2552         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2553         struct safexcel_crypto_priv *priv = ctx->priv;
2554         struct crypto_aes_ctx aes;
2555         u32 hashkey[AES_BLOCK_SIZE >> 2];
2556         int ret, i;
2557
2558         ret = aes_expandkey(&aes, key, len);
2559         if (ret) {
2560                 memzero_explicit(&aes, sizeof(aes));
2561                 return ret;
2562         }
2563
2564         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2565                 for (i = 0; i < len / sizeof(u32); i++) {
2566                         if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) {
2567                                 ctx->base.needs_inv = true;
2568                                 break;
2569                         }
2570                 }
2571         }
2572
2573         for (i = 0; i < len / sizeof(u32); i++)
2574                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
2575
2576         ctx->key_len = len;
2577
2578         /* Compute hash key by encrypting zeroes with cipher key */
2579         crypto_cipher_clear_flags(ctx->hkaes, CRYPTO_TFM_REQ_MASK);
2580         crypto_cipher_set_flags(ctx->hkaes, crypto_aead_get_flags(ctfm) &
2581                                 CRYPTO_TFM_REQ_MASK);
2582         ret = crypto_cipher_setkey(ctx->hkaes, key, len);
2583         if (ret)
2584                 return ret;
2585
2586         memset(hashkey, 0, AES_BLOCK_SIZE);
2587         crypto_cipher_encrypt_one(ctx->hkaes, (u8 *)hashkey, (u8 *)hashkey);
2588
2589         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2590                 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) {
2591                         if (be32_to_cpu(ctx->ipad[i]) != hashkey[i]) {
2592                                 ctx->base.needs_inv = true;
2593                                 break;
2594                         }
2595                 }
2596         }
2597
2598         for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++)
2599                 ctx->ipad[i] = cpu_to_be32(hashkey[i]);
2600
2601         memzero_explicit(hashkey, AES_BLOCK_SIZE);
2602         memzero_explicit(&aes, sizeof(aes));
2603         return 0;
2604 }
2605
2606 static int safexcel_aead_gcm_cra_init(struct crypto_tfm *tfm)
2607 {
2608         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2609
2610         safexcel_aead_cra_init(tfm);
2611         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_GHASH;
2612         ctx->state_sz = GHASH_BLOCK_SIZE;
2613         ctx->xcm = EIP197_XCM_MODE_GCM;
2614         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */
2615
2616         ctx->hkaes = crypto_alloc_cipher("aes", 0, 0);
2617         return PTR_ERR_OR_ZERO(ctx->hkaes);
2618 }
2619
2620 static void safexcel_aead_gcm_cra_exit(struct crypto_tfm *tfm)
2621 {
2622         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2623
2624         crypto_free_cipher(ctx->hkaes);
2625         safexcel_aead_cra_exit(tfm);
2626 }
2627
2628 static int safexcel_aead_gcm_setauthsize(struct crypto_aead *tfm,
2629                                          unsigned int authsize)
2630 {
2631         return crypto_gcm_check_authsize(authsize);
2632 }
2633
2634 struct safexcel_alg_template safexcel_alg_gcm = {
2635         .type = SAFEXCEL_ALG_TYPE_AEAD,
2636         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH,
2637         .alg.aead = {
2638                 .setkey = safexcel_aead_gcm_setkey,
2639                 .setauthsize = safexcel_aead_gcm_setauthsize,
2640                 .encrypt = safexcel_aead_encrypt,
2641                 .decrypt = safexcel_aead_decrypt,
2642                 .ivsize = GCM_AES_IV_SIZE,
2643                 .maxauthsize = GHASH_DIGEST_SIZE,
2644                 .base = {
2645                         .cra_name = "gcm(aes)",
2646                         .cra_driver_name = "safexcel-gcm-aes",
2647                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2648                         .cra_flags = CRYPTO_ALG_ASYNC |
2649                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2650                         .cra_blocksize = 1,
2651                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2652                         .cra_alignmask = 0,
2653                         .cra_init = safexcel_aead_gcm_cra_init,
2654                         .cra_exit = safexcel_aead_gcm_cra_exit,
2655                         .cra_module = THIS_MODULE,
2656                 },
2657         },
2658 };
2659
2660 static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key,
2661                                     unsigned int len)
2662 {
2663         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
2664         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2665         struct safexcel_crypto_priv *priv = ctx->priv;
2666         struct crypto_aes_ctx aes;
2667         int ret, i;
2668
2669         ret = aes_expandkey(&aes, key, len);
2670         if (ret) {
2671                 memzero_explicit(&aes, sizeof(aes));
2672                 return ret;
2673         }
2674
2675         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2676                 for (i = 0; i < len / sizeof(u32); i++) {
2677                         if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) {
2678                                 ctx->base.needs_inv = true;
2679                                 break;
2680                         }
2681                 }
2682         }
2683
2684         for (i = 0; i < len / sizeof(u32); i++) {
2685                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
2686                 ctx->ipad[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] =
2687                         cpu_to_be32(aes.key_enc[i]);
2688         }
2689
2690         ctx->key_len = len;
2691         ctx->state_sz = 2 * AES_BLOCK_SIZE + len;
2692
2693         if (len == AES_KEYSIZE_192)
2694                 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2695         else if (len == AES_KEYSIZE_256)
2696                 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2697         else
2698                 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2699
2700         memzero_explicit(&aes, sizeof(aes));
2701         return 0;
2702 }
2703
2704 static int safexcel_aead_ccm_cra_init(struct crypto_tfm *tfm)
2705 {
2706         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2707
2708         safexcel_aead_cra_init(tfm);
2709         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2710         ctx->state_sz = 3 * AES_BLOCK_SIZE;
2711         ctx->xcm = EIP197_XCM_MODE_CCM;
2712         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */
2713         ctx->ctrinit = 0;
2714         return 0;
2715 }
2716
2717 static int safexcel_aead_ccm_setauthsize(struct crypto_aead *tfm,
2718                                          unsigned int authsize)
2719 {
2720         /* Borrowed from crypto/ccm.c */
2721         switch (authsize) {
2722         case 4:
2723         case 6:
2724         case 8:
2725         case 10:
2726         case 12:
2727         case 14:
2728         case 16:
2729                 break;
2730         default:
2731                 return -EINVAL;
2732         }
2733
2734         return 0;
2735 }
2736
2737 static int safexcel_ccm_encrypt(struct aead_request *req)
2738 {
2739         struct safexcel_cipher_req *creq = aead_request_ctx(req);
2740
2741         if (req->iv[0] < 1 || req->iv[0] > 7)
2742                 return -EINVAL;
2743
2744         return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT);
2745 }
2746
2747 static int safexcel_ccm_decrypt(struct aead_request *req)
2748 {
2749         struct safexcel_cipher_req *creq = aead_request_ctx(req);
2750
2751         if (req->iv[0] < 1 || req->iv[0] > 7)
2752                 return -EINVAL;
2753
2754         return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT);
2755 }
2756
2757 struct safexcel_alg_template safexcel_alg_ccm = {
2758         .type = SAFEXCEL_ALG_TYPE_AEAD,
2759         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL,
2760         .alg.aead = {
2761                 .setkey = safexcel_aead_ccm_setkey,
2762                 .setauthsize = safexcel_aead_ccm_setauthsize,
2763                 .encrypt = safexcel_ccm_encrypt,
2764                 .decrypt = safexcel_ccm_decrypt,
2765                 .ivsize = AES_BLOCK_SIZE,
2766                 .maxauthsize = AES_BLOCK_SIZE,
2767                 .base = {
2768                         .cra_name = "ccm(aes)",
2769                         .cra_driver_name = "safexcel-ccm-aes",
2770                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2771                         .cra_flags = CRYPTO_ALG_ASYNC |
2772                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2773                         .cra_blocksize = 1,
2774                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2775                         .cra_alignmask = 0,
2776                         .cra_init = safexcel_aead_ccm_cra_init,
2777                         .cra_exit = safexcel_aead_cra_exit,
2778                         .cra_module = THIS_MODULE,
2779                 },
2780         },
2781 };
2782
2783 static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx,
2784                                      const u8 *key)
2785 {
2786         struct safexcel_crypto_priv *priv = ctx->priv;
2787
2788         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma)
2789                 if (memcmp(ctx->key, key, CHACHA_KEY_SIZE))
2790                         ctx->base.needs_inv = true;
2791
2792         memcpy(ctx->key, key, CHACHA_KEY_SIZE);
2793         ctx->key_len = CHACHA_KEY_SIZE;
2794 }
2795
2796 static int safexcel_skcipher_chacha20_setkey(struct crypto_skcipher *ctfm,
2797                                              const u8 *key, unsigned int len)
2798 {
2799         struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm);
2800
2801         if (len != CHACHA_KEY_SIZE)
2802                 return -EINVAL;
2803
2804         safexcel_chacha20_setkey(ctx, key);
2805
2806         return 0;
2807 }
2808
2809 static int safexcel_skcipher_chacha20_cra_init(struct crypto_tfm *tfm)
2810 {
2811         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2812
2813         safexcel_skcipher_cra_init(tfm);
2814         ctx->alg  = SAFEXCEL_CHACHA20;
2815         ctx->ctrinit = 0;
2816         ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32;
2817         return 0;
2818 }
2819
2820 struct safexcel_alg_template safexcel_alg_chacha20 = {
2821         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
2822         .algo_mask = SAFEXCEL_ALG_CHACHA20,
2823         .alg.skcipher = {
2824                 .setkey = safexcel_skcipher_chacha20_setkey,
2825                 .encrypt = safexcel_encrypt,
2826                 .decrypt = safexcel_decrypt,
2827                 .min_keysize = CHACHA_KEY_SIZE,
2828                 .max_keysize = CHACHA_KEY_SIZE,
2829                 .ivsize = CHACHA_IV_SIZE,
2830                 .base = {
2831                         .cra_name = "chacha20",
2832                         .cra_driver_name = "safexcel-chacha20",
2833                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2834                         .cra_flags = CRYPTO_ALG_ASYNC |
2835                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2836                         .cra_blocksize = 1,
2837                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2838                         .cra_alignmask = 0,
2839                         .cra_init = safexcel_skcipher_chacha20_cra_init,
2840                         .cra_exit = safexcel_skcipher_cra_exit,
2841                         .cra_module = THIS_MODULE,
2842                 },
2843         },
2844 };
2845
2846 static int safexcel_aead_chachapoly_setkey(struct crypto_aead *ctfm,
2847                                     const u8 *key, unsigned int len)
2848 {
2849         struct safexcel_cipher_ctx *ctx = crypto_aead_ctx(ctfm);
2850
2851         if (ctx->aead  == EIP197_AEAD_TYPE_IPSEC_ESP &&
2852             len > EIP197_AEAD_IPSEC_NONCE_SIZE) {
2853                 /* ESP variant has nonce appended to key */
2854                 len -= EIP197_AEAD_IPSEC_NONCE_SIZE;
2855                 ctx->nonce = *(u32 *)(key + len);
2856         }
2857         if (len != CHACHA_KEY_SIZE)
2858                 return -EINVAL;
2859
2860         safexcel_chacha20_setkey(ctx, key);
2861
2862         return 0;
2863 }
2864
2865 static int safexcel_aead_chachapoly_setauthsize(struct crypto_aead *tfm,
2866                                          unsigned int authsize)
2867 {
2868         if (authsize != POLY1305_DIGEST_SIZE)
2869                 return -EINVAL;
2870         return 0;
2871 }
2872
2873 static int safexcel_aead_chachapoly_crypt(struct aead_request *req,
2874                                           enum safexcel_cipher_direction dir)
2875 {
2876         struct safexcel_cipher_req *creq = aead_request_ctx(req);
2877         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2878         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
2879         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2880         struct aead_request *subreq = aead_request_ctx(req);
2881         u32 key[CHACHA_KEY_SIZE / sizeof(u32) + 1];
2882         int ret = 0;
2883
2884         /*
2885          * Instead of wasting time detecting umpteen silly corner cases,
2886          * just dump all "small" requests to the fallback implementation.
2887          * HW would not be faster on such small requests anyway.
2888          */
2889         if (likely((ctx->aead != EIP197_AEAD_TYPE_IPSEC_ESP ||
2890                     req->assoclen >= EIP197_AEAD_IPSEC_IV_SIZE) &&
2891                    req->cryptlen > POLY1305_DIGEST_SIZE)) {
2892                 return safexcel_queue_req(&req->base, creq, dir);
2893         }
2894
2895         /* HW cannot do full (AAD+payload) zero length, use fallback */
2896         memcpy(key, ctx->key, CHACHA_KEY_SIZE);
2897         if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) {
2898                 /* ESP variant has nonce appended to the key */
2899                 key[CHACHA_KEY_SIZE / sizeof(u32)] = ctx->nonce;
2900                 ret = crypto_aead_setkey(ctx->fback, (u8 *)key,
2901                                          CHACHA_KEY_SIZE +
2902                                          EIP197_AEAD_IPSEC_NONCE_SIZE);
2903         } else {
2904                 ret = crypto_aead_setkey(ctx->fback, (u8 *)key,
2905                                          CHACHA_KEY_SIZE);
2906         }
2907         if (ret) {
2908                 crypto_aead_clear_flags(aead, CRYPTO_TFM_REQ_MASK);
2909                 crypto_aead_set_flags(aead, crypto_aead_get_flags(ctx->fback) &
2910                                             CRYPTO_TFM_REQ_MASK);
2911                 return ret;
2912         }
2913
2914         aead_request_set_tfm(subreq, ctx->fback);
2915         aead_request_set_callback(subreq, req->base.flags, req->base.complete,
2916                                   req->base.data);
2917         aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
2918                                req->iv);
2919         aead_request_set_ad(subreq, req->assoclen);
2920
2921         return (dir ==  SAFEXCEL_ENCRYPT) ?
2922                 crypto_aead_encrypt(subreq) :
2923                 crypto_aead_decrypt(subreq);
2924 }
2925
2926 static int safexcel_aead_chachapoly_encrypt(struct aead_request *req)
2927 {
2928         return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_ENCRYPT);
2929 }
2930
2931 static int safexcel_aead_chachapoly_decrypt(struct aead_request *req)
2932 {
2933         return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_DECRYPT);
2934 }
2935
2936 static int safexcel_aead_fallback_cra_init(struct crypto_tfm *tfm)
2937 {
2938         struct crypto_aead *aead = __crypto_aead_cast(tfm);
2939         struct aead_alg *alg = crypto_aead_alg(aead);
2940         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2941
2942         safexcel_aead_cra_init(tfm);
2943
2944         /* Allocate fallback implementation */
2945         ctx->fback = crypto_alloc_aead(alg->base.cra_name, 0,
2946                                        CRYPTO_ALG_ASYNC |
2947                                        CRYPTO_ALG_NEED_FALLBACK);
2948         if (IS_ERR(ctx->fback))
2949                 return PTR_ERR(ctx->fback);
2950
2951         crypto_aead_set_reqsize(aead, max(sizeof(struct safexcel_cipher_req),
2952                                           sizeof(struct aead_request) +
2953                                           crypto_aead_reqsize(ctx->fback)));
2954
2955         return 0;
2956 }
2957
2958 static int safexcel_aead_chachapoly_cra_init(struct crypto_tfm *tfm)
2959 {
2960         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2961
2962         safexcel_aead_fallback_cra_init(tfm);
2963         ctx->alg  = SAFEXCEL_CHACHA20;
2964         ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32 |
2965                     CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK;
2966         ctx->ctrinit = 0;
2967         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_POLY1305;
2968         ctx->state_sz = 0; /* Precomputed by HW */
2969         return 0;
2970 }
2971
2972 static void safexcel_aead_fallback_cra_exit(struct crypto_tfm *tfm)
2973 {
2974         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2975
2976         crypto_free_aead(ctx->fback);
2977         safexcel_aead_cra_exit(tfm);
2978 }
2979
2980 struct safexcel_alg_template safexcel_alg_chachapoly = {
2981         .type = SAFEXCEL_ALG_TYPE_AEAD,
2982         .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305,
2983         .alg.aead = {
2984                 .setkey = safexcel_aead_chachapoly_setkey,
2985                 .setauthsize = safexcel_aead_chachapoly_setauthsize,
2986                 .encrypt = safexcel_aead_chachapoly_encrypt,
2987                 .decrypt = safexcel_aead_chachapoly_decrypt,
2988                 .ivsize = CHACHAPOLY_IV_SIZE,
2989                 .maxauthsize = POLY1305_DIGEST_SIZE,
2990                 .base = {
2991                         .cra_name = "rfc7539(chacha20,poly1305)",
2992                         .cra_driver_name = "safexcel-chacha20-poly1305",
2993                         /* +1 to put it above HW chacha + SW poly */
2994                         .cra_priority = SAFEXCEL_CRA_PRIORITY + 1,
2995                         .cra_flags = CRYPTO_ALG_ASYNC |
2996                                      CRYPTO_ALG_KERN_DRIVER_ONLY |
2997                                      CRYPTO_ALG_NEED_FALLBACK,
2998                         .cra_blocksize = 1,
2999                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3000                         .cra_alignmask = 0,
3001                         .cra_init = safexcel_aead_chachapoly_cra_init,
3002                         .cra_exit = safexcel_aead_fallback_cra_exit,
3003                         .cra_module = THIS_MODULE,
3004                 },
3005         },
3006 };
3007
3008 static int safexcel_aead_chachapolyesp_cra_init(struct crypto_tfm *tfm)
3009 {
3010         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3011         int ret;
3012
3013         ret = safexcel_aead_chachapoly_cra_init(tfm);
3014         ctx->aead  = EIP197_AEAD_TYPE_IPSEC_ESP;
3015         ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE;
3016         return ret;
3017 }
3018
3019 struct safexcel_alg_template safexcel_alg_chachapoly_esp = {
3020         .type = SAFEXCEL_ALG_TYPE_AEAD,
3021         .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305,
3022         .alg.aead = {
3023                 .setkey = safexcel_aead_chachapoly_setkey,
3024                 .setauthsize = safexcel_aead_chachapoly_setauthsize,
3025                 .encrypt = safexcel_aead_chachapoly_encrypt,
3026                 .decrypt = safexcel_aead_chachapoly_decrypt,
3027                 .ivsize = CHACHAPOLY_IV_SIZE - EIP197_AEAD_IPSEC_NONCE_SIZE,
3028                 .maxauthsize = POLY1305_DIGEST_SIZE,
3029                 .base = {
3030                         .cra_name = "rfc7539esp(chacha20,poly1305)",
3031                         .cra_driver_name = "safexcel-chacha20-poly1305-esp",
3032                         /* +1 to put it above HW chacha + SW poly */
3033                         .cra_priority = SAFEXCEL_CRA_PRIORITY + 1,
3034                         .cra_flags = CRYPTO_ALG_ASYNC |
3035                                      CRYPTO_ALG_KERN_DRIVER_ONLY |
3036                                      CRYPTO_ALG_NEED_FALLBACK,
3037                         .cra_blocksize = 1,
3038                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3039                         .cra_alignmask = 0,
3040                         .cra_init = safexcel_aead_chachapolyesp_cra_init,
3041                         .cra_exit = safexcel_aead_fallback_cra_exit,
3042                         .cra_module = THIS_MODULE,
3043                 },
3044         },
3045 };
3046
3047 static int safexcel_skcipher_sm4_setkey(struct crypto_skcipher *ctfm,
3048                                         const u8 *key, unsigned int len)
3049 {
3050         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
3051         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3052         struct safexcel_crypto_priv *priv = ctx->priv;
3053
3054         if (len != SM4_KEY_SIZE)
3055                 return -EINVAL;
3056
3057         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma)
3058                 if (memcmp(ctx->key, key, SM4_KEY_SIZE))
3059                         ctx->base.needs_inv = true;
3060
3061         memcpy(ctx->key, key, SM4_KEY_SIZE);
3062         ctx->key_len = SM4_KEY_SIZE;
3063
3064         return 0;
3065 }
3066
3067 static int safexcel_sm4_blk_encrypt(struct skcipher_request *req)
3068 {
3069         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3070         if (req->cryptlen & (SM4_BLOCK_SIZE - 1))
3071                 return -EINVAL;
3072         else
3073                 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
3074                                           SAFEXCEL_ENCRYPT);
3075 }
3076
3077 static int safexcel_sm4_blk_decrypt(struct skcipher_request *req)
3078 {
3079         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3080         if (req->cryptlen & (SM4_BLOCK_SIZE - 1))
3081                 return -EINVAL;
3082         else
3083                 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
3084                                           SAFEXCEL_DECRYPT);
3085 }
3086
3087 static int safexcel_skcipher_sm4_ecb_cra_init(struct crypto_tfm *tfm)
3088 {
3089         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3090
3091         safexcel_skcipher_cra_init(tfm);
3092         ctx->alg  = SAFEXCEL_SM4;
3093         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
3094         ctx->blocksz = 0;
3095         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
3096         return 0;
3097 }
3098
3099 struct safexcel_alg_template safexcel_alg_ecb_sm4 = {
3100         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
3101         .algo_mask = SAFEXCEL_ALG_SM4,
3102         .alg.skcipher = {
3103                 .setkey = safexcel_skcipher_sm4_setkey,
3104                 .encrypt = safexcel_sm4_blk_encrypt,
3105                 .decrypt = safexcel_sm4_blk_decrypt,
3106                 .min_keysize = SM4_KEY_SIZE,
3107                 .max_keysize = SM4_KEY_SIZE,
3108                 .base = {
3109                         .cra_name = "ecb(sm4)",
3110                         .cra_driver_name = "safexcel-ecb-sm4",
3111                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3112                         .cra_flags = CRYPTO_ALG_ASYNC |
3113                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3114                         .cra_blocksize = SM4_BLOCK_SIZE,
3115                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3116                         .cra_alignmask = 0,
3117                         .cra_init = safexcel_skcipher_sm4_ecb_cra_init,
3118                         .cra_exit = safexcel_skcipher_cra_exit,
3119                         .cra_module = THIS_MODULE,
3120                 },
3121         },
3122 };
3123
3124 static int safexcel_skcipher_sm4_cbc_cra_init(struct crypto_tfm *tfm)
3125 {
3126         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3127
3128         safexcel_skcipher_cra_init(tfm);
3129         ctx->alg  = SAFEXCEL_SM4;
3130         ctx->blocksz = SM4_BLOCK_SIZE;
3131         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
3132         return 0;
3133 }
3134
3135 struct safexcel_alg_template safexcel_alg_cbc_sm4 = {
3136         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
3137         .algo_mask = SAFEXCEL_ALG_SM4,
3138         .alg.skcipher = {
3139                 .setkey = safexcel_skcipher_sm4_setkey,
3140                 .encrypt = safexcel_sm4_blk_encrypt,
3141                 .decrypt = safexcel_sm4_blk_decrypt,
3142                 .min_keysize = SM4_KEY_SIZE,
3143                 .max_keysize = SM4_KEY_SIZE,
3144                 .ivsize = SM4_BLOCK_SIZE,
3145                 .base = {
3146                         .cra_name = "cbc(sm4)",
3147                         .cra_driver_name = "safexcel-cbc-sm4",
3148                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3149                         .cra_flags = CRYPTO_ALG_ASYNC |
3150                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3151                         .cra_blocksize = SM4_BLOCK_SIZE,
3152                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3153                         .cra_alignmask = 0,
3154                         .cra_init = safexcel_skcipher_sm4_cbc_cra_init,
3155                         .cra_exit = safexcel_skcipher_cra_exit,
3156                         .cra_module = THIS_MODULE,
3157                 },
3158         },
3159 };
3160
3161 static int safexcel_skcipher_sm4_ofb_cra_init(struct crypto_tfm *tfm)
3162 {
3163         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3164
3165         safexcel_skcipher_cra_init(tfm);
3166         ctx->alg  = SAFEXCEL_SM4;
3167         ctx->blocksz = SM4_BLOCK_SIZE;
3168         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB;
3169         return 0;
3170 }
3171
3172 struct safexcel_alg_template safexcel_alg_ofb_sm4 = {
3173         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
3174         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB,
3175         .alg.skcipher = {
3176                 .setkey = safexcel_skcipher_sm4_setkey,
3177                 .encrypt = safexcel_encrypt,
3178                 .decrypt = safexcel_decrypt,
3179                 .min_keysize = SM4_KEY_SIZE,
3180                 .max_keysize = SM4_KEY_SIZE,
3181                 .ivsize = SM4_BLOCK_SIZE,
3182                 .base = {
3183                         .cra_name = "ofb(sm4)",
3184                         .cra_driver_name = "safexcel-ofb-sm4",
3185                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3186                         .cra_flags = CRYPTO_ALG_ASYNC |
3187                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3188                         .cra_blocksize = 1,
3189                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3190                         .cra_alignmask = 0,
3191                         .cra_init = safexcel_skcipher_sm4_ofb_cra_init,
3192                         .cra_exit = safexcel_skcipher_cra_exit,
3193                         .cra_module = THIS_MODULE,
3194                 },
3195         },
3196 };
3197
3198 static int safexcel_skcipher_sm4_cfb_cra_init(struct crypto_tfm *tfm)
3199 {
3200         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3201
3202         safexcel_skcipher_cra_init(tfm);
3203         ctx->alg  = SAFEXCEL_SM4;
3204         ctx->blocksz = SM4_BLOCK_SIZE;
3205         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB;
3206         return 0;
3207 }
3208
3209 struct safexcel_alg_template safexcel_alg_cfb_sm4 = {
3210         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
3211         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB,
3212         .alg.skcipher = {
3213                 .setkey = safexcel_skcipher_sm4_setkey,
3214                 .encrypt = safexcel_encrypt,
3215                 .decrypt = safexcel_decrypt,
3216                 .min_keysize = SM4_KEY_SIZE,
3217                 .max_keysize = SM4_KEY_SIZE,
3218                 .ivsize = SM4_BLOCK_SIZE,
3219                 .base = {
3220                         .cra_name = "cfb(sm4)",
3221                         .cra_driver_name = "safexcel-cfb-sm4",
3222                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3223                         .cra_flags = CRYPTO_ALG_ASYNC |
3224                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3225                         .cra_blocksize = 1,
3226                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3227                         .cra_alignmask = 0,
3228                         .cra_init = safexcel_skcipher_sm4_cfb_cra_init,
3229                         .cra_exit = safexcel_skcipher_cra_exit,
3230                         .cra_module = THIS_MODULE,
3231                 },
3232         },
3233 };
3234
3235 static int safexcel_skcipher_sm4ctr_setkey(struct crypto_skcipher *ctfm,
3236                                            const u8 *key, unsigned int len)
3237 {
3238         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
3239         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3240
3241         /* last 4 bytes of key are the nonce! */
3242         ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE);
3243         /* exclude the nonce here */
3244         len -= CTR_RFC3686_NONCE_SIZE;
3245
3246         return safexcel_skcipher_sm4_setkey(ctfm, key, len);
3247 }
3248
3249 static int safexcel_skcipher_sm4_ctr_cra_init(struct crypto_tfm *tfm)
3250 {
3251         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3252
3253         safexcel_skcipher_cra_init(tfm);
3254         ctx->alg  = SAFEXCEL_SM4;
3255         ctx->blocksz = SM4_BLOCK_SIZE;
3256         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD;
3257         return 0;
3258 }
3259
3260 struct safexcel_alg_template safexcel_alg_ctr_sm4 = {
3261         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
3262         .algo_mask = SAFEXCEL_ALG_SM4,
3263         .alg.skcipher = {
3264                 .setkey = safexcel_skcipher_sm4ctr_setkey,
3265                 .encrypt = safexcel_encrypt,
3266                 .decrypt = safexcel_decrypt,
3267                 /* Add nonce size */
3268                 .min_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
3269                 .max_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
3270                 .ivsize = CTR_RFC3686_IV_SIZE,
3271                 .base = {
3272                         .cra_name = "rfc3686(ctr(sm4))",
3273                         .cra_driver_name = "safexcel-ctr-sm4",
3274                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3275                         .cra_flags = CRYPTO_ALG_ASYNC |
3276                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3277                         .cra_blocksize = 1,
3278                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3279                         .cra_alignmask = 0,
3280                         .cra_init = safexcel_skcipher_sm4_ctr_cra_init,
3281                         .cra_exit = safexcel_skcipher_cra_exit,
3282                         .cra_module = THIS_MODULE,
3283                 },
3284         },
3285 };
3286
3287 static int safexcel_aead_sm4_blk_encrypt(struct aead_request *req)
3288 {
3289         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3290         if (req->cryptlen & (SM4_BLOCK_SIZE - 1))
3291                 return -EINVAL;
3292
3293         return safexcel_queue_req(&req->base, aead_request_ctx(req),
3294                                   SAFEXCEL_ENCRYPT);
3295 }
3296
3297 static int safexcel_aead_sm4_blk_decrypt(struct aead_request *req)
3298 {
3299         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3300
3301         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3302         if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1))
3303                 return -EINVAL;
3304
3305         return safexcel_queue_req(&req->base, aead_request_ctx(req),
3306                                   SAFEXCEL_DECRYPT);
3307 }
3308
3309 static int safexcel_aead_sm4cbc_sha1_cra_init(struct crypto_tfm *tfm)
3310 {
3311         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3312
3313         safexcel_aead_cra_init(tfm);
3314         ctx->alg = SAFEXCEL_SM4;
3315         ctx->blocksz = SM4_BLOCK_SIZE;
3316         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
3317         ctx->state_sz = SHA1_DIGEST_SIZE;
3318         return 0;
3319 }
3320
3321 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4 = {
3322         .type = SAFEXCEL_ALG_TYPE_AEAD,
3323         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1,
3324         .alg.aead = {
3325                 .setkey = safexcel_aead_setkey,
3326                 .encrypt = safexcel_aead_sm4_blk_encrypt,
3327                 .decrypt = safexcel_aead_sm4_blk_decrypt,
3328                 .ivsize = SM4_BLOCK_SIZE,
3329                 .maxauthsize = SHA1_DIGEST_SIZE,
3330                 .base = {
3331                         .cra_name = "authenc(hmac(sha1),cbc(sm4))",
3332                         .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-sm4",
3333                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3334                         .cra_flags = CRYPTO_ALG_ASYNC |
3335                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3336                         .cra_blocksize = SM4_BLOCK_SIZE,
3337                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3338                         .cra_alignmask = 0,
3339                         .cra_init = safexcel_aead_sm4cbc_sha1_cra_init,
3340                         .cra_exit = safexcel_aead_cra_exit,
3341                         .cra_module = THIS_MODULE,
3342                 },
3343         },
3344 };
3345
3346 static int safexcel_aead_fallback_setkey(struct crypto_aead *ctfm,
3347                                          const u8 *key, unsigned int len)
3348 {
3349         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
3350         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3351
3352         /* Keep fallback cipher synchronized */
3353         return crypto_aead_setkey(ctx->fback, (u8 *)key, len) ?:
3354                safexcel_aead_setkey(ctfm, key, len);
3355 }
3356
3357 static int safexcel_aead_fallback_setauthsize(struct crypto_aead *ctfm,
3358                                               unsigned int authsize)
3359 {
3360         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
3361         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3362
3363         /* Keep fallback cipher synchronized */
3364         return crypto_aead_setauthsize(ctx->fback, authsize);
3365 }
3366
3367 static int safexcel_aead_fallback_crypt(struct aead_request *req,
3368                                         enum safexcel_cipher_direction dir)
3369 {
3370         struct crypto_aead *aead = crypto_aead_reqtfm(req);
3371         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
3372         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3373         struct aead_request *subreq = aead_request_ctx(req);
3374
3375         aead_request_set_tfm(subreq, ctx->fback);
3376         aead_request_set_callback(subreq, req->base.flags, req->base.complete,
3377                                   req->base.data);
3378         aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
3379                                req->iv);
3380         aead_request_set_ad(subreq, req->assoclen);
3381
3382         return (dir ==  SAFEXCEL_ENCRYPT) ?
3383                 crypto_aead_encrypt(subreq) :
3384                 crypto_aead_decrypt(subreq);
3385 }
3386
3387 static int safexcel_aead_sm4cbc_sm3_encrypt(struct aead_request *req)
3388 {
3389         struct safexcel_cipher_req *creq = aead_request_ctx(req);
3390
3391         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3392         if (req->cryptlen & (SM4_BLOCK_SIZE - 1))
3393                 return -EINVAL;
3394         else if (req->cryptlen || req->assoclen) /* If input length > 0 only */
3395                 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT);
3396
3397         /* HW cannot do full (AAD+payload) zero length, use fallback */
3398         return safexcel_aead_fallback_crypt(req, SAFEXCEL_ENCRYPT);
3399 }
3400
3401 static int safexcel_aead_sm4cbc_sm3_decrypt(struct aead_request *req)
3402 {
3403         struct safexcel_cipher_req *creq = aead_request_ctx(req);
3404         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3405
3406         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3407         if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1))
3408                 return -EINVAL;
3409         else if (req->cryptlen > crypto_aead_authsize(tfm) || req->assoclen)
3410                 /* If input length > 0 only */
3411                 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT);
3412
3413         /* HW cannot do full (AAD+payload) zero length, use fallback */
3414         return safexcel_aead_fallback_crypt(req, SAFEXCEL_DECRYPT);
3415 }
3416
3417 static int safexcel_aead_sm4cbc_sm3_cra_init(struct crypto_tfm *tfm)
3418 {
3419         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3420
3421         safexcel_aead_fallback_cra_init(tfm);
3422         ctx->alg = SAFEXCEL_SM4;
3423         ctx->blocksz = SM4_BLOCK_SIZE;
3424         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
3425         ctx->state_sz = SM3_DIGEST_SIZE;
3426         return 0;
3427 }
3428
3429 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4 = {
3430         .type = SAFEXCEL_ALG_TYPE_AEAD,
3431         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3,
3432         .alg.aead = {
3433                 .setkey = safexcel_aead_fallback_setkey,
3434                 .setauthsize = safexcel_aead_fallback_setauthsize,
3435                 .encrypt = safexcel_aead_sm4cbc_sm3_encrypt,
3436                 .decrypt = safexcel_aead_sm4cbc_sm3_decrypt,
3437                 .ivsize = SM4_BLOCK_SIZE,
3438                 .maxauthsize = SM3_DIGEST_SIZE,
3439                 .base = {
3440                         .cra_name = "authenc(hmac(sm3),cbc(sm4))",
3441                         .cra_driver_name = "safexcel-authenc-hmac-sm3-cbc-sm4",
3442                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3443                         .cra_flags = CRYPTO_ALG_ASYNC |
3444                                      CRYPTO_ALG_KERN_DRIVER_ONLY |
3445                                      CRYPTO_ALG_NEED_FALLBACK,
3446                         .cra_blocksize = SM4_BLOCK_SIZE,
3447                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3448                         .cra_alignmask = 0,
3449                         .cra_init = safexcel_aead_sm4cbc_sm3_cra_init,
3450                         .cra_exit = safexcel_aead_fallback_cra_exit,
3451                         .cra_module = THIS_MODULE,
3452                 },
3453         },
3454 };
3455
3456 static int safexcel_aead_sm4ctr_sha1_cra_init(struct crypto_tfm *tfm)
3457 {
3458         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3459
3460         safexcel_aead_sm4cbc_sha1_cra_init(tfm);
3461         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD;
3462         return 0;
3463 }
3464
3465 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_sm4 = {
3466         .type = SAFEXCEL_ALG_TYPE_AEAD,
3467         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1,
3468         .alg.aead = {
3469                 .setkey = safexcel_aead_setkey,
3470                 .encrypt = safexcel_aead_encrypt,
3471                 .decrypt = safexcel_aead_decrypt,
3472                 .ivsize = CTR_RFC3686_IV_SIZE,
3473                 .maxauthsize = SHA1_DIGEST_SIZE,
3474                 .base = {
3475                         .cra_name = "authenc(hmac(sha1),rfc3686(ctr(sm4)))",
3476                         .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-sm4",
3477                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3478                         .cra_flags = CRYPTO_ALG_ASYNC |
3479                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3480                         .cra_blocksize = 1,
3481                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3482                         .cra_alignmask = 0,
3483                         .cra_init = safexcel_aead_sm4ctr_sha1_cra_init,
3484                         .cra_exit = safexcel_aead_cra_exit,
3485                         .cra_module = THIS_MODULE,
3486                 },
3487         },
3488 };
3489
3490 static int safexcel_aead_sm4ctr_sm3_cra_init(struct crypto_tfm *tfm)
3491 {
3492         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3493
3494         safexcel_aead_sm4cbc_sm3_cra_init(tfm);
3495         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD;
3496         return 0;
3497 }
3498
3499 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4 = {
3500         .type = SAFEXCEL_ALG_TYPE_AEAD,
3501         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3,
3502         .alg.aead = {
3503                 .setkey = safexcel_aead_setkey,
3504                 .encrypt = safexcel_aead_encrypt,
3505                 .decrypt = safexcel_aead_decrypt,
3506                 .ivsize = CTR_RFC3686_IV_SIZE,
3507                 .maxauthsize = SM3_DIGEST_SIZE,
3508                 .base = {
3509                         .cra_name = "authenc(hmac(sm3),rfc3686(ctr(sm4)))",
3510                         .cra_driver_name = "safexcel-authenc-hmac-sm3-ctr-sm4",
3511                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3512                         .cra_flags = CRYPTO_ALG_ASYNC |
3513                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3514                         .cra_blocksize = 1,
3515                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3516                         .cra_alignmask = 0,
3517                         .cra_init = safexcel_aead_sm4ctr_sm3_cra_init,
3518                         .cra_exit = safexcel_aead_cra_exit,
3519                         .cra_module = THIS_MODULE,
3520                 },
3521         },
3522 };
3523
3524 static int safexcel_rfc4106_gcm_setkey(struct crypto_aead *ctfm, const u8 *key,
3525                                        unsigned int len)
3526 {
3527         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
3528         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3529
3530         /* last 4 bytes of key are the nonce! */
3531         ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE);
3532
3533         len -= CTR_RFC3686_NONCE_SIZE;
3534         return safexcel_aead_gcm_setkey(ctfm, key, len);
3535 }
3536
3537 static int safexcel_rfc4106_gcm_setauthsize(struct crypto_aead *tfm,
3538                                             unsigned int authsize)
3539 {
3540         return crypto_rfc4106_check_authsize(authsize);
3541 }
3542
3543 static int safexcel_rfc4106_encrypt(struct aead_request *req)
3544 {
3545         return crypto_ipsec_check_assoclen(req->assoclen) ?:
3546                safexcel_aead_encrypt(req);
3547 }
3548
3549 static int safexcel_rfc4106_decrypt(struct aead_request *req)
3550 {
3551         return crypto_ipsec_check_assoclen(req->assoclen) ?:
3552                safexcel_aead_decrypt(req);
3553 }
3554
3555 static int safexcel_rfc4106_gcm_cra_init(struct crypto_tfm *tfm)
3556 {
3557         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3558         int ret;
3559
3560         ret = safexcel_aead_gcm_cra_init(tfm);
3561         ctx->aead  = EIP197_AEAD_TYPE_IPSEC_ESP;
3562         ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE;
3563         return ret;
3564 }
3565
3566 struct safexcel_alg_template safexcel_alg_rfc4106_gcm = {
3567         .type = SAFEXCEL_ALG_TYPE_AEAD,
3568         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH,
3569         .alg.aead = {
3570                 .setkey = safexcel_rfc4106_gcm_setkey,
3571                 .setauthsize = safexcel_rfc4106_gcm_setauthsize,
3572                 .encrypt = safexcel_rfc4106_encrypt,
3573                 .decrypt = safexcel_rfc4106_decrypt,
3574                 .ivsize = GCM_RFC4106_IV_SIZE,
3575                 .maxauthsize = GHASH_DIGEST_SIZE,
3576                 .base = {
3577                         .cra_name = "rfc4106(gcm(aes))",
3578                         .cra_driver_name = "safexcel-rfc4106-gcm-aes",
3579                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3580                         .cra_flags = CRYPTO_ALG_ASYNC |
3581                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3582                         .cra_blocksize = 1,
3583                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3584                         .cra_alignmask = 0,
3585                         .cra_init = safexcel_rfc4106_gcm_cra_init,
3586                         .cra_exit = safexcel_aead_gcm_cra_exit,
3587                 },
3588         },
3589 };
3590
3591 static int safexcel_rfc4543_gcm_setauthsize(struct crypto_aead *tfm,
3592                                             unsigned int authsize)
3593 {
3594         if (authsize != GHASH_DIGEST_SIZE)
3595                 return -EINVAL;
3596
3597         return 0;
3598 }
3599
3600 static int safexcel_rfc4543_gcm_cra_init(struct crypto_tfm *tfm)
3601 {
3602         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3603         int ret;
3604
3605         ret = safexcel_aead_gcm_cra_init(tfm);
3606         ctx->aead  = EIP197_AEAD_TYPE_IPSEC_ESP_GMAC;
3607         return ret;
3608 }
3609
3610 struct safexcel_alg_template safexcel_alg_rfc4543_gcm = {
3611         .type = SAFEXCEL_ALG_TYPE_AEAD,
3612         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH,
3613         .alg.aead = {
3614                 .setkey = safexcel_rfc4106_gcm_setkey,
3615                 .setauthsize = safexcel_rfc4543_gcm_setauthsize,
3616                 .encrypt = safexcel_rfc4106_encrypt,
3617                 .decrypt = safexcel_rfc4106_decrypt,
3618                 .ivsize = GCM_RFC4543_IV_SIZE,
3619                 .maxauthsize = GHASH_DIGEST_SIZE,
3620                 .base = {
3621                         .cra_name = "rfc4543(gcm(aes))",
3622                         .cra_driver_name = "safexcel-rfc4543-gcm-aes",
3623                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3624                         .cra_flags = CRYPTO_ALG_ASYNC |
3625                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3626                         .cra_blocksize = 1,
3627                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3628                         .cra_alignmask = 0,
3629                         .cra_init = safexcel_rfc4543_gcm_cra_init,
3630                         .cra_exit = safexcel_aead_gcm_cra_exit,
3631                 },
3632         },
3633 };
3634
3635 static int safexcel_rfc4309_ccm_setkey(struct crypto_aead *ctfm, const u8 *key,
3636                                        unsigned int len)
3637 {
3638         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
3639         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3640
3641         /* First byte of the nonce = L = always 3 for RFC4309 (4 byte ctr) */
3642         *(u8 *)&ctx->nonce = EIP197_AEAD_IPSEC_COUNTER_SIZE - 1;
3643         /* last 3 bytes of key are the nonce! */
3644         memcpy((u8 *)&ctx->nonce + 1, key + len -
3645                EIP197_AEAD_IPSEC_CCM_NONCE_SIZE,
3646                EIP197_AEAD_IPSEC_CCM_NONCE_SIZE);
3647
3648         len -= EIP197_AEAD_IPSEC_CCM_NONCE_SIZE;
3649         return safexcel_aead_ccm_setkey(ctfm, key, len);
3650 }
3651
3652 static int safexcel_rfc4309_ccm_setauthsize(struct crypto_aead *tfm,
3653                                             unsigned int authsize)
3654 {
3655         /* Borrowed from crypto/ccm.c */
3656         switch (authsize) {
3657         case 8:
3658         case 12:
3659         case 16:
3660                 break;
3661         default:
3662                 return -EINVAL;
3663         }
3664
3665         return 0;
3666 }
3667
3668 static int safexcel_rfc4309_ccm_encrypt(struct aead_request *req)
3669 {
3670         struct safexcel_cipher_req *creq = aead_request_ctx(req);
3671
3672         /* Borrowed from crypto/ccm.c */
3673         if (req->assoclen != 16 && req->assoclen != 20)
3674                 return -EINVAL;
3675
3676         return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT);
3677 }
3678
3679 static int safexcel_rfc4309_ccm_decrypt(struct aead_request *req)
3680 {
3681         struct safexcel_cipher_req *creq = aead_request_ctx(req);
3682
3683         /* Borrowed from crypto/ccm.c */
3684         if (req->assoclen != 16 && req->assoclen != 20)
3685                 return -EINVAL;
3686
3687         return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT);
3688 }
3689
3690 static int safexcel_rfc4309_ccm_cra_init(struct crypto_tfm *tfm)
3691 {
3692         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3693         int ret;
3694
3695         ret = safexcel_aead_ccm_cra_init(tfm);
3696         ctx->aead  = EIP197_AEAD_TYPE_IPSEC_ESP;
3697         ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE;
3698         return ret;
3699 }
3700
3701 struct safexcel_alg_template safexcel_alg_rfc4309_ccm = {
3702         .type = SAFEXCEL_ALG_TYPE_AEAD,
3703         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL,
3704         .alg.aead = {
3705                 .setkey = safexcel_rfc4309_ccm_setkey,
3706                 .setauthsize = safexcel_rfc4309_ccm_setauthsize,
3707                 .encrypt = safexcel_rfc4309_ccm_encrypt,
3708                 .decrypt = safexcel_rfc4309_ccm_decrypt,
3709                 .ivsize = EIP197_AEAD_IPSEC_IV_SIZE,
3710                 .maxauthsize = AES_BLOCK_SIZE,
3711                 .base = {
3712                         .cra_name = "rfc4309(ccm(aes))",
3713                         .cra_driver_name = "safexcel-rfc4309-ccm-aes",
3714                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3715                         .cra_flags = CRYPTO_ALG_ASYNC |
3716                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3717                         .cra_blocksize = 1,
3718                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3719                         .cra_alignmask = 0,
3720                         .cra_init = safexcel_rfc4309_ccm_cra_init,
3721                         .cra_exit = safexcel_aead_cra_exit,
3722                         .cra_module = THIS_MODULE,
3723                 },
3724         },
3725 };