drivers/crypto/inside-secure/safexcel_cipher.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2017 Marvell
   4  *
   5  * Antoine Tenart <[email protected]>
   6  */
   7
   8 #include <asm/unaligned.h>
   9 #include <linux/device.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/dmapool.h>
  12 #include <crypto/aead.h>
  13 #include <crypto/aes.h>
  14 #include <crypto/authenc.h>
  15 #include <crypto/chacha.h>
  16 #include <crypto/ctr.h>
  17 #include <crypto/internal/des.h>
  18 #include <crypto/gcm.h>
  19 #include <crypto/ghash.h>
  20 #include <crypto/poly1305.h>
  21 #include <crypto/sha1.h>
  22 #include <crypto/sha2.h>
  23 #include <crypto/sm3.h>
  24 #include <crypto/sm4.h>
  25 #include <crypto/xts.h>
  26 #include <crypto/skcipher.h>
  27 #include <crypto/internal/aead.h>
  28 #include <crypto/internal/skcipher.h>
  29
  30 #include "safexcel.h"
  31
  32 enum safexcel_cipher_direction {
  33         SAFEXCEL_ENCRYPT,
  34         SAFEXCEL_DECRYPT,
  35 };
  36
  37 enum safexcel_cipher_alg {
  38         SAFEXCEL_DES,
  39         SAFEXCEL_3DES,
  40         SAFEXCEL_AES,
  41         SAFEXCEL_CHACHA20,
  42         SAFEXCEL_SM4,
  43 };
  44
  45 struct safexcel_cipher_ctx {
  46         struct safexcel_context base;
  47         struct safexcel_crypto_priv *priv;
  48
  49         u32 mode;
  50         enum safexcel_cipher_alg alg;
  51         u8 aead; /* !=0=AEAD, 2=IPSec ESP AEAD, 3=IPsec ESP GMAC */
  52         u8 xcm;  /* 0=authenc, 1=GCM, 2 reserved for CCM */
  53         u8 aadskip;
  54         u8 blocksz;
  55         u32 ivmask;
  56         u32 ctrinit;
  57
  58         __le32 key[16];
  59         u32 nonce;
  60         unsigned int key_len, xts;
  61
  62         /* All the below is AEAD specific */
  63         u32 hash_alg;
  64         u32 state_sz;
  65
  66         struct crypto_aead *fback;
  67 };
  68
  69 struct safexcel_cipher_req {
  70         enum safexcel_cipher_direction direction;
  71         /* Number of result descriptors associated to the request */
  72         unsigned int rdescs;
  73         bool needs_inv;
  74         int  nr_src, nr_dst;
  75 };
  76
  77 static int safexcel_skcipher_iv(struct safexcel_cipher_ctx *ctx, u8 *iv,
  78                                 struct safexcel_command_desc *cdesc)
  79 {
  80         if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) {
  81                 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
  82                 /* 32 bit nonce */
  83                 cdesc->control_data.token[0] = ctx->nonce;
  84                 /* 64 bit IV part */
  85                 memcpy(&cdesc->control_data.token[1], iv, 8);
  86                 /* 32 bit counter, start at 0 or 1 (big endian!) */
  87                 cdesc->control_data.token[3] =
  88                         (__force u32)cpu_to_be32(ctx->ctrinit);
  89                 return 4;
  90         }
  91         if (ctx->alg == SAFEXCEL_CHACHA20) {
  92                 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
  93                 /* 96 bit nonce part */
  94                 memcpy(&cdesc->control_data.token[0], &iv[4], 12);
  95                 /* 32 bit counter */
  96                 cdesc->control_data.token[3] = *(u32 *)iv;
  97                 return 4;
  98         }
  99
 100         cdesc->control_data.options |= ctx->ivmask;
 101         memcpy(cdesc->control_data.token, iv, ctx->blocksz);
 102         return ctx->blocksz / sizeof(u32);
 103 }
 104
 105 static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
 106                                     struct safexcel_command_desc *cdesc,
 107                                     struct safexcel_token *atoken,
 108                                     u32 length)
 109 {
 110         struct safexcel_token *token;
 111         int ivlen;
 112
 113         ivlen = safexcel_skcipher_iv(ctx, iv, cdesc);
 114         if (ivlen == 4) {
 115                 /* No space in cdesc, instruction moves to atoken */
 116                 cdesc->additional_cdata_size = 1;
 117                 token = atoken;
 118         } else {
 119                 /* Everything fits in cdesc */
 120                 token = (struct safexcel_token *)(cdesc->control_data.token + 2);
 121                 /* Need to pad with NOP */
 122                 eip197_noop_token(&token[1]);
 123         }
 124
 125         token->opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 126         token->packet_length = length;
 127         token->stat = EIP197_TOKEN_STAT_LAST_PACKET |
 128                       EIP197_TOKEN_STAT_LAST_HASH;
 129         token->instructions = EIP197_TOKEN_INS_LAST |
 130                               EIP197_TOKEN_INS_TYPE_CRYPTO |
 131                               EIP197_TOKEN_INS_TYPE_OUTPUT;
 132 }
 133
 134 static void safexcel_aead_iv(struct safexcel_cipher_ctx *ctx, u8 *iv,
 135                              struct safexcel_command_desc *cdesc)
 136 {
 137         if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD ||
 138             ctx->aead & EIP197_AEAD_TYPE_IPSEC_ESP) { /* _ESP and _ESP_GMAC */
 139                 /* 32 bit nonce */
 140                 cdesc->control_data.token[0] = ctx->nonce;
 141                 /* 64 bit IV part */
 142                 memcpy(&cdesc->control_data.token[1], iv, 8);
 143                 /* 32 bit counter, start at 0 or 1 (big endian!) */
 144                 cdesc->control_data.token[3] =
 145                         (__force u32)cpu_to_be32(ctx->ctrinit);
 146                 return;
 147         }
 148         if (ctx->xcm == EIP197_XCM_MODE_GCM || ctx->alg == SAFEXCEL_CHACHA20) {
 149                 /* 96 bit IV part */
 150                 memcpy(&cdesc->control_data.token[0], iv, 12);
 151                 /* 32 bit counter, start at 0 or 1 (big endian!) */
 152                 cdesc->control_data.token[3] =
 153                         (__force u32)cpu_to_be32(ctx->ctrinit);
 154                 return;
 155         }
 156         /* CBC */
 157         memcpy(cdesc->control_data.token, iv, ctx->blocksz);
 158 }
 159
 160 static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
 161                                 struct safexcel_command_desc *cdesc,
 162                                 struct safexcel_token *atoken,
 163                                 enum safexcel_cipher_direction direction,
 164                                 u32 cryptlen, u32 assoclen, u32 digestsize)
 165 {
 166         struct safexcel_token *aadref;
 167         int atoksize = 2; /* Start with minimum size */
 168         int assocadj = assoclen - ctx->aadskip, aadalign;
 169
 170         /* Always 4 dwords of embedded IV  for AEAD modes */
 171         cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
 172
 173         if (direction == SAFEXCEL_DECRYPT)
 174                 cryptlen -= digestsize;
 175
 176         if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM)) {
 177                 /* Construct IV block B0 for the CBC-MAC */
 178                 u8 *final_iv = (u8 *)cdesc->control_data.token;
 179                 u8 *cbcmaciv = (u8 *)&atoken[1];
 180                 __le32 *aadlen = (__le32 *)&atoken[5];
 181
 182                 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) {
 183                         /* Length + nonce */
 184                         cdesc->control_data.token[0] = ctx->nonce;
 185                         /* Fixup flags byte */
 186                         *(__le32 *)cbcmaciv =
 187                                 cpu_to_le32(ctx->nonce |
 188                                             ((assocadj > 0) << 6) |
 189                                             ((digestsize - 2) << 2));
 190                         /* 64 bit IV part */
 191                         memcpy(&cdesc->control_data.token[1], iv, 8);
 192                         memcpy(cbcmaciv + 4, iv, 8);
 193                         /* Start counter at 0 */
 194                         cdesc->control_data.token[3] = 0;
 195                         /* Message length */
 196                         *(__be32 *)(cbcmaciv + 12) = cpu_to_be32(cryptlen);
 197                 } else {
 198                         /* Variable length IV part */
 199                         memcpy(final_iv, iv, 15 - iv[0]);
 200                         memcpy(cbcmaciv, iv, 15 - iv[0]);
 201                         /* Start variable length counter at 0 */
 202                         memset(final_iv + 15 - iv[0], 0, iv[0] + 1);
 203                         memset(cbcmaciv + 15 - iv[0], 0, iv[0] - 1);
 204                         /* fixup flags byte */
 205                         cbcmaciv[0] |= ((assocadj > 0) << 6) |
 206                                        ((digestsize - 2) << 2);
 207                         /* insert lower 2 bytes of message length */
 208                         cbcmaciv[14] = cryptlen >> 8;
 209                         cbcmaciv[15] = cryptlen & 255;
 210                 }
 211
 212                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 213                 atoken->packet_length = AES_BLOCK_SIZE +
 214                                         ((assocadj > 0) << 1);
 215                 atoken->stat = 0;
 216                 atoken->instructions = EIP197_TOKEN_INS_ORIGIN_TOKEN |
 217                                        EIP197_TOKEN_INS_TYPE_HASH;
 218
 219                 if (likely(assocadj)) {
 220                         *aadlen = cpu_to_le32((assocadj >> 8) |
 221                                               (assocadj & 255) << 8);
 222                         atoken += 6;
 223                         atoksize += 7;
 224                 } else {
 225                         atoken += 5;
 226                         atoksize += 6;
 227                 }
 228
 229                 /* Process AAD data */
 230                 aadref = atoken;
 231                 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 232                 atoken->packet_length = assocadj;
 233                 atoken->stat = 0;
 234                 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH;
 235                 atoken++;
 236
 237                 /* For CCM only, align AAD data towards hash engine */
 238                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 239                 aadalign = (assocadj + 2) & 15;
 240                 atoken->packet_length = assocadj && aadalign ?
 241                                                 16 - aadalign :
 242                                                 0;
 243                 if (likely(cryptlen)) {
 244                         atoken->stat = 0;
 245                         atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH;
 246                 } else {
 247                         atoken->stat = EIP197_TOKEN_STAT_LAST_HASH;
 248                         atoken->instructions = EIP197_TOKEN_INS_LAST |
 249                                                EIP197_TOKEN_INS_TYPE_HASH;
 250                 }
 251         } else {
 252                 safexcel_aead_iv(ctx, iv, cdesc);
 253
 254                 /* Process AAD data */
 255                 aadref = atoken;
 256                 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 257                 atoken->packet_length = assocadj;
 258                 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH;
 259                 atoken->instructions = EIP197_TOKEN_INS_LAST |
 260                                        EIP197_TOKEN_INS_TYPE_HASH;
 261         }
 262         atoken++;
 263
 264         if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) {
 265                 /* For ESP mode (and not GMAC), skip over the IV */
 266                 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 267                 atoken->packet_length = EIP197_AEAD_IPSEC_IV_SIZE;
 268                 atoken->stat = 0;
 269                 atoken->instructions = 0;
 270                 atoken++;
 271                 atoksize++;
 272         } else if (unlikely(ctx->alg == SAFEXCEL_CHACHA20 &&
 273                             direction == SAFEXCEL_DECRYPT)) {
 274                 /* Poly-chacha decryption needs a dummy NOP here ... */
 275                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 276                 atoken->packet_length = 16; /* According to Op Manual */
 277                 atoken->stat = 0;
 278                 atoken->instructions = 0;
 279                 atoken++;
 280                 atoksize++;
 281         }
 282
 283         if  (ctx->xcm) {
 284                 /* For GCM and CCM, obtain enc(Y0) */
 285                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES;
 286                 atoken->packet_length = 0;
 287                 atoken->stat = 0;
 288                 atoken->instructions = AES_BLOCK_SIZE;
 289                 atoken++;
 290
 291                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 292                 atoken->packet_length = AES_BLOCK_SIZE;
 293                 atoken->stat = 0;
 294                 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
 295                                        EIP197_TOKEN_INS_TYPE_CRYPTO;
 296                 atoken++;
 297                 atoksize += 2;
 298         }
 299
 300         if (likely(cryptlen || ctx->alg == SAFEXCEL_CHACHA20)) {
 301                 /* Fixup stat field for AAD direction instruction */
 302                 aadref->stat = 0;
 303
 304                 /* Process crypto data */
 305                 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 306                 atoken->packet_length = cryptlen;
 307
 308                 if (unlikely(ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) {
 309                         /* Fixup instruction field for AAD dir instruction */
 310                         aadref->instructions = EIP197_TOKEN_INS_TYPE_HASH;
 311
 312                         /* Do not send to crypt engine in case of GMAC */
 313                         atoken->instructions = EIP197_TOKEN_INS_LAST |
 314                                                EIP197_TOKEN_INS_TYPE_HASH |
 315                                                EIP197_TOKEN_INS_TYPE_OUTPUT;
 316                 } else {
 317                         atoken->instructions = EIP197_TOKEN_INS_LAST |
 318                                                EIP197_TOKEN_INS_TYPE_CRYPTO |
 319                                                EIP197_TOKEN_INS_TYPE_HASH |
 320                                                EIP197_TOKEN_INS_TYPE_OUTPUT;
 321                 }
 322
 323                 cryptlen &= 15;
 324                 if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM && cryptlen)) {
 325                         atoken->stat = 0;
 326                         /* For CCM only, pad crypto data to the hash engine */
 327                         atoken++;
 328                         atoksize++;
 329                         atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 330                         atoken->packet_length = 16 - cryptlen;
 331                         atoken->stat = EIP197_TOKEN_STAT_LAST_HASH;
 332                         atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH;
 333                 } else {
 334                         atoken->stat = EIP197_TOKEN_STAT_LAST_HASH;
 335                 }
 336                 atoken++;
 337                 atoksize++;
 338         }
 339
 340         if (direction == SAFEXCEL_ENCRYPT) {
 341                 /* Append ICV */
 342                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 343                 atoken->packet_length = digestsize;
 344                 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH |
 345                                EIP197_TOKEN_STAT_LAST_PACKET;
 346                 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
 347                                        EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
 348         } else {
 349                 /* Extract ICV */
 350                 atoken->opcode = EIP197_TOKEN_OPCODE_RETRIEVE;
 351                 atoken->packet_length = digestsize;
 352                 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH |
 353                                EIP197_TOKEN_STAT_LAST_PACKET;
 354                 atoken->instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
 355                 atoken++;
 356                 atoksize++;
 357
 358                 /* Verify ICV */
 359                 atoken->opcode = EIP197_TOKEN_OPCODE_VERIFY;
 360                 atoken->packet_length = digestsize |
 361                                         EIP197_TOKEN_HASH_RESULT_VERIFY;
 362                 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH |
 363                                EIP197_TOKEN_STAT_LAST_PACKET;
 364                 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT;
 365         }
 366
 367         /* Fixup length of the token in the command descriptor */
 368         cdesc->additional_cdata_size = atoksize;
 369 }
 370
 371 static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm,
 372                                         const u8 *key, unsigned int len)
 373 {
 374         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
 375         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 376         struct safexcel_crypto_priv *priv = ctx->base.priv;
 377         struct crypto_aes_ctx aes;
 378         int ret, i;
 379
 380         ret = aes_expandkey(&aes, key, len);
 381         if (ret)
 382                 return ret;
 383
 384         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
 385                 for (i = 0; i < len / sizeof(u32); i++) {
 386                         if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) {
 387                                 ctx->base.needs_inv = true;
 388                                 break;
 389                         }
 390                 }
 391         }
 392
 393         for (i = 0; i < len / sizeof(u32); i++)
 394                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
 395
 396         ctx->key_len = len;
 397
 398         memzero_explicit(&aes, sizeof(aes));
 399         return 0;
 400 }
 401
 402 static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key,
 403                                 unsigned int len)
 404 {
 405         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
 406         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 407         struct safexcel_crypto_priv *priv = ctx->base.priv;
 408         struct crypto_authenc_keys keys;
 409         struct crypto_aes_ctx aes;
 410         int err = -EINVAL, i;
 411         const char *alg;
 412
 413         if (unlikely(crypto_authenc_extractkeys(&keys, key, len)))
 414                 goto badkey;
 415
 416         if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) {
 417                 /* Must have at least space for the nonce here */
 418                 if (unlikely(keys.enckeylen < CTR_RFC3686_NONCE_SIZE))
 419                         goto badkey;
 420                 /* last 4 bytes of key are the nonce! */
 421                 ctx->nonce = *(u32 *)(keys.enckey + keys.enckeylen -
 422                                       CTR_RFC3686_NONCE_SIZE);
 423                 /* exclude the nonce here */
 424                 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
 425         }
 426
 427         /* Encryption key */
 428         switch (ctx->alg) {
 429         case SAFEXCEL_DES:
 430                 err = verify_aead_des_key(ctfm, keys.enckey, keys.enckeylen);
 431                 if (unlikely(err))
 432                         goto badkey;
 433                 break;
 434         case SAFEXCEL_3DES:
 435                 err = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen);
 436                 if (unlikely(err))
 437                         goto badkey;
 438                 break;
 439         case SAFEXCEL_AES:
 440                 err = aes_expandkey(&aes, keys.enckey, keys.enckeylen);
 441                 if (unlikely(err))
 442                         goto badkey;
 443                 break;
 444         case SAFEXCEL_SM4:
 445                 if (unlikely(keys.enckeylen != SM4_KEY_SIZE))
 446                         goto badkey;
 447                 break;
 448         default:
 449                 dev_err(priv->dev, "aead: unsupported cipher algorithm\n");
 450                 goto badkey;
 451         }
 452
 453         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
 454                 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) {
 455                         if (le32_to_cpu(ctx->key[i]) !=
 456                             ((u32 *)keys.enckey)[i]) {
 457                                 ctx->base.needs_inv = true;
 458                                 break;
 459                         }
 460                 }
 461         }
 462
 463         /* Auth key */
 464         switch (ctx->hash_alg) {
 465         case CONTEXT_CONTROL_CRYPTO_ALG_SHA1:
 466                 alg = "safexcel-sha1";
 467                 break;
 468         case CONTEXT_CONTROL_CRYPTO_ALG_SHA224:
 469                 alg = "safexcel-sha224";
 470                 break;
 471         case CONTEXT_CONTROL_CRYPTO_ALG_SHA256:
 472                 alg = "safexcel-sha256";
 473                 break;
 474         case CONTEXT_CONTROL_CRYPTO_ALG_SHA384:
 475                 alg = "safexcel-sha384";
 476                 break;
 477         case CONTEXT_CONTROL_CRYPTO_ALG_SHA512:
 478                 alg = "safexcel-sha512";
 479                 break;
 480         case CONTEXT_CONTROL_CRYPTO_ALG_SM3:
 481                 alg = "safexcel-sm3";
 482                 break;
 483         default:
 484                 dev_err(priv->dev, "aead: unsupported hash algorithm\n");
 485                 goto badkey;
 486         }
 487
 488         if (safexcel_hmac_setkey(&ctx->base, keys.authkey, keys.authkeylen,
 489                                  alg, ctx->state_sz))
 490                 goto badkey;
 491
 492         /* Now copy the keys into the context */
 493         for (i = 0; i < keys.enckeylen / sizeof(u32); i++)
 494                 ctx->key[i] = cpu_to_le32(((u32 *)keys.enckey)[i]);
 495         ctx->key_len = keys.enckeylen;
 496
 497         memzero_explicit(&keys, sizeof(keys));
 498         return 0;
 499
 500 badkey:
 501         memzero_explicit(&keys, sizeof(keys));
 502         return err;
 503 }
 504
 505 static int safexcel_context_control(struct safexcel_cipher_ctx *ctx,
 506                                     struct crypto_async_request *async,
 507                                     struct safexcel_cipher_req *sreq,
 508                                     struct safexcel_command_desc *cdesc)
 509 {
 510         struct safexcel_crypto_priv *priv = ctx->base.priv;
 511         int ctrl_size = ctx->key_len / sizeof(u32);
 512
 513         cdesc->control_data.control1 = ctx->mode;
 514
 515         if (ctx->aead) {
 516                 /* Take in account the ipad+opad digests */
 517                 if (ctx->xcm) {
 518                         ctrl_size += ctx->state_sz / sizeof(u32);
 519                         cdesc->control_data.control0 =
 520                                 CONTEXT_CONTROL_KEY_EN |
 521                                 CONTEXT_CONTROL_DIGEST_XCM |
 522                                 ctx->hash_alg |
 523                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 524                 } else if (ctx->alg == SAFEXCEL_CHACHA20) {
 525                         /* Chacha20-Poly1305 */
 526                         cdesc->control_data.control0 =
 527                                 CONTEXT_CONTROL_KEY_EN |
 528                                 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 |
 529                                 (sreq->direction == SAFEXCEL_ENCRYPT ?
 530                                         CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT :
 531                                         CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN) |
 532                                 ctx->hash_alg |
 533                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 534                         return 0;
 535                 } else {
 536                         ctrl_size += ctx->state_sz / sizeof(u32) * 2;
 537                         cdesc->control_data.control0 =
 538                                 CONTEXT_CONTROL_KEY_EN |
 539                                 CONTEXT_CONTROL_DIGEST_HMAC |
 540                                 ctx->hash_alg |
 541                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 542                 }
 543
 544                 if (sreq->direction == SAFEXCEL_ENCRYPT &&
 545                     (ctx->xcm == EIP197_XCM_MODE_CCM ||
 546                      ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC))
 547                         cdesc->control_data.control0 |=
 548                                 CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT;
 549                 else if (sreq->direction == SAFEXCEL_ENCRYPT)
 550                         cdesc->control_data.control0 |=
 551                                 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT;
 552                 else if (ctx->xcm == EIP197_XCM_MODE_CCM)
 553                         cdesc->control_data.control0 |=
 554                                 CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN;
 555                 else
 556                         cdesc->control_data.control0 |=
 557                                 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN;
 558         } else {
 559                 if (sreq->direction == SAFEXCEL_ENCRYPT)
 560                         cdesc->control_data.control0 =
 561                                 CONTEXT_CONTROL_TYPE_CRYPTO_OUT |
 562                                 CONTEXT_CONTROL_KEY_EN |
 563                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 564                 else
 565                         cdesc->control_data.control0 =
 566                                 CONTEXT_CONTROL_TYPE_CRYPTO_IN |
 567                                 CONTEXT_CONTROL_KEY_EN |
 568                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 569         }
 570
 571         if (ctx->alg == SAFEXCEL_DES) {
 572                 cdesc->control_data.control0 |=
 573                         CONTEXT_CONTROL_CRYPTO_ALG_DES;
 574         } else if (ctx->alg == SAFEXCEL_3DES) {
 575                 cdesc->control_data.control0 |=
 576                         CONTEXT_CONTROL_CRYPTO_ALG_3DES;
 577         } else if (ctx->alg == SAFEXCEL_AES) {
 578                 switch (ctx->key_len >> ctx->xts) {
 579                 case AES_KEYSIZE_128:
 580                         cdesc->control_data.control0 |=
 581                                 CONTEXT_CONTROL_CRYPTO_ALG_AES128;
 582                         break;
 583                 case AES_KEYSIZE_192:
 584                         cdesc->control_data.control0 |=
 585                                 CONTEXT_CONTROL_CRYPTO_ALG_AES192;
 586                         break;
 587                 case AES_KEYSIZE_256:
 588                         cdesc->control_data.control0 |=
 589                                 CONTEXT_CONTROL_CRYPTO_ALG_AES256;
 590                         break;
 591                 default:
 592                         dev_err(priv->dev, "aes keysize not supported: %u\n",
 593                                 ctx->key_len >> ctx->xts);
 594                         return -EINVAL;
 595                 }
 596         } else if (ctx->alg == SAFEXCEL_CHACHA20) {
 597                 cdesc->control_data.control0 |=
 598                         CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20;
 599         } else if (ctx->alg == SAFEXCEL_SM4) {
 600                 cdesc->control_data.control0 |=
 601                         CONTEXT_CONTROL_CRYPTO_ALG_SM4;
 602         }
 603
 604         return 0;
 605 }
 606
 607 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring,
 608                                       struct crypto_async_request *async,
 609                                       struct scatterlist *src,
 610                                       struct scatterlist *dst,
 611                                       unsigned int cryptlen,
 612                                       struct safexcel_cipher_req *sreq,
 613                                       bool *should_complete, int *ret)
 614 {
 615         struct skcipher_request *areq = skcipher_request_cast(async);
 616         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq);
 617         struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(skcipher);
 618         struct safexcel_result_desc *rdesc;
 619         int ndesc = 0;
 620
 621         *ret = 0;
 622
 623         if (unlikely(!sreq->rdescs))
 624                 return 0;
 625
 626         while (sreq->rdescs--) {
 627                 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 628                 if (IS_ERR(rdesc)) {
 629                         dev_err(priv->dev,
 630                                 "cipher: result: could not retrieve the result descriptor\n");
 631                         *ret = PTR_ERR(rdesc);
 632                         break;
 633                 }
 634
 635                 if (likely(!*ret))
 636                         *ret = safexcel_rdesc_check_errors(priv, rdesc);
 637
 638                 ndesc++;
 639         }
 640
 641         safexcel_complete(priv, ring);
 642
 643         if (src == dst) {
 644                 if (sreq->nr_src > 0)
 645                         dma_unmap_sg(priv->dev, src, sreq->nr_src,
 646                                      DMA_BIDIRECTIONAL);
 647         } else {
 648                 if (sreq->nr_src > 0)
 649                         dma_unmap_sg(priv->dev, src, sreq->nr_src,
 650                                      DMA_TO_DEVICE);
 651                 if (sreq->nr_dst > 0)
 652                         dma_unmap_sg(priv->dev, dst, sreq->nr_dst,
 653                                      DMA_FROM_DEVICE);
 654         }
 655
 656         /*
 657          * Update IV in req from last crypto output word for CBC modes
 658          */
 659         if ((!ctx->aead) && (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) &&
 660             (sreq->direction == SAFEXCEL_ENCRYPT)) {
 661                 /* For encrypt take the last output word */
 662                 sg_pcopy_to_buffer(dst, sreq->nr_dst, areq->iv,
 663                                    crypto_skcipher_ivsize(skcipher),
 664                                    (cryptlen -
 665                                     crypto_skcipher_ivsize(skcipher)));
 666         }
 667
 668         *should_complete = true;
 669
 670         return ndesc;
 671 }
 672
 673 static int safexcel_send_req(struct crypto_async_request *base, int ring,
 674                              struct safexcel_cipher_req *sreq,
 675                              struct scatterlist *src, struct scatterlist *dst,
 676                              unsigned int cryptlen, unsigned int assoclen,
 677                              unsigned int digestsize, u8 *iv, int *commands,
 678                              int *results)
 679 {
 680         struct skcipher_request *areq = skcipher_request_cast(base);
 681         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq);
 682         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 683         struct safexcel_crypto_priv *priv = ctx->base.priv;
 684         struct safexcel_command_desc *cdesc;
 685         struct safexcel_command_desc *first_cdesc = NULL;
 686         struct safexcel_result_desc *rdesc, *first_rdesc = NULL;
 687         struct scatterlist *sg;
 688         unsigned int totlen;
 689         unsigned int totlen_src = cryptlen + assoclen;
 690         unsigned int totlen_dst = totlen_src;
 691         struct safexcel_token *atoken;
 692         int n_cdesc = 0, n_rdesc = 0;
 693         int queued, i, ret = 0;
 694         bool first = true;
 695
 696         sreq->nr_src = sg_nents_for_len(src, totlen_src);
 697
 698         if (ctx->aead) {
 699                 /*
 700                  * AEAD has auth tag appended to output for encrypt and
 701                  * removed from the output for decrypt!
 702                  */
 703                 if (sreq->direction == SAFEXCEL_DECRYPT)
 704                         totlen_dst -= digestsize;
 705                 else
 706                         totlen_dst += digestsize;
 707
 708                 memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32),
 709                        &ctx->base.ipad, ctx->state_sz);
 710                 if (!ctx->xcm)
 711                         memcpy(ctx->base.ctxr->data + (ctx->key_len +
 712                                ctx->state_sz) / sizeof(u32), &ctx->base.opad,
 713                                ctx->state_sz);
 714         } else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) &&
 715                    (sreq->direction == SAFEXCEL_DECRYPT)) {
 716                 /*
 717                  * Save IV from last crypto input word for CBC modes in decrypt
 718                  * direction. Need to do this first in case of inplace operation
 719                  * as it will be overwritten.
 720                  */
 721                 sg_pcopy_to_buffer(src, sreq->nr_src, areq->iv,
 722                                    crypto_skcipher_ivsize(skcipher),
 723                                    (totlen_src -
 724                                     crypto_skcipher_ivsize(skcipher)));
 725         }
 726
 727         sreq->nr_dst = sg_nents_for_len(dst, totlen_dst);
 728
 729         /*
 730          * Remember actual input length, source buffer length may be
 731          * updated in case of inline operation below.
 732          */
 733         totlen = totlen_src;
 734         queued = totlen_src;
 735
 736         if (src == dst) {
 737                 sreq->nr_src = max(sreq->nr_src, sreq->nr_dst);
 738                 sreq->nr_dst = sreq->nr_src;
 739                 if (unlikely((totlen_src || totlen_dst) &&
 740                     (sreq->nr_src <= 0))) {
 741                         dev_err(priv->dev, "In-place buffer not large enough (need %d bytes)!",
 742                                 max(totlen_src, totlen_dst));
 743                         return -EINVAL;
 744                 }
 745                 if (sreq->nr_src > 0)
 746                         dma_map_sg(priv->dev, src, sreq->nr_src,
 747                                    DMA_BIDIRECTIONAL);
 748         } else {
 749                 if (unlikely(totlen_src && (sreq->nr_src <= 0))) {
 750                         dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!",
 751                                 totlen_src);
 752                         return -EINVAL;
 753                 }
 754
 755                 if (sreq->nr_src > 0)
 756                         dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE);
 757
 758                 if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) {
 759                         dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!",
 760                                 totlen_dst);
 761                         ret = -EINVAL;
 762                         goto unmap;
 763                 }
 764
 765                 if (sreq->nr_dst > 0)
 766                         dma_map_sg(priv->dev, dst, sreq->nr_dst,
 767                                    DMA_FROM_DEVICE);
 768         }
 769
 770         memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len);
 771
 772         if (!totlen) {
 773                 /*
 774                  * The EIP97 cannot deal with zero length input packets!
 775                  * So stuff a dummy command descriptor indicating a 1 byte
 776                  * (dummy) input packet, using the context record as source.
 777                  */
 778                 first_cdesc = safexcel_add_cdesc(priv, ring,
 779                                                  1, 1, ctx->base.ctxr_dma,
 780                                                  1, 1, ctx->base.ctxr_dma,
 781                                                  &atoken);
 782                 if (IS_ERR(first_cdesc)) {
 783                         /* No space left in the command descriptor ring */
 784                         ret = PTR_ERR(first_cdesc);
 785                         goto cdesc_rollback;
 786                 }
 787                 n_cdesc = 1;
 788                 goto skip_cdesc;
 789         }
 790
 791         /* command descriptors */
 792         for_each_sg(src, sg, sreq->nr_src, i) {
 793                 int len = sg_dma_len(sg);
 794
 795                 /* Do not overflow the request */
 796                 if (queued < len)
 797                         len = queued;
 798
 799                 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
 800                                            !(queued - len),
 801                                            sg_dma_address(sg), len, totlen,
 802                                            ctx->base.ctxr_dma, &atoken);
 803                 if (IS_ERR(cdesc)) {
 804                         /* No space left in the command descriptor ring */
 805                         ret = PTR_ERR(cdesc);
 806                         goto cdesc_rollback;
 807                 }
 808
 809                 if (!n_cdesc)
 810                         first_cdesc = cdesc;
 811
 812                 n_cdesc++;
 813                 queued -= len;
 814                 if (!queued)
 815                         break;
 816         }
 817 skip_cdesc:
 818         /* Add context control words and token to first command descriptor */
 819         safexcel_context_control(ctx, base, sreq, first_cdesc);
 820         if (ctx->aead)
 821                 safexcel_aead_token(ctx, iv, first_cdesc, atoken,
 822                                     sreq->direction, cryptlen,
 823                                     assoclen, digestsize);
 824         else
 825                 safexcel_skcipher_token(ctx, iv, first_cdesc, atoken,
 826                                         cryptlen);
 827
 828         /* result descriptors */
 829         for_each_sg(dst, sg, sreq->nr_dst, i) {
 830                 bool last = (i == sreq->nr_dst - 1);
 831                 u32 len = sg_dma_len(sg);
 832
 833                 /* only allow the part of the buffer we know we need */
 834                 if (len > totlen_dst)
 835                         len = totlen_dst;
 836                 if (unlikely(!len))
 837                         break;
 838                 totlen_dst -= len;
 839
 840                 /* skip over AAD space in buffer - not written */
 841                 if (assoclen) {
 842                         if (assoclen >= len) {
 843                                 assoclen -= len;
 844                                 continue;
 845                         }
 846                         rdesc = safexcel_add_rdesc(priv, ring, first, last,
 847                                                    sg_dma_address(sg) +
 848                                                    assoclen,
 849                                                    len - assoclen);
 850                         assoclen = 0;
 851                 } else {
 852                         rdesc = safexcel_add_rdesc(priv, ring, first, last,
 853                                                    sg_dma_address(sg),
 854                                                    len);
 855                 }
 856                 if (IS_ERR(rdesc)) {
 857                         /* No space left in the result descriptor ring */
 858                         ret = PTR_ERR(rdesc);
 859                         goto rdesc_rollback;
 860                 }
 861                 if (first) {
 862                         first_rdesc = rdesc;
 863                         first = false;
 864                 }
 865                 n_rdesc++;
 866         }
 867
 868         if (unlikely(first)) {
 869                 /*
 870                  * Special case: AEAD decrypt with only AAD data.
 871                  * In this case there is NO output data from the engine,
 872                  * but the engine still needs a result descriptor!
 873                  * Create a dummy one just for catching the result token.
 874                  */
 875                 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
 876                 if (IS_ERR(rdesc)) {
 877                         /* No space left in the result descriptor ring */
 878                         ret = PTR_ERR(rdesc);
 879                         goto rdesc_rollback;
 880                 }
 881                 first_rdesc = rdesc;
 882                 n_rdesc = 1;
 883         }
 884
 885         safexcel_rdr_req_set(priv, ring, first_rdesc, base);
 886
 887         *commands = n_cdesc;
 888         *results = n_rdesc;
 889         return 0;
 890
 891 rdesc_rollback:
 892         for (i = 0; i < n_rdesc; i++)
 893                 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr);
 894 cdesc_rollback:
 895         for (i = 0; i < n_cdesc; i++)
 896                 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
 897 unmap:
 898         if (src == dst) {
 899                 if (sreq->nr_src > 0)
 900                         dma_unmap_sg(priv->dev, src, sreq->nr_src,
 901                                      DMA_BIDIRECTIONAL);
 902         } else {
 903                 if (sreq->nr_src > 0)
 904                         dma_unmap_sg(priv->dev, src, sreq->nr_src,
 905                                      DMA_TO_DEVICE);
 906                 if (sreq->nr_dst > 0)
 907                         dma_unmap_sg(priv->dev, dst, sreq->nr_dst,
 908                                      DMA_FROM_DEVICE);
 909         }
 910
 911         return ret;
 912 }
 913
 914 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
 915                                       int ring,
 916                                       struct crypto_async_request *base,
 917                                       struct safexcel_cipher_req *sreq,
 918                                       bool *should_complete, int *ret)
 919 {
 920         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 921         struct safexcel_result_desc *rdesc;
 922         int ndesc = 0, enq_ret;
 923
 924         *ret = 0;
 925
 926         if (unlikely(!sreq->rdescs))
 927                 return 0;
 928
 929         while (sreq->rdescs--) {
 930                 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 931                 if (IS_ERR(rdesc)) {
 932                         dev_err(priv->dev,
 933                                 "cipher: invalidate: could not retrieve the result descriptor\n");
 934                         *ret = PTR_ERR(rdesc);
 935                         break;
 936                 }
 937
 938                 if (likely(!*ret))
 939                         *ret = safexcel_rdesc_check_errors(priv, rdesc);
 940
 941                 ndesc++;
 942         }
 943
 944         safexcel_complete(priv, ring);
 945
 946         if (ctx->base.exit_inv) {
 947                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
 948                               ctx->base.ctxr_dma);
 949
 950                 *should_complete = true;
 951
 952                 return ndesc;
 953         }
 954
 955         ring = safexcel_select_ring(priv);
 956         ctx->base.ring = ring;
 957
 958         spin_lock_bh(&priv->ring[ring].queue_lock);
 959         enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, base);
 960         spin_unlock_bh(&priv->ring[ring].queue_lock);
 961
 962         if (enq_ret != -EINPROGRESS)
 963                 *ret = enq_ret;
 964
 965         queue_work(priv->ring[ring].workqueue,
 966                    &priv->ring[ring].work_data.work);
 967
 968         *should_complete = false;
 969
 970         return ndesc;
 971 }
 972
 973 static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv,
 974                                            int ring,
 975                                            struct crypto_async_request *async,
 976                                            bool *should_complete, int *ret)
 977 {
 978         struct skcipher_request *req = skcipher_request_cast(async);
 979         struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
 980         int err;
 981
 982         if (sreq->needs_inv) {
 983                 sreq->needs_inv = false;
 984                 err = safexcel_handle_inv_result(priv, ring, async, sreq,
 985                                                  should_complete, ret);
 986         } else {
 987                 err = safexcel_handle_req_result(priv, ring, async, req->src,
 988                                                  req->dst, req->cryptlen, sreq,
 989                                                  should_complete, ret);
 990         }
 991
 992         return err;
 993 }
 994
 995 static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv,
 996                                        int ring,
 997                                        struct crypto_async_request *async,
 998                                        bool *should_complete, int *ret)
 999 {
1000         struct aead_request *req = aead_request_cast(async);
1001         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1002         struct safexcel_cipher_req *sreq = aead_request_ctx(req);
1003         int err;
1004
1005         if (sreq->needs_inv) {
1006                 sreq->needs_inv = false;
1007                 err = safexcel_handle_inv_result(priv, ring, async, sreq,
1008                                                  should_complete, ret);
1009         } else {
1010                 err = safexcel_handle_req_result(priv, ring, async, req->src,
1011                                                  req->dst,
1012                                                  req->cryptlen + crypto_aead_authsize(tfm),
1013                                                  sreq, should_complete, ret);
1014         }
1015
1016         return err;
1017 }
1018
1019 static int safexcel_cipher_send_inv(struct crypto_async_request *base,
1020                                     int ring, int *commands, int *results)
1021 {
1022         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
1023         struct safexcel_crypto_priv *priv = ctx->base.priv;
1024         int ret;
1025
1026         ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring);
1027         if (unlikely(ret))
1028                 return ret;
1029
1030         *commands = 1;
1031         *results = 1;
1032
1033         return 0;
1034 }
1035
1036 static int safexcel_skcipher_send(struct crypto_async_request *async, int ring,
1037                                   int *commands, int *results)
1038 {
1039         struct skcipher_request *req = skcipher_request_cast(async);
1040         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
1041         struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
1042         struct safexcel_crypto_priv *priv = ctx->base.priv;
1043         int ret;
1044
1045         BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
1046
1047         if (sreq->needs_inv) {
1048                 ret = safexcel_cipher_send_inv(async, ring, commands, results);
1049         } else {
1050                 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1051                 u8 input_iv[AES_BLOCK_SIZE];
1052
1053                 /*
1054                  * Save input IV in case of CBC decrypt mode
1055                  * Will be overwritten with output IV prior to use!
1056                  */
1057                 memcpy(input_iv, req->iv, crypto_skcipher_ivsize(skcipher));
1058
1059                 ret = safexcel_send_req(async, ring, sreq, req->src,
1060                                         req->dst, req->cryptlen, 0, 0, input_iv,
1061                                         commands, results);
1062         }
1063
1064         sreq->rdescs = *results;
1065         return ret;
1066 }
1067
1068 static int safexcel_aead_send(struct crypto_async_request *async, int ring,
1069                               int *commands, int *results)
1070 {
1071         struct aead_request *req = aead_request_cast(async);
1072         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1073         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
1074         struct safexcel_cipher_req *sreq = aead_request_ctx(req);
1075         struct safexcel_crypto_priv *priv = ctx->base.priv;
1076         int ret;
1077
1078         BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
1079
1080         if (sreq->needs_inv)
1081                 ret = safexcel_cipher_send_inv(async, ring, commands, results);
1082         else
1083                 ret = safexcel_send_req(async, ring, sreq, req->src, req->dst,
1084                                         req->cryptlen, req->assoclen,
1085                                         crypto_aead_authsize(tfm), req->iv,
1086                                         commands, results);
1087         sreq->rdescs = *results;
1088         return ret;
1089 }
1090
1091 static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm,
1092                                     struct crypto_async_request *base,
1093                                     struct safexcel_cipher_req *sreq,
1094                                     struct crypto_wait *result)
1095 {
1096         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1097         struct safexcel_crypto_priv *priv = ctx->base.priv;
1098         int ring = ctx->base.ring;
1099         int err;
1100
1101         ctx = crypto_tfm_ctx(base->tfm);
1102         ctx->base.exit_inv = true;
1103         sreq->needs_inv = true;
1104
1105         spin_lock_bh(&priv->ring[ring].queue_lock);
1106         crypto_enqueue_request(&priv->ring[ring].queue, base);
1107         spin_unlock_bh(&priv->ring[ring].queue_lock);
1108
1109         queue_work(priv->ring[ring].workqueue,
1110                    &priv->ring[ring].work_data.work);
1111
1112         err = crypto_wait_req(-EINPROGRESS, result);
1113
1114         if (err) {
1115                 dev_warn(priv->dev,
1116                         "cipher: sync: invalidate: completion error %d\n",
1117                          err);
1118                 return err;
1119         }
1120
1121         return 0;
1122 }
1123
1124 static int safexcel_skcipher_exit_inv(struct crypto_tfm *tfm)
1125 {
1126         EIP197_REQUEST_ON_STACK(req, skcipher, EIP197_SKCIPHER_REQ_SIZE);
1127         struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
1128         DECLARE_CRYPTO_WAIT(result);
1129
1130         memset(req, 0, sizeof(struct skcipher_request));
1131
1132         skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1133                                       crypto_req_done, &result);
1134         skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm));
1135
1136         return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
1137 }
1138
1139 static int safexcel_aead_exit_inv(struct crypto_tfm *tfm)
1140 {
1141         EIP197_REQUEST_ON_STACK(req, aead, EIP197_AEAD_REQ_SIZE);
1142         struct safexcel_cipher_req *sreq = aead_request_ctx(req);
1143         DECLARE_CRYPTO_WAIT(result);
1144
1145         memset(req, 0, sizeof(struct aead_request));
1146
1147         aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1148                                   crypto_req_done, &result);
1149         aead_request_set_tfm(req, __crypto_aead_cast(tfm));
1150
1151         return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
1152 }
1153
1154 static int safexcel_queue_req(struct crypto_async_request *base,
1155                         struct safexcel_cipher_req *sreq,
1156                         enum safexcel_cipher_direction dir)
1157 {
1158         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
1159         struct safexcel_crypto_priv *priv = ctx->base.priv;
1160         int ret, ring;
1161
1162         sreq->needs_inv = false;
1163         sreq->direction = dir;
1164
1165         if (ctx->base.ctxr) {
1166                 if (priv->flags & EIP197_TRC_CACHE && ctx->base.needs_inv) {
1167                         sreq->needs_inv = true;
1168                         ctx->base.needs_inv = false;
1169                 }
1170         } else {
1171                 ctx->base.ring = safexcel_select_ring(priv);
1172                 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
1173                                                  EIP197_GFP_FLAGS(*base),
1174                                                  &ctx->base.ctxr_dma);
1175                 if (!ctx->base.ctxr)
1176                         return -ENOMEM;
1177         }
1178
1179         ring = ctx->base.ring;
1180
1181         spin_lock_bh(&priv->ring[ring].queue_lock);
1182         ret = crypto_enqueue_request(&priv->ring[ring].queue, base);
1183         spin_unlock_bh(&priv->ring[ring].queue_lock);
1184
1185         queue_work(priv->ring[ring].workqueue,
1186                    &priv->ring[ring].work_data.work);
1187
1188         return ret;
1189 }
1190
1191 static int safexcel_encrypt(struct skcipher_request *req)
1192 {
1193         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1194                         SAFEXCEL_ENCRYPT);
1195 }
1196
1197 static int safexcel_decrypt(struct skcipher_request *req)
1198 {
1199         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1200                         SAFEXCEL_DECRYPT);
1201 }
1202
1203 static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm)
1204 {
1205         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1206         struct safexcel_alg_template *tmpl =
1207                 container_of(tfm->__crt_alg, struct safexcel_alg_template,
1208                              alg.skcipher.base);
1209
1210         crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm),
1211                                     sizeof(struct safexcel_cipher_req));
1212
1213         ctx->base.priv = tmpl->priv;
1214
1215         ctx->base.send = safexcel_skcipher_send;
1216         ctx->base.handle_result = safexcel_skcipher_handle_result;
1217         ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD;
1218         ctx->ctrinit = 1;
1219         return 0;
1220 }
1221
1222 static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm)
1223 {
1224         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1225
1226         memzero_explicit(ctx->key, sizeof(ctx->key));
1227
1228         /* context not allocated, skip invalidation */
1229         if (!ctx->base.ctxr)
1230                 return -ENOMEM;
1231
1232         memzero_explicit(ctx->base.ctxr->data, sizeof(ctx->base.ctxr->data));
1233         return 0;
1234 }
1235
1236 static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm)
1237 {
1238         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1239         struct safexcel_crypto_priv *priv = ctx->base.priv;
1240         int ret;
1241
1242         if (safexcel_cipher_cra_exit(tfm))
1243                 return;
1244
1245         if (priv->flags & EIP197_TRC_CACHE) {
1246                 ret = safexcel_skcipher_exit_inv(tfm);
1247                 if (ret)
1248                         dev_warn(priv->dev, "skcipher: invalidation error %d\n",
1249                                  ret);
1250         } else {
1251                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
1252                               ctx->base.ctxr_dma);
1253         }
1254 }
1255
1256 static void safexcel_aead_cra_exit(struct crypto_tfm *tfm)
1257 {
1258         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1259         struct safexcel_crypto_priv *priv = ctx->base.priv;
1260         int ret;
1261
1262         if (safexcel_cipher_cra_exit(tfm))
1263                 return;
1264
1265         if (priv->flags & EIP197_TRC_CACHE) {
1266                 ret = safexcel_aead_exit_inv(tfm);
1267                 if (ret)
1268                         dev_warn(priv->dev, "aead: invalidation error %d\n",
1269                                  ret);
1270         } else {
1271                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
1272                               ctx->base.ctxr_dma);
1273         }
1274 }
1275
1276 static int safexcel_skcipher_aes_ecb_cra_init(struct crypto_tfm *tfm)
1277 {
1278         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1279
1280         safexcel_skcipher_cra_init(tfm);
1281         ctx->alg  = SAFEXCEL_AES;
1282         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
1283         ctx->blocksz = 0;
1284         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1285         return 0;
1286 }
1287
1288 struct safexcel_alg_template safexcel_alg_ecb_aes = {
1289         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1290         .algo_mask = SAFEXCEL_ALG_AES,
1291         .alg.skcipher = {
1292                 .setkey = safexcel_skcipher_aes_setkey,
1293                 .encrypt = safexcel_encrypt,
1294                 .decrypt = safexcel_decrypt,
1295                 .min_keysize = AES_MIN_KEY_SIZE,
1296                 .max_keysize = AES_MAX_KEY_SIZE,
1297                 .base = {
1298                         .cra_name = "ecb(aes)",
1299                         .cra_driver_name = "safexcel-ecb-aes",
1300                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1301                         .cra_flags = CRYPTO_ALG_ASYNC |
1302                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1303                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1304                         .cra_blocksize = AES_BLOCK_SIZE,
1305                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1306                         .cra_alignmask = 0,
1307                         .cra_init = safexcel_skcipher_aes_ecb_cra_init,
1308                         .cra_exit = safexcel_skcipher_cra_exit,
1309                         .cra_module = THIS_MODULE,
1310                 },
1311         },
1312 };
1313
1314 static int safexcel_skcipher_aes_cbc_cra_init(struct crypto_tfm *tfm)
1315 {
1316         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1317
1318         safexcel_skcipher_cra_init(tfm);
1319         ctx->alg  = SAFEXCEL_AES;
1320         ctx->blocksz = AES_BLOCK_SIZE;
1321         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
1322         return 0;
1323 }
1324
1325 struct safexcel_alg_template safexcel_alg_cbc_aes = {
1326         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1327         .algo_mask = SAFEXCEL_ALG_AES,
1328         .alg.skcipher = {
1329                 .setkey = safexcel_skcipher_aes_setkey,
1330                 .encrypt = safexcel_encrypt,
1331                 .decrypt = safexcel_decrypt,
1332                 .min_keysize = AES_MIN_KEY_SIZE,
1333                 .max_keysize = AES_MAX_KEY_SIZE,
1334                 .ivsize = AES_BLOCK_SIZE,
1335                 .base = {
1336                         .cra_name = "cbc(aes)",
1337                         .cra_driver_name = "safexcel-cbc-aes",
1338                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1339                         .cra_flags = CRYPTO_ALG_ASYNC |
1340                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1341                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1342                         .cra_blocksize = AES_BLOCK_SIZE,
1343                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1344                         .cra_alignmask = 0,
1345                         .cra_init = safexcel_skcipher_aes_cbc_cra_init,
1346                         .cra_exit = safexcel_skcipher_cra_exit,
1347                         .cra_module = THIS_MODULE,
1348                 },
1349         },
1350 };
1351
1352 static int safexcel_skcipher_aes_cfb_cra_init(struct crypto_tfm *tfm)
1353 {
1354         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1355
1356         safexcel_skcipher_cra_init(tfm);
1357         ctx->alg  = SAFEXCEL_AES;
1358         ctx->blocksz = AES_BLOCK_SIZE;
1359         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB;
1360         return 0;
1361 }
1362
1363 struct safexcel_alg_template safexcel_alg_cfb_aes = {
1364         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1365         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB,
1366         .alg.skcipher = {
1367                 .setkey = safexcel_skcipher_aes_setkey,
1368                 .encrypt = safexcel_encrypt,
1369                 .decrypt = safexcel_decrypt,
1370                 .min_keysize = AES_MIN_KEY_SIZE,
1371                 .max_keysize = AES_MAX_KEY_SIZE,
1372                 .ivsize = AES_BLOCK_SIZE,
1373                 .base = {
1374                         .cra_name = "cfb(aes)",
1375                         .cra_driver_name = "safexcel-cfb-aes",
1376                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1377                         .cra_flags = CRYPTO_ALG_ASYNC |
1378                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1379                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1380                         .cra_blocksize = 1,
1381                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1382                         .cra_alignmask = 0,
1383                         .cra_init = safexcel_skcipher_aes_cfb_cra_init,
1384                         .cra_exit = safexcel_skcipher_cra_exit,
1385                         .cra_module = THIS_MODULE,
1386                 },
1387         },
1388 };
1389
1390 static int safexcel_skcipher_aes_ofb_cra_init(struct crypto_tfm *tfm)
1391 {
1392         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1393
1394         safexcel_skcipher_cra_init(tfm);
1395         ctx->alg  = SAFEXCEL_AES;
1396         ctx->blocksz = AES_BLOCK_SIZE;
1397         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB;
1398         return 0;
1399 }
1400
1401 struct safexcel_alg_template safexcel_alg_ofb_aes = {
1402         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1403         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB,
1404         .alg.skcipher = {
1405                 .setkey = safexcel_skcipher_aes_setkey,
1406                 .encrypt = safexcel_encrypt,
1407                 .decrypt = safexcel_decrypt,
1408                 .min_keysize = AES_MIN_KEY_SIZE,
1409                 .max_keysize = AES_MAX_KEY_SIZE,
1410                 .ivsize = AES_BLOCK_SIZE,
1411                 .base = {
1412                         .cra_name = "ofb(aes)",
1413                         .cra_driver_name = "safexcel-ofb-aes",
1414                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1415                         .cra_flags = CRYPTO_ALG_ASYNC |
1416                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1417                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1418                         .cra_blocksize = 1,
1419                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1420                         .cra_alignmask = 0,
1421                         .cra_init = safexcel_skcipher_aes_ofb_cra_init,
1422                         .cra_exit = safexcel_skcipher_cra_exit,
1423                         .cra_module = THIS_MODULE,
1424                 },
1425         },
1426 };
1427
1428 static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm,
1429                                            const u8 *key, unsigned int len)
1430 {
1431         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
1432         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1433         struct safexcel_crypto_priv *priv = ctx->base.priv;
1434         struct crypto_aes_ctx aes;
1435         int ret, i;
1436         unsigned int keylen;
1437
1438         /* last 4 bytes of key are the nonce! */
1439         ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE);
1440         /* exclude the nonce here */
1441         keylen = len - CTR_RFC3686_NONCE_SIZE;
1442         ret = aes_expandkey(&aes, key, keylen);
1443         if (ret)
1444                 return ret;
1445
1446         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
1447                 for (i = 0; i < keylen / sizeof(u32); i++) {
1448                         if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) {
1449                                 ctx->base.needs_inv = true;
1450                                 break;
1451                         }
1452                 }
1453         }
1454
1455         for (i = 0; i < keylen / sizeof(u32); i++)
1456                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
1457
1458         ctx->key_len = keylen;
1459
1460         memzero_explicit(&aes, sizeof(aes));
1461         return 0;
1462 }
1463
1464 static int safexcel_skcipher_aes_ctr_cra_init(struct crypto_tfm *tfm)
1465 {
1466         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1467
1468         safexcel_skcipher_cra_init(tfm);
1469         ctx->alg  = SAFEXCEL_AES;
1470         ctx->blocksz = AES_BLOCK_SIZE;
1471         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD;
1472         return 0;
1473 }
1474
1475 struct safexcel_alg_template safexcel_alg_ctr_aes = {
1476         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1477         .algo_mask = SAFEXCEL_ALG_AES,
1478         .alg.skcipher = {
1479                 .setkey = safexcel_skcipher_aesctr_setkey,
1480                 .encrypt = safexcel_encrypt,
1481                 .decrypt = safexcel_decrypt,
1482                 /* Add nonce size */
1483                 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
1484                 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
1485                 .ivsize = CTR_RFC3686_IV_SIZE,
1486                 .base = {
1487                         .cra_name = "rfc3686(ctr(aes))",
1488                         .cra_driver_name = "safexcel-ctr-aes",
1489                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1490                         .cra_flags = CRYPTO_ALG_ASYNC |
1491                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1492                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1493                         .cra_blocksize = 1,
1494                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1495                         .cra_alignmask = 0,
1496                         .cra_init = safexcel_skcipher_aes_ctr_cra_init,
1497                         .cra_exit = safexcel_skcipher_cra_exit,
1498                         .cra_module = THIS_MODULE,
1499                 },
1500         },
1501 };
1502
1503 static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key,
1504                                unsigned int len)
1505 {
1506         struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm);
1507         struct safexcel_crypto_priv *priv = ctx->base.priv;
1508         int ret;
1509
1510         ret = verify_skcipher_des_key(ctfm, key);
1511         if (ret)
1512                 return ret;
1513
1514         /* if context exits and key changed, need to invalidate it */
1515         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma)
1516                 if (memcmp(ctx->key, key, len))
1517                         ctx->base.needs_inv = true;
1518
1519         memcpy(ctx->key, key, len);
1520         ctx->key_len = len;
1521
1522         return 0;
1523 }
1524
1525 static int safexcel_skcipher_des_cbc_cra_init(struct crypto_tfm *tfm)
1526 {
1527         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1528
1529         safexcel_skcipher_cra_init(tfm);
1530         ctx->alg  = SAFEXCEL_DES;
1531         ctx->blocksz = DES_BLOCK_SIZE;
1532         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1533         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
1534         return 0;
1535 }
1536
1537 struct safexcel_alg_template safexcel_alg_cbc_des = {
1538         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1539         .algo_mask = SAFEXCEL_ALG_DES,
1540         .alg.skcipher = {
1541                 .setkey = safexcel_des_setkey,
1542                 .encrypt = safexcel_encrypt,
1543                 .decrypt = safexcel_decrypt,
1544                 .min_keysize = DES_KEY_SIZE,
1545                 .max_keysize = DES_KEY_SIZE,
1546                 .ivsize = DES_BLOCK_SIZE,
1547                 .base = {
1548                         .cra_name = "cbc(des)",
1549                         .cra_driver_name = "safexcel-cbc-des",
1550                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1551                         .cra_flags = CRYPTO_ALG_ASYNC |
1552                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1553                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1554                         .cra_blocksize = DES_BLOCK_SIZE,
1555                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1556                         .cra_alignmask = 0,
1557                         .cra_init = safexcel_skcipher_des_cbc_cra_init,
1558                         .cra_exit = safexcel_skcipher_cra_exit,
1559                         .cra_module = THIS_MODULE,
1560                 },
1561         },
1562 };
1563
1564 static int safexcel_skcipher_des_ecb_cra_init(struct crypto_tfm *tfm)
1565 {
1566         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1567
1568         safexcel_skcipher_cra_init(tfm);
1569         ctx->alg  = SAFEXCEL_DES;
1570         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
1571         ctx->blocksz = 0;
1572         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1573         return 0;
1574 }
1575
1576 struct safexcel_alg_template safexcel_alg_ecb_des = {
1577         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1578         .algo_mask = SAFEXCEL_ALG_DES,
1579         .alg.skcipher = {
1580                 .setkey = safexcel_des_setkey,
1581                 .encrypt = safexcel_encrypt,
1582                 .decrypt = safexcel_decrypt,
1583                 .min_keysize = DES_KEY_SIZE,
1584                 .max_keysize = DES_KEY_SIZE,
1585                 .base = {
1586                         .cra_name = "ecb(des)",
1587                         .cra_driver_name = "safexcel-ecb-des",
1588                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1589                         .cra_flags = CRYPTO_ALG_ASYNC |
1590                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1591                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1592                         .cra_blocksize = DES_BLOCK_SIZE,
1593                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1594                         .cra_alignmask = 0,
1595                         .cra_init = safexcel_skcipher_des_ecb_cra_init,
1596                         .cra_exit = safexcel_skcipher_cra_exit,
1597                         .cra_module = THIS_MODULE,
1598                 },
1599         },
1600 };
1601
1602 static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm,
1603                                    const u8 *key, unsigned int len)
1604 {
1605         struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm);
1606         struct safexcel_crypto_priv *priv = ctx->base.priv;
1607         int err;
1608
1609         err = verify_skcipher_des3_key(ctfm, key);
1610         if (err)
1611                 return err;
1612
1613         /* if context exits and key changed, need to invalidate it */
1614         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma)
1615                 if (memcmp(ctx->key, key, len))
1616                         ctx->base.needs_inv = true;
1617
1618         memcpy(ctx->key, key, len);
1619         ctx->key_len = len;
1620
1621         return 0;
1622 }
1623
1624 static int safexcel_skcipher_des3_cbc_cra_init(struct crypto_tfm *tfm)
1625 {
1626         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1627
1628         safexcel_skcipher_cra_init(tfm);
1629         ctx->alg  = SAFEXCEL_3DES;
1630         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
1631         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1632         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
1633         return 0;
1634 }
1635
1636 struct safexcel_alg_template safexcel_alg_cbc_des3_ede = {
1637         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1638         .algo_mask = SAFEXCEL_ALG_DES,
1639         .alg.skcipher = {
1640                 .setkey = safexcel_des3_ede_setkey,
1641                 .encrypt = safexcel_encrypt,
1642                 .decrypt = safexcel_decrypt,
1643                 .min_keysize = DES3_EDE_KEY_SIZE,
1644                 .max_keysize = DES3_EDE_KEY_SIZE,
1645                 .ivsize = DES3_EDE_BLOCK_SIZE,
1646                 .base = {
1647                         .cra_name = "cbc(des3_ede)",
1648                         .cra_driver_name = "safexcel-cbc-des3_ede",
1649                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1650                         .cra_flags = CRYPTO_ALG_ASYNC |
1651                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1652                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1653                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1654                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1655                         .cra_alignmask = 0,
1656                         .cra_init = safexcel_skcipher_des3_cbc_cra_init,
1657                         .cra_exit = safexcel_skcipher_cra_exit,
1658                         .cra_module = THIS_MODULE,
1659                 },
1660         },
1661 };
1662
1663 static int safexcel_skcipher_des3_ecb_cra_init(struct crypto_tfm *tfm)
1664 {
1665         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1666
1667         safexcel_skcipher_cra_init(tfm);
1668         ctx->alg  = SAFEXCEL_3DES;
1669         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
1670         ctx->blocksz = 0;
1671         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1672         return 0;
1673 }
1674
1675 struct safexcel_alg_template safexcel_alg_ecb_des3_ede = {
1676         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1677         .algo_mask = SAFEXCEL_ALG_DES,
1678         .alg.skcipher = {
1679                 .setkey = safexcel_des3_ede_setkey,
1680                 .encrypt = safexcel_encrypt,
1681                 .decrypt = safexcel_decrypt,
1682                 .min_keysize = DES3_EDE_KEY_SIZE,
1683                 .max_keysize = DES3_EDE_KEY_SIZE,
1684                 .base = {
1685                         .cra_name = "ecb(des3_ede)",
1686                         .cra_driver_name = "safexcel-ecb-des3_ede",
1687                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1688                         .cra_flags = CRYPTO_ALG_ASYNC |
1689                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1690                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1691                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1692                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1693                         .cra_alignmask = 0,
1694                         .cra_init = safexcel_skcipher_des3_ecb_cra_init,
1695                         .cra_exit = safexcel_skcipher_cra_exit,
1696                         .cra_module = THIS_MODULE,
1697                 },
1698         },
1699 };
1700
1701 static int safexcel_aead_encrypt(struct aead_request *req)
1702 {
1703         struct safexcel_cipher_req *creq = aead_request_ctx(req);
1704
1705         return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT);
1706 }
1707
1708 static int safexcel_aead_decrypt(struct aead_request *req)
1709 {
1710         struct safexcel_cipher_req *creq = aead_request_ctx(req);
1711
1712         return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT);
1713 }
1714
1715 static int safexcel_aead_cra_init(struct crypto_tfm *tfm)
1716 {
1717         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1718         struct safexcel_alg_template *tmpl =
1719                 container_of(tfm->__crt_alg, struct safexcel_alg_template,
1720                              alg.aead.base);
1721
1722         crypto_aead_set_reqsize(__crypto_aead_cast(tfm),
1723                                 sizeof(struct safexcel_cipher_req));
1724
1725         ctx->base.priv = tmpl->priv;
1726
1727         ctx->alg  = SAFEXCEL_AES; /* default */
1728         ctx->blocksz = AES_BLOCK_SIZE;
1729         ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD;
1730         ctx->ctrinit = 1;
1731         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; /* default */
1732         ctx->aead = true;
1733         ctx->base.send = safexcel_aead_send;
1734         ctx->base.handle_result = safexcel_aead_handle_result;
1735         return 0;
1736 }
1737
1738 static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm)
1739 {
1740         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1741
1742         safexcel_aead_cra_init(tfm);
1743         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
1744         ctx->state_sz = SHA1_DIGEST_SIZE;
1745         return 0;
1746 }
1747
1748 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = {
1749         .type = SAFEXCEL_ALG_TYPE_AEAD,
1750         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1,
1751         .alg.aead = {
1752                 .setkey = safexcel_aead_setkey,
1753                 .encrypt = safexcel_aead_encrypt,
1754                 .decrypt = safexcel_aead_decrypt,
1755                 .ivsize = AES_BLOCK_SIZE,
1756                 .maxauthsize = SHA1_DIGEST_SIZE,
1757                 .base = {
1758                         .cra_name = "authenc(hmac(sha1),cbc(aes))",
1759                         .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes",
1760                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1761                         .cra_flags = CRYPTO_ALG_ASYNC |
1762                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1763                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1764                         .cra_blocksize = AES_BLOCK_SIZE,
1765                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1766                         .cra_alignmask = 0,
1767                         .cra_init = safexcel_aead_sha1_cra_init,
1768                         .cra_exit = safexcel_aead_cra_exit,
1769                         .cra_module = THIS_MODULE,
1770                 },
1771         },
1772 };
1773
1774 static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm)
1775 {
1776         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1777
1778         safexcel_aead_cra_init(tfm);
1779         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1780         ctx->state_sz = SHA256_DIGEST_SIZE;
1781         return 0;
1782 }
1783
1784 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = {
1785         .type = SAFEXCEL_ALG_TYPE_AEAD,
1786         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
1787         .alg.aead = {
1788                 .setkey = safexcel_aead_setkey,
1789                 .encrypt = safexcel_aead_encrypt,
1790                 .decrypt = safexcel_aead_decrypt,
1791                 .ivsize = AES_BLOCK_SIZE,
1792                 .maxauthsize = SHA256_DIGEST_SIZE,
1793                 .base = {
1794                         .cra_name = "authenc(hmac(sha256),cbc(aes))",
1795                         .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes",
1796                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1797                         .cra_flags = CRYPTO_ALG_ASYNC |
1798                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1799                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1800                         .cra_blocksize = AES_BLOCK_SIZE,
1801                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1802                         .cra_alignmask = 0,
1803                         .cra_init = safexcel_aead_sha256_cra_init,
1804                         .cra_exit = safexcel_aead_cra_exit,
1805                         .cra_module = THIS_MODULE,
1806                 },
1807         },
1808 };
1809
1810 static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm)
1811 {
1812         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1813
1814         safexcel_aead_cra_init(tfm);
1815         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1816         ctx->state_sz = SHA256_DIGEST_SIZE;
1817         return 0;
1818 }
1819
1820 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = {
1821         .type = SAFEXCEL_ALG_TYPE_AEAD,
1822         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
1823         .alg.aead = {
1824                 .setkey = safexcel_aead_setkey,
1825                 .encrypt = safexcel_aead_encrypt,
1826                 .decrypt = safexcel_aead_decrypt,
1827                 .ivsize = AES_BLOCK_SIZE,
1828                 .maxauthsize = SHA224_DIGEST_SIZE,
1829                 .base = {
1830                         .cra_name = "authenc(hmac(sha224),cbc(aes))",
1831                         .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes",
1832                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1833                         .cra_flags = CRYPTO_ALG_ASYNC |
1834                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1835                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1836                         .cra_blocksize = AES_BLOCK_SIZE,
1837                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1838                         .cra_alignmask = 0,
1839                         .cra_init = safexcel_aead_sha224_cra_init,
1840                         .cra_exit = safexcel_aead_cra_exit,
1841                         .cra_module = THIS_MODULE,
1842                 },
1843         },
1844 };
1845
1846 static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm)
1847 {
1848         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1849
1850         safexcel_aead_cra_init(tfm);
1851         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1852         ctx->state_sz = SHA512_DIGEST_SIZE;
1853         return 0;
1854 }
1855
1856 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = {
1857         .type = SAFEXCEL_ALG_TYPE_AEAD,
1858         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
1859         .alg.aead = {
1860                 .setkey = safexcel_aead_setkey,
1861                 .encrypt = safexcel_aead_encrypt,
1862                 .decrypt = safexcel_aead_decrypt,
1863                 .ivsize = AES_BLOCK_SIZE,
1864                 .maxauthsize = SHA512_DIGEST_SIZE,
1865                 .base = {
1866                         .cra_name = "authenc(hmac(sha512),cbc(aes))",
1867                         .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes",
1868                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1869                         .cra_flags = CRYPTO_ALG_ASYNC |
1870                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1871                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1872                         .cra_blocksize = AES_BLOCK_SIZE,
1873                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1874                         .cra_alignmask = 0,
1875                         .cra_init = safexcel_aead_sha512_cra_init,
1876                         .cra_exit = safexcel_aead_cra_exit,
1877                         .cra_module = THIS_MODULE,
1878                 },
1879         },
1880 };
1881
1882 static int safexcel_aead_sha384_cra_init(struct crypto_tfm *tfm)
1883 {
1884         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1885
1886         safexcel_aead_cra_init(tfm);
1887         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1888         ctx->state_sz = SHA512_DIGEST_SIZE;
1889         return 0;
1890 }
1891
1892 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = {
1893         .type = SAFEXCEL_ALG_TYPE_AEAD,
1894         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
1895         .alg.aead = {
1896                 .setkey = safexcel_aead_setkey,
1897                 .encrypt = safexcel_aead_encrypt,
1898                 .decrypt = safexcel_aead_decrypt,
1899                 .ivsize = AES_BLOCK_SIZE,
1900                 .maxauthsize = SHA384_DIGEST_SIZE,
1901                 .base = {
1902                         .cra_name = "authenc(hmac(sha384),cbc(aes))",
1903                         .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-aes",
1904                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1905                         .cra_flags = CRYPTO_ALG_ASYNC |
1906                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1907                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1908                         .cra_blocksize = AES_BLOCK_SIZE,
1909                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1910                         .cra_alignmask = 0,
1911                         .cra_init = safexcel_aead_sha384_cra_init,
1912                         .cra_exit = safexcel_aead_cra_exit,
1913                         .cra_module = THIS_MODULE,
1914                 },
1915         },
1916 };
1917
1918 static int safexcel_aead_sha1_des3_cra_init(struct crypto_tfm *tfm)
1919 {
1920         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1921
1922         safexcel_aead_sha1_cra_init(tfm);
1923         ctx->alg = SAFEXCEL_3DES; /* override default */
1924         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
1925         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1926         return 0;
1927 }
1928
1929 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede = {
1930         .type = SAFEXCEL_ALG_TYPE_AEAD,
1931         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1,
1932         .alg.aead = {
1933                 .setkey = safexcel_aead_setkey,
1934                 .encrypt = safexcel_aead_encrypt,
1935                 .decrypt = safexcel_aead_decrypt,
1936                 .ivsize = DES3_EDE_BLOCK_SIZE,
1937                 .maxauthsize = SHA1_DIGEST_SIZE,
1938                 .base = {
1939                         .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
1940                         .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des3_ede",
1941                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1942                         .cra_flags = CRYPTO_ALG_ASYNC |
1943                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1944                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1945                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1946                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1947                         .cra_alignmask = 0,
1948                         .cra_init = safexcel_aead_sha1_des3_cra_init,
1949                         .cra_exit = safexcel_aead_cra_exit,
1950                         .cra_module = THIS_MODULE,
1951                 },
1952         },
1953 };
1954
1955 static int safexcel_aead_sha256_des3_cra_init(struct crypto_tfm *tfm)
1956 {
1957         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1958
1959         safexcel_aead_sha256_cra_init(tfm);
1960         ctx->alg = SAFEXCEL_3DES; /* override default */
1961         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
1962         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1963         return 0;
1964 }
1965
1966 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ede = {
1967         .type = SAFEXCEL_ALG_TYPE_AEAD,
1968         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256,
1969         .alg.aead = {
1970                 .setkey = safexcel_aead_setkey,
1971                 .encrypt = safexcel_aead_encrypt,
1972                 .decrypt = safexcel_aead_decrypt,
1973                 .ivsize = DES3_EDE_BLOCK_SIZE,
1974                 .maxauthsize = SHA256_DIGEST_SIZE,
1975                 .base = {
1976                         .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
1977                         .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des3_ede",
1978                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1979                         .cra_flags = CRYPTO_ALG_ASYNC |
1980                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1981                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1982                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1983                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1984                         .cra_alignmask = 0,
1985                         .cra_init = safexcel_aead_sha256_des3_cra_init,
1986                         .cra_exit = safexcel_aead_cra_exit,
1987                         .cra_module = THIS_MODULE,
1988                 },
1989         },
1990 };
1991
1992 static int safexcel_aead_sha224_des3_cra_init(struct crypto_tfm *tfm)
1993 {
1994         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1995
1996         safexcel_aead_sha224_cra_init(tfm);
1997         ctx->alg = SAFEXCEL_3DES; /* override default */
1998         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
1999         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2000         return 0;
2001 }
2002
2003 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede = {
2004         .type = SAFEXCEL_ALG_TYPE_AEAD,
2005         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256,
2006         .alg.aead = {
2007                 .setkey = safexcel_aead_setkey,
2008                 .encrypt = safexcel_aead_encrypt,
2009                 .decrypt = safexcel_aead_decrypt,
2010                 .ivsize = DES3_EDE_BLOCK_SIZE,
2011                 .maxauthsize = SHA224_DIGEST_SIZE,
2012                 .base = {
2013                         .cra_name = "authenc(hmac(sha224),cbc(des3_ede))",
2014                         .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des3_ede",
2015                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2016                         .cra_flags = CRYPTO_ALG_ASYNC |
2017                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2018                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2019                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2020                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2021                         .cra_alignmask = 0,
2022                         .cra_init = safexcel_aead_sha224_des3_cra_init,
2023                         .cra_exit = safexcel_aead_cra_exit,
2024                         .cra_module = THIS_MODULE,
2025                 },
2026         },
2027 };
2028
2029 static int safexcel_aead_sha512_des3_cra_init(struct crypto_tfm *tfm)
2030 {
2031         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2032
2033         safexcel_aead_sha512_cra_init(tfm);
2034         ctx->alg = SAFEXCEL_3DES; /* override default */
2035         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
2036         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2037         return 0;
2038 }
2039
2040 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede = {
2041         .type = SAFEXCEL_ALG_TYPE_AEAD,
2042         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512,
2043         .alg.aead = {
2044                 .setkey = safexcel_aead_setkey,
2045                 .encrypt = safexcel_aead_encrypt,
2046                 .decrypt = safexcel_aead_decrypt,
2047                 .ivsize = DES3_EDE_BLOCK_SIZE,
2048                 .maxauthsize = SHA512_DIGEST_SIZE,
2049                 .base = {
2050                         .cra_name = "authenc(hmac(sha512),cbc(des3_ede))",
2051                         .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des3_ede",
2052                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2053                         .cra_flags = CRYPTO_ALG_ASYNC |
2054                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2055                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2056                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2057                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2058                         .cra_alignmask = 0,
2059                         .cra_init = safexcel_aead_sha512_des3_cra_init,
2060                         .cra_exit = safexcel_aead_cra_exit,
2061                         .cra_module = THIS_MODULE,
2062                 },
2063         },
2064 };
2065
2066 static int safexcel_aead_sha384_des3_cra_init(struct crypto_tfm *tfm)
2067 {
2068         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2069
2070         safexcel_aead_sha384_cra_init(tfm);
2071         ctx->alg = SAFEXCEL_3DES; /* override default */
2072         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
2073         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2074         return 0;
2075 }
2076
2077 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede = {
2078         .type = SAFEXCEL_ALG_TYPE_AEAD,
2079         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512,
2080         .alg.aead = {
2081                 .setkey = safexcel_aead_setkey,
2082                 .encrypt = safexcel_aead_encrypt,
2083                 .decrypt = safexcel_aead_decrypt,
2084                 .ivsize = DES3_EDE_BLOCK_SIZE,
2085                 .maxauthsize = SHA384_DIGEST_SIZE,
2086                 .base = {
2087                         .cra_name = "authenc(hmac(sha384),cbc(des3_ede))",
2088                         .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des3_ede",
2089                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2090                         .cra_flags = CRYPTO_ALG_ASYNC |
2091                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2092                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2093                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2094                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2095                         .cra_alignmask = 0,
2096                         .cra_init = safexcel_aead_sha384_des3_cra_init,
2097                         .cra_exit = safexcel_aead_cra_exit,
2098                         .cra_module = THIS_MODULE,
2099                 },
2100         },
2101 };
2102
2103 static int safexcel_aead_sha1_des_cra_init(struct crypto_tfm *tfm)
2104 {
2105         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2106
2107         safexcel_aead_sha1_cra_init(tfm);
2108         ctx->alg = SAFEXCEL_DES; /* override default */
2109         ctx->blocksz = DES_BLOCK_SIZE;
2110         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2111         return 0;
2112 }
2113
2114 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des = {
2115         .type = SAFEXCEL_ALG_TYPE_AEAD,
2116         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1,
2117         .alg.aead = {
2118                 .setkey = safexcel_aead_setkey,
2119                 .encrypt = safexcel_aead_encrypt,
2120                 .decrypt = safexcel_aead_decrypt,
2121                 .ivsize = DES_BLOCK_SIZE,
2122                 .maxauthsize = SHA1_DIGEST_SIZE,
2123                 .base = {
2124                         .cra_name = "authenc(hmac(sha1),cbc(des))",
2125                         .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des",
2126                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2127                         .cra_flags = CRYPTO_ALG_ASYNC |
2128                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2129                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2130                         .cra_blocksize = DES_BLOCK_SIZE,
2131                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2132                         .cra_alignmask = 0,
2133                         .cra_init = safexcel_aead_sha1_des_cra_init,
2134                         .cra_exit = safexcel_aead_cra_exit,
2135                         .cra_module = THIS_MODULE,
2136                 },
2137         },
2138 };
2139
2140 static int safexcel_aead_sha256_des_cra_init(struct crypto_tfm *tfm)
2141 {
2142         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2143
2144         safexcel_aead_sha256_cra_init(tfm);
2145         ctx->alg = SAFEXCEL_DES; /* override default */
2146         ctx->blocksz = DES_BLOCK_SIZE;
2147         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2148         return 0;
2149 }
2150
2151 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des = {
2152         .type = SAFEXCEL_ALG_TYPE_AEAD,
2153         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256,
2154         .alg.aead = {
2155                 .setkey = safexcel_aead_setkey,
2156                 .encrypt = safexcel_aead_encrypt,
2157                 .decrypt = safexcel_aead_decrypt,
2158                 .ivsize = DES_BLOCK_SIZE,
2159                 .maxauthsize = SHA256_DIGEST_SIZE,
2160                 .base = {
2161                         .cra_name = "authenc(hmac(sha256),cbc(des))",
2162                         .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des",
2163                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2164                         .cra_flags = CRYPTO_ALG_ASYNC |
2165                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2166                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2167                         .cra_blocksize = DES_BLOCK_SIZE,
2168                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2169                         .cra_alignmask = 0,
2170                         .cra_init = safexcel_aead_sha256_des_cra_init,
2171                         .cra_exit = safexcel_aead_cra_exit,
2172                         .cra_module = THIS_MODULE,
2173                 },
2174         },
2175 };
2176
2177 static int safexcel_aead_sha224_des_cra_init(struct crypto_tfm *tfm)
2178 {
2179         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2180
2181         safexcel_aead_sha224_cra_init(tfm);
2182         ctx->alg = SAFEXCEL_DES; /* override default */
2183         ctx->blocksz = DES_BLOCK_SIZE;
2184         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2185         return 0;
2186 }
2187
2188 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des = {
2189         .type = SAFEXCEL_ALG_TYPE_AEAD,
2190         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256,
2191         .alg.aead = {
2192                 .setkey = safexcel_aead_setkey,
2193                 .encrypt = safexcel_aead_encrypt,
2194                 .decrypt = safexcel_aead_decrypt,
2195                 .ivsize = DES_BLOCK_SIZE,
2196                 .maxauthsize = SHA224_DIGEST_SIZE,
2197                 .base = {
2198                         .cra_name = "authenc(hmac(sha224),cbc(des))",
2199                         .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des",
2200                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2201                         .cra_flags = CRYPTO_ALG_ASYNC |
2202                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2203                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2204                         .cra_blocksize = DES_BLOCK_SIZE,
2205                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2206                         .cra_alignmask = 0,
2207                         .cra_init = safexcel_aead_sha224_des_cra_init,
2208                         .cra_exit = safexcel_aead_cra_exit,
2209                         .cra_module = THIS_MODULE,
2210                 },
2211         },
2212 };
2213
2214 static int safexcel_aead_sha512_des_cra_init(struct crypto_tfm *tfm)
2215 {
2216         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2217
2218         safexcel_aead_sha512_cra_init(tfm);
2219         ctx->alg = SAFEXCEL_DES; /* override default */
2220         ctx->blocksz = DES_BLOCK_SIZE;
2221         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2222         return 0;
2223 }
2224
2225 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des = {
2226         .type = SAFEXCEL_ALG_TYPE_AEAD,
2227         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512,
2228         .alg.aead = {
2229                 .setkey = safexcel_aead_setkey,
2230                 .encrypt = safexcel_aead_encrypt,
2231                 .decrypt = safexcel_aead_decrypt,
2232                 .ivsize = DES_BLOCK_SIZE,
2233                 .maxauthsize = SHA512_DIGEST_SIZE,
2234                 .base = {
2235                         .cra_name = "authenc(hmac(sha512),cbc(des))",
2236                         .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des",
2237                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2238                         .cra_flags = CRYPTO_ALG_ASYNC |
2239                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2240                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2241                         .cra_blocksize = DES_BLOCK_SIZE,
2242                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2243                         .cra_alignmask = 0,
2244                         .cra_init = safexcel_aead_sha512_des_cra_init,
2245                         .cra_exit = safexcel_aead_cra_exit,
2246                         .cra_module = THIS_MODULE,
2247                 },
2248         },
2249 };
2250
2251 static int safexcel_aead_sha384_des_cra_init(struct crypto_tfm *tfm)
2252 {
2253         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2254
2255         safexcel_aead_sha384_cra_init(tfm);
2256         ctx->alg = SAFEXCEL_DES; /* override default */
2257         ctx->blocksz = DES_BLOCK_SIZE;
2258         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2259         return 0;
2260 }
2261
2262 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des = {
2263         .type = SAFEXCEL_ALG_TYPE_AEAD,
2264         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512,
2265         .alg.aead = {
2266                 .setkey = safexcel_aead_setkey,
2267                 .encrypt = safexcel_aead_encrypt,
2268                 .decrypt = safexcel_aead_decrypt,
2269                 .ivsize = DES_BLOCK_SIZE,
2270                 .maxauthsize = SHA384_DIGEST_SIZE,
2271                 .base = {
2272                         .cra_name = "authenc(hmac(sha384),cbc(des))",
2273                         .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des",
2274                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2275                         .cra_flags = CRYPTO_ALG_ASYNC |
2276                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2277                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2278                         .cra_blocksize = DES_BLOCK_SIZE,
2279                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2280                         .cra_alignmask = 0,
2281                         .cra_init = safexcel_aead_sha384_des_cra_init,
2282                         .cra_exit = safexcel_aead_cra_exit,
2283                         .cra_module = THIS_MODULE,
2284                 },
2285         },
2286 };
2287
2288 static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm)
2289 {
2290         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2291
2292         safexcel_aead_sha1_cra_init(tfm);
2293         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
2294         return 0;
2295 }
2296
2297 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_aes = {
2298         .type = SAFEXCEL_ALG_TYPE_AEAD,
2299         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1,
2300         .alg.aead = {
2301                 .setkey = safexcel_aead_setkey,
2302                 .encrypt = safexcel_aead_encrypt,
2303                 .decrypt = safexcel_aead_decrypt,
2304                 .ivsize = CTR_RFC3686_IV_SIZE,
2305                 .maxauthsize = SHA1_DIGEST_SIZE,
2306                 .base = {
2307                         .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
2308                         .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-aes",
2309                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2310                         .cra_flags = CRYPTO_ALG_ASYNC |
2311                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2312                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2313                         .cra_blocksize = 1,
2314                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2315                         .cra_alignmask = 0,
2316                         .cra_init = safexcel_aead_sha1_ctr_cra_init,
2317                         .cra_exit = safexcel_aead_cra_exit,
2318                         .cra_module = THIS_MODULE,
2319                 },
2320         },
2321 };
2322
2323 static int safexcel_aead_sha256_ctr_cra_init(struct crypto_tfm *tfm)
2324 {
2325         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2326
2327         safexcel_aead_sha256_cra_init(tfm);
2328         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
2329         return 0;
2330 }
2331
2332 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_ctr_aes = {
2333         .type = SAFEXCEL_ALG_TYPE_AEAD,
2334         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
2335         .alg.aead = {
2336                 .setkey = safexcel_aead_setkey,
2337                 .encrypt = safexcel_aead_encrypt,
2338                 .decrypt = safexcel_aead_decrypt,
2339                 .ivsize = CTR_RFC3686_IV_SIZE,
2340                 .maxauthsize = SHA256_DIGEST_SIZE,
2341                 .base = {
2342                         .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
2343                         .cra_driver_name = "safexcel-authenc-hmac-sha256-ctr-aes",
2344                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2345                         .cra_flags = CRYPTO_ALG_ASYNC |
2346                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2347                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2348                         .cra_blocksize = 1,
2349                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2350                         .cra_alignmask = 0,
2351                         .cra_init = safexcel_aead_sha256_ctr_cra_init,
2352                         .cra_exit = safexcel_aead_cra_exit,
2353                         .cra_module = THIS_MODULE,
2354                 },
2355         },
2356 };
2357
2358 static int safexcel_aead_sha224_ctr_cra_init(struct crypto_tfm *tfm)
2359 {
2360         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2361
2362         safexcel_aead_sha224_cra_init(tfm);
2363         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
2364         return 0;
2365 }
2366
2367 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_ctr_aes = {
2368         .type = SAFEXCEL_ALG_TYPE_AEAD,
2369         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
2370         .alg.aead = {
2371                 .setkey = safexcel_aead_setkey,
2372                 .encrypt = safexcel_aead_encrypt,
2373                 .decrypt = safexcel_aead_decrypt,
2374                 .ivsize = CTR_RFC3686_IV_SIZE,
2375                 .maxauthsize = SHA224_DIGEST_SIZE,
2376                 .base = {
2377                         .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
2378                         .cra_driver_name = "safexcel-authenc-hmac-sha224-ctr-aes",
2379                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2380                         .cra_flags = CRYPTO_ALG_ASYNC |
2381                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2382                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2383                         .cra_blocksize = 1,
2384                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2385                         .cra_alignmask = 0,
2386                         .cra_init = safexcel_aead_sha224_ctr_cra_init,
2387                         .cra_exit = safexcel_aead_cra_exit,
2388                         .cra_module = THIS_MODULE,
2389                 },
2390         },
2391 };
2392
2393 static int safexcel_aead_sha512_ctr_cra_init(struct crypto_tfm *tfm)
2394 {
2395         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2396
2397         safexcel_aead_sha512_cra_init(tfm);
2398         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
2399         return 0;
2400 }
2401
2402 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes = {
2403         .type = SAFEXCEL_ALG_TYPE_AEAD,
2404         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
2405         .alg.aead = {
2406                 .setkey = safexcel_aead_setkey,
2407                 .encrypt = safexcel_aead_encrypt,
2408                 .decrypt = safexcel_aead_decrypt,
2409                 .ivsize = CTR_RFC3686_IV_SIZE,
2410                 .maxauthsize = SHA512_DIGEST_SIZE,
2411                 .base = {
2412                         .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
2413                         .cra_driver_name = "safexcel-authenc-hmac-sha512-ctr-aes",
2414                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2415                         .cra_flags = CRYPTO_ALG_ASYNC |
2416                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2417                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2418                         .cra_blocksize = 1,
2419                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2420                         .cra_alignmask = 0,
2421                         .cra_init = safexcel_aead_sha512_ctr_cra_init,
2422                         .cra_exit = safexcel_aead_cra_exit,
2423                         .cra_module = THIS_MODULE,
2424                 },
2425         },
2426 };
2427
2428 static int safexcel_aead_sha384_ctr_cra_init(struct crypto_tfm *tfm)
2429 {
2430         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2431
2432         safexcel_aead_sha384_cra_init(tfm);
2433         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
2434         return 0;
2435 }
2436
2437 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_ctr_aes = {
2438         .type = SAFEXCEL_ALG_TYPE_AEAD,
2439         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
2440         .alg.aead = {
2441                 .setkey = safexcel_aead_setkey,
2442                 .encrypt = safexcel_aead_encrypt,
2443                 .decrypt = safexcel_aead_decrypt,
2444                 .ivsize = CTR_RFC3686_IV_SIZE,
2445                 .maxauthsize = SHA384_DIGEST_SIZE,
2446                 .base = {
2447                         .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
2448                         .cra_driver_name = "safexcel-authenc-hmac-sha384-ctr-aes",
2449                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2450                         .cra_flags = CRYPTO_ALG_ASYNC |
2451                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2452                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2453                         .cra_blocksize = 1,
2454                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2455                         .cra_alignmask = 0,
2456                         .cra_init = safexcel_aead_sha384_ctr_cra_init,
2457                         .cra_exit = safexcel_aead_cra_exit,
2458                         .cra_module = THIS_MODULE,
2459                 },
2460         },
2461 };
2462
2463 static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm,
2464                                            const u8 *key, unsigned int len)
2465 {
2466         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
2467         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2468         struct safexcel_crypto_priv *priv = ctx->base.priv;
2469         struct crypto_aes_ctx aes;
2470         int ret, i;
2471         unsigned int keylen;
2472
2473         /* Check for illegal XTS keys */
2474         ret = xts_verify_key(ctfm, key, len);
2475         if (ret)
2476                 return ret;
2477
2478         /* Only half of the key data is cipher key */
2479         keylen = (len >> 1);
2480         ret = aes_expandkey(&aes, key, keylen);
2481         if (ret)
2482                 return ret;
2483
2484         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2485                 for (i = 0; i < keylen / sizeof(u32); i++) {
2486                         if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) {
2487                                 ctx->base.needs_inv = true;
2488                                 break;
2489                         }
2490                 }
2491         }
2492
2493         for (i = 0; i < keylen / sizeof(u32); i++)
2494                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
2495
2496         /* The other half is the tweak key */
2497         ret = aes_expandkey(&aes, (u8 *)(key + keylen), keylen);
2498         if (ret)
2499                 return ret;
2500
2501         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2502                 for (i = 0; i < keylen / sizeof(u32); i++) {
2503                         if (le32_to_cpu(ctx->key[i + keylen / sizeof(u32)]) !=
2504                             aes.key_enc[i]) {
2505                                 ctx->base.needs_inv = true;
2506                                 break;
2507                         }
2508                 }
2509         }
2510
2511         for (i = 0; i < keylen / sizeof(u32); i++)
2512                 ctx->key[i + keylen / sizeof(u32)] =
2513                         cpu_to_le32(aes.key_enc[i]);
2514
2515         ctx->key_len = keylen << 1;
2516
2517         memzero_explicit(&aes, sizeof(aes));
2518         return 0;
2519 }
2520
2521 static int safexcel_skcipher_aes_xts_cra_init(struct crypto_tfm *tfm)
2522 {
2523         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2524
2525         safexcel_skcipher_cra_init(tfm);
2526         ctx->alg  = SAFEXCEL_AES;
2527         ctx->blocksz = AES_BLOCK_SIZE;
2528         ctx->xts  = 1;
2529         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XTS;
2530         return 0;
2531 }
2532
2533 static int safexcel_encrypt_xts(struct skcipher_request *req)
2534 {
2535         if (req->cryptlen < XTS_BLOCK_SIZE)
2536                 return -EINVAL;
2537         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
2538                                   SAFEXCEL_ENCRYPT);
2539 }
2540
2541 static int safexcel_decrypt_xts(struct skcipher_request *req)
2542 {
2543         if (req->cryptlen < XTS_BLOCK_SIZE)
2544                 return -EINVAL;
2545         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
2546                                   SAFEXCEL_DECRYPT);
2547 }
2548
2549 struct safexcel_alg_template safexcel_alg_xts_aes = {
2550         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
2551         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XTS,
2552         .alg.skcipher = {
2553                 .setkey = safexcel_skcipher_aesxts_setkey,
2554                 .encrypt = safexcel_encrypt_xts,
2555                 .decrypt = safexcel_decrypt_xts,
2556                 /* XTS actually uses 2 AES keys glued together */
2557                 .min_keysize = AES_MIN_KEY_SIZE * 2,
2558                 .max_keysize = AES_MAX_KEY_SIZE * 2,
2559                 .ivsize = XTS_BLOCK_SIZE,
2560                 .base = {
2561                         .cra_name = "xts(aes)",
2562                         .cra_driver_name = "safexcel-xts-aes",
2563                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2564                         .cra_flags = CRYPTO_ALG_ASYNC |
2565                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2566                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2567                         .cra_blocksize = XTS_BLOCK_SIZE,
2568                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2569                         .cra_alignmask = 0,
2570                         .cra_init = safexcel_skcipher_aes_xts_cra_init,
2571                         .cra_exit = safexcel_skcipher_cra_exit,
2572                         .cra_module = THIS_MODULE,
2573                 },
2574         },
2575 };
2576
2577 static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key,
2578                                     unsigned int len)
2579 {
2580         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
2581         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2582         struct safexcel_crypto_priv *priv = ctx->base.priv;
2583         struct crypto_aes_ctx aes;
2584         u32 hashkey[AES_BLOCK_SIZE >> 2];
2585         int ret, i;
2586
2587         ret = aes_expandkey(&aes, key, len);
2588         if (ret) {
2589                 memzero_explicit(&aes, sizeof(aes));
2590                 return ret;
2591         }
2592
2593         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2594                 for (i = 0; i < len / sizeof(u32); i++) {
2595                         if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) {
2596                                 ctx->base.needs_inv = true;
2597                                 break;
2598                         }
2599                 }
2600         }
2601
2602         for (i = 0; i < len / sizeof(u32); i++)
2603                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
2604
2605         ctx->key_len = len;
2606
2607         /* Compute hash key by encrypting zeroes with cipher key */
2608         memset(hashkey, 0, AES_BLOCK_SIZE);
2609         aes_encrypt(&aes, (u8 *)hashkey, (u8 *)hashkey);
2610
2611         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2612                 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) {
2613                         if (be32_to_cpu(ctx->base.ipad.be[i]) != hashkey[i]) {
2614                                 ctx->base.needs_inv = true;
2615                                 break;
2616                         }
2617                 }
2618         }
2619
2620         for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++)
2621                 ctx->base.ipad.be[i] = cpu_to_be32(hashkey[i]);
2622
2623         memzero_explicit(hashkey, AES_BLOCK_SIZE);
2624         memzero_explicit(&aes, sizeof(aes));
2625         return 0;
2626 }
2627
2628 static int safexcel_aead_gcm_cra_init(struct crypto_tfm *tfm)
2629 {
2630         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2631
2632         safexcel_aead_cra_init(tfm);
2633         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_GHASH;
2634         ctx->state_sz = GHASH_BLOCK_SIZE;
2635         ctx->xcm = EIP197_XCM_MODE_GCM;
2636         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */
2637
2638         return 0;
2639 }
2640
2641 static void safexcel_aead_gcm_cra_exit(struct crypto_tfm *tfm)
2642 {
2643         safexcel_aead_cra_exit(tfm);
2644 }
2645
2646 static int safexcel_aead_gcm_setauthsize(struct crypto_aead *tfm,
2647                                          unsigned int authsize)
2648 {
2649         return crypto_gcm_check_authsize(authsize);
2650 }
2651
2652 struct safexcel_alg_template safexcel_alg_gcm = {
2653         .type = SAFEXCEL_ALG_TYPE_AEAD,
2654         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH,
2655         .alg.aead = {
2656                 .setkey = safexcel_aead_gcm_setkey,
2657                 .setauthsize = safexcel_aead_gcm_setauthsize,
2658                 .encrypt = safexcel_aead_encrypt,
2659                 .decrypt = safexcel_aead_decrypt,
2660                 .ivsize = GCM_AES_IV_SIZE,
2661                 .maxauthsize = GHASH_DIGEST_SIZE,
2662                 .base = {
2663                         .cra_name = "gcm(aes)",
2664                         .cra_driver_name = "safexcel-gcm-aes",
2665                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2666                         .cra_flags = CRYPTO_ALG_ASYNC |
2667                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2668                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2669                         .cra_blocksize = 1,
2670                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2671                         .cra_alignmask = 0,
2672                         .cra_init = safexcel_aead_gcm_cra_init,
2673                         .cra_exit = safexcel_aead_gcm_cra_exit,
2674                         .cra_module = THIS_MODULE,
2675                 },
2676         },
2677 };
2678
2679 static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key,
2680                                     unsigned int len)
2681 {
2682         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
2683         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2684         struct safexcel_crypto_priv *priv = ctx->base.priv;
2685         struct crypto_aes_ctx aes;
2686         int ret, i;
2687
2688         ret = aes_expandkey(&aes, key, len);
2689         if (ret) {
2690                 memzero_explicit(&aes, sizeof(aes));
2691                 return ret;
2692         }
2693
2694         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2695                 for (i = 0; i < len / sizeof(u32); i++) {
2696                         if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) {
2697                                 ctx->base.needs_inv = true;
2698                                 break;
2699                         }
2700                 }
2701         }
2702
2703         for (i = 0; i < len / sizeof(u32); i++) {
2704                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
2705                 ctx->base.ipad.be[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] =
2706                         cpu_to_be32(aes.key_enc[i]);
2707         }
2708
2709         ctx->key_len = len;
2710         ctx->state_sz = 2 * AES_BLOCK_SIZE + len;
2711
2712         if (len == AES_KEYSIZE_192)
2713                 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2714         else if (len == AES_KEYSIZE_256)
2715                 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2716         else
2717                 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2718
2719         memzero_explicit(&aes, sizeof(aes));
2720         return 0;
2721 }
2722
2723 static int safexcel_aead_ccm_cra_init(struct crypto_tfm *tfm)
2724 {
2725         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2726
2727         safexcel_aead_cra_init(tfm);
2728         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2729         ctx->state_sz = 3 * AES_BLOCK_SIZE;
2730         ctx->xcm = EIP197_XCM_MODE_CCM;
2731         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */
2732         ctx->ctrinit = 0;
2733         return 0;
2734 }
2735
2736 static int safexcel_aead_ccm_setauthsize(struct crypto_aead *tfm,
2737                                          unsigned int authsize)
2738 {
2739         /* Borrowed from crypto/ccm.c */
2740         switch (authsize) {
2741         case 4:
2742         case 6:
2743         case 8:
2744         case 10:
2745         case 12:
2746         case 14:
2747         case 16:
2748                 break;
2749         default:
2750                 return -EINVAL;
2751         }
2752
2753         return 0;
2754 }
2755
2756 static int safexcel_ccm_encrypt(struct aead_request *req)
2757 {
2758         struct safexcel_cipher_req *creq = aead_request_ctx(req);
2759
2760         if (req->iv[0] < 1 || req->iv[0] > 7)
2761                 return -EINVAL;
2762
2763         return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT);
2764 }
2765
2766 static int safexcel_ccm_decrypt(struct aead_request *req)
2767 {
2768         struct safexcel_cipher_req *creq = aead_request_ctx(req);
2769
2770         if (req->iv[0] < 1 || req->iv[0] > 7)
2771                 return -EINVAL;
2772
2773         return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT);
2774 }
2775
2776 struct safexcel_alg_template safexcel_alg_ccm = {
2777         .type = SAFEXCEL_ALG_TYPE_AEAD,
2778         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL,
2779         .alg.aead = {
2780                 .setkey = safexcel_aead_ccm_setkey,
2781                 .setauthsize = safexcel_aead_ccm_setauthsize,
2782                 .encrypt = safexcel_ccm_encrypt,
2783                 .decrypt = safexcel_ccm_decrypt,
2784                 .ivsize = AES_BLOCK_SIZE,
2785                 .maxauthsize = AES_BLOCK_SIZE,
2786                 .base = {
2787                         .cra_name = "ccm(aes)",
2788                         .cra_driver_name = "safexcel-ccm-aes",
2789                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2790                         .cra_flags = CRYPTO_ALG_ASYNC |
2791                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2792                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2793                         .cra_blocksize = 1,
2794                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2795                         .cra_alignmask = 0,
2796                         .cra_init = safexcel_aead_ccm_cra_init,
2797                         .cra_exit = safexcel_aead_cra_exit,
2798                         .cra_module = THIS_MODULE,
2799                 },
2800         },
2801 };
2802
2803 static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx,
2804                                      const u8 *key)
2805 {
2806         struct safexcel_crypto_priv *priv = ctx->base.priv;
2807
2808         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma)
2809                 if (memcmp(ctx->key, key, CHACHA_KEY_SIZE))
2810                         ctx->base.needs_inv = true;
2811
2812         memcpy(ctx->key, key, CHACHA_KEY_SIZE);
2813         ctx->key_len = CHACHA_KEY_SIZE;
2814 }
2815
2816 static int safexcel_skcipher_chacha20_setkey(struct crypto_skcipher *ctfm,
2817                                              const u8 *key, unsigned int len)
2818 {
2819         struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm);
2820
2821         if (len != CHACHA_KEY_SIZE)
2822                 return -EINVAL;
2823
2824         safexcel_chacha20_setkey(ctx, key);
2825
2826         return 0;
2827 }
2828
2829 static int safexcel_skcipher_chacha20_cra_init(struct crypto_tfm *tfm)
2830 {
2831         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2832
2833         safexcel_skcipher_cra_init(tfm);
2834         ctx->alg  = SAFEXCEL_CHACHA20;
2835         ctx->ctrinit = 0;
2836         ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32;
2837         return 0;
2838 }
2839
2840 struct safexcel_alg_template safexcel_alg_chacha20 = {
2841         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
2842         .algo_mask = SAFEXCEL_ALG_CHACHA20,
2843         .alg.skcipher = {
2844                 .setkey = safexcel_skcipher_chacha20_setkey,
2845                 .encrypt = safexcel_encrypt,
2846                 .decrypt = safexcel_decrypt,
2847                 .min_keysize = CHACHA_KEY_SIZE,
2848                 .max_keysize = CHACHA_KEY_SIZE,
2849                 .ivsize = CHACHA_IV_SIZE,
2850                 .base = {
2851                         .cra_name = "chacha20",
2852                         .cra_driver_name = "safexcel-chacha20",
2853                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2854                         .cra_flags = CRYPTO_ALG_ASYNC |
2855                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2856                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2857                         .cra_blocksize = 1,
2858                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2859                         .cra_alignmask = 0,
2860                         .cra_init = safexcel_skcipher_chacha20_cra_init,
2861                         .cra_exit = safexcel_skcipher_cra_exit,
2862                         .cra_module = THIS_MODULE,
2863                 },
2864         },
2865 };
2866
2867 static int safexcel_aead_chachapoly_setkey(struct crypto_aead *ctfm,
2868                                     const u8 *key, unsigned int len)
2869 {
2870         struct safexcel_cipher_ctx *ctx = crypto_aead_ctx(ctfm);
2871
2872         if (ctx->aead  == EIP197_AEAD_TYPE_IPSEC_ESP &&
2873             len > EIP197_AEAD_IPSEC_NONCE_SIZE) {
2874                 /* ESP variant has nonce appended to key */
2875                 len -= EIP197_AEAD_IPSEC_NONCE_SIZE;
2876                 ctx->nonce = *(u32 *)(key + len);
2877         }
2878         if (len != CHACHA_KEY_SIZE)
2879                 return -EINVAL;
2880
2881         safexcel_chacha20_setkey(ctx, key);
2882
2883         return 0;
2884 }
2885
2886 static int safexcel_aead_chachapoly_setauthsize(struct crypto_aead *tfm,
2887                                          unsigned int authsize)
2888 {
2889         if (authsize != POLY1305_DIGEST_SIZE)
2890                 return -EINVAL;
2891         return 0;
2892 }
2893
2894 static int safexcel_aead_chachapoly_crypt(struct aead_request *req,
2895                                           enum safexcel_cipher_direction dir)
2896 {
2897         struct safexcel_cipher_req *creq = aead_request_ctx(req);
2898         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2899         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
2900         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2901         struct aead_request *subreq = aead_request_ctx(req);
2902         u32 key[CHACHA_KEY_SIZE / sizeof(u32) + 1];
2903         int ret = 0;
2904
2905         /*
2906          * Instead of wasting time detecting umpteen silly corner cases,
2907          * just dump all "small" requests to the fallback implementation.
2908          * HW would not be faster on such small requests anyway.
2909          */
2910         if (likely((ctx->aead != EIP197_AEAD_TYPE_IPSEC_ESP ||
2911                     req->assoclen >= EIP197_AEAD_IPSEC_IV_SIZE) &&
2912                    req->cryptlen > POLY1305_DIGEST_SIZE)) {
2913                 return safexcel_queue_req(&req->base, creq, dir);
2914         }
2915
2916         /* HW cannot do full (AAD+payload) zero length, use fallback */
2917         memcpy(key, ctx->key, CHACHA_KEY_SIZE);
2918         if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) {
2919                 /* ESP variant has nonce appended to the key */
2920                 key[CHACHA_KEY_SIZE / sizeof(u32)] = ctx->nonce;
2921                 ret = crypto_aead_setkey(ctx->fback, (u8 *)key,
2922                                          CHACHA_KEY_SIZE +
2923                                          EIP197_AEAD_IPSEC_NONCE_SIZE);
2924         } else {
2925                 ret = crypto_aead_setkey(ctx->fback, (u8 *)key,
2926                                          CHACHA_KEY_SIZE);
2927         }
2928         if (ret) {
2929                 crypto_aead_clear_flags(aead, CRYPTO_TFM_REQ_MASK);
2930                 crypto_aead_set_flags(aead, crypto_aead_get_flags(ctx->fback) &
2931                                             CRYPTO_TFM_REQ_MASK);
2932                 return ret;
2933         }
2934
2935         aead_request_set_tfm(subreq, ctx->fback);
2936         aead_request_set_callback(subreq, req->base.flags, req->base.complete,
2937                                   req->base.data);
2938         aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
2939                                req->iv);
2940         aead_request_set_ad(subreq, req->assoclen);
2941
2942         return (dir ==  SAFEXCEL_ENCRYPT) ?
2943                 crypto_aead_encrypt(subreq) :
2944                 crypto_aead_decrypt(subreq);
2945 }
2946
2947 static int safexcel_aead_chachapoly_encrypt(struct aead_request *req)
2948 {
2949         return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_ENCRYPT);
2950 }
2951
2952 static int safexcel_aead_chachapoly_decrypt(struct aead_request *req)
2953 {
2954         return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_DECRYPT);
2955 }
2956
2957 static int safexcel_aead_fallback_cra_init(struct crypto_tfm *tfm)
2958 {
2959         struct crypto_aead *aead = __crypto_aead_cast(tfm);
2960         struct aead_alg *alg = crypto_aead_alg(aead);
2961         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2962
2963         safexcel_aead_cra_init(tfm);
2964
2965         /* Allocate fallback implementation */
2966         ctx->fback = crypto_alloc_aead(alg->base.cra_name, 0,
2967                                        CRYPTO_ALG_ASYNC |
2968                                        CRYPTO_ALG_NEED_FALLBACK);
2969         if (IS_ERR(ctx->fback))
2970                 return PTR_ERR(ctx->fback);
2971
2972         crypto_aead_set_reqsize(aead, max(sizeof(struct safexcel_cipher_req),
2973                                           sizeof(struct aead_request) +
2974                                           crypto_aead_reqsize(ctx->fback)));
2975
2976         return 0;
2977 }
2978
2979 static int safexcel_aead_chachapoly_cra_init(struct crypto_tfm *tfm)
2980 {
2981         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2982
2983         safexcel_aead_fallback_cra_init(tfm);
2984         ctx->alg  = SAFEXCEL_CHACHA20;
2985         ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32 |
2986                     CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK;
2987         ctx->ctrinit = 0;
2988         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_POLY1305;
2989         ctx->state_sz = 0; /* Precomputed by HW */
2990         return 0;
2991 }
2992
2993 static void safexcel_aead_fallback_cra_exit(struct crypto_tfm *tfm)
2994 {
2995         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2996
2997         crypto_free_aead(ctx->fback);
2998         safexcel_aead_cra_exit(tfm);
2999 }
3000
3001 struct safexcel_alg_template safexcel_alg_chachapoly = {
3002         .type = SAFEXCEL_ALG_TYPE_AEAD,
3003         .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305,
3004         .alg.aead = {
3005                 .setkey = safexcel_aead_chachapoly_setkey,
3006                 .setauthsize = safexcel_aead_chachapoly_setauthsize,
3007                 .encrypt = safexcel_aead_chachapoly_encrypt,
3008                 .decrypt = safexcel_aead_chachapoly_decrypt,
3009                 .ivsize = CHACHAPOLY_IV_SIZE,
3010                 .maxauthsize = POLY1305_DIGEST_SIZE,
3011                 .base = {
3012                         .cra_name = "rfc7539(chacha20,poly1305)",
3013                         .cra_driver_name = "safexcel-chacha20-poly1305",
3014                         /* +1 to put it above HW chacha + SW poly */
3015                         .cra_priority = SAFEXCEL_CRA_PRIORITY + 1,
3016                         .cra_flags = CRYPTO_ALG_ASYNC |
3017                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3018                                      CRYPTO_ALG_KERN_DRIVER_ONLY |
3019                                      CRYPTO_ALG_NEED_FALLBACK,
3020                         .cra_blocksize = 1,
3021                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3022                         .cra_alignmask = 0,
3023                         .cra_init = safexcel_aead_chachapoly_cra_init,
3024                         .cra_exit = safexcel_aead_fallback_cra_exit,
3025                         .cra_module = THIS_MODULE,
3026                 },
3027         },
3028 };
3029
3030 static int safexcel_aead_chachapolyesp_cra_init(struct crypto_tfm *tfm)
3031 {
3032         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3033         int ret;
3034
3035         ret = safexcel_aead_chachapoly_cra_init(tfm);
3036         ctx->aead  = EIP197_AEAD_TYPE_IPSEC_ESP;
3037         ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE;
3038         return ret;
3039 }
3040
3041 struct safexcel_alg_template safexcel_alg_chachapoly_esp = {
3042         .type = SAFEXCEL_ALG_TYPE_AEAD,
3043         .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305,
3044         .alg.aead = {
3045                 .setkey = safexcel_aead_chachapoly_setkey,
3046                 .setauthsize = safexcel_aead_chachapoly_setauthsize,
3047                 .encrypt = safexcel_aead_chachapoly_encrypt,
3048                 .decrypt = safexcel_aead_chachapoly_decrypt,
3049                 .ivsize = CHACHAPOLY_IV_SIZE - EIP197_AEAD_IPSEC_NONCE_SIZE,
3050                 .maxauthsize = POLY1305_DIGEST_SIZE,
3051                 .base = {
3052                         .cra_name = "rfc7539esp(chacha20,poly1305)",
3053                         .cra_driver_name = "safexcel-chacha20-poly1305-esp",
3054                         /* +1 to put it above HW chacha + SW poly */
3055                         .cra_priority = SAFEXCEL_CRA_PRIORITY + 1,
3056                         .cra_flags = CRYPTO_ALG_ASYNC |
3057                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3058                                      CRYPTO_ALG_KERN_DRIVER_ONLY |
3059                                      CRYPTO_ALG_NEED_FALLBACK,
3060                         .cra_blocksize = 1,
3061                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3062                         .cra_alignmask = 0,
3063                         .cra_init = safexcel_aead_chachapolyesp_cra_init,
3064                         .cra_exit = safexcel_aead_fallback_cra_exit,
3065                         .cra_module = THIS_MODULE,
3066                 },
3067         },
3068 };
3069
3070 static int safexcel_skcipher_sm4_setkey(struct crypto_skcipher *ctfm,
3071                                         const u8 *key, unsigned int len)
3072 {
3073         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
3074         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3075         struct safexcel_crypto_priv *priv = ctx->base.priv;
3076
3077         if (len != SM4_KEY_SIZE)
3078                 return -EINVAL;
3079
3080         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma)
3081                 if (memcmp(ctx->key, key, SM4_KEY_SIZE))
3082                         ctx->base.needs_inv = true;
3083
3084         memcpy(ctx->key, key, SM4_KEY_SIZE);
3085         ctx->key_len = SM4_KEY_SIZE;
3086
3087         return 0;
3088 }
3089
3090 static int safexcel_sm4_blk_encrypt(struct skcipher_request *req)
3091 {
3092         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3093         if (req->cryptlen & (SM4_BLOCK_SIZE - 1))
3094                 return -EINVAL;
3095         else
3096                 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
3097                                           SAFEXCEL_ENCRYPT);
3098 }
3099
3100 static int safexcel_sm4_blk_decrypt(struct skcipher_request *req)
3101 {
3102         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3103         if (req->cryptlen & (SM4_BLOCK_SIZE - 1))
3104                 return -EINVAL;
3105         else
3106                 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
3107                                           SAFEXCEL_DECRYPT);
3108 }
3109
3110 static int safexcel_skcipher_sm4_ecb_cra_init(struct crypto_tfm *tfm)
3111 {
3112         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3113
3114         safexcel_skcipher_cra_init(tfm);
3115         ctx->alg  = SAFEXCEL_SM4;
3116         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
3117         ctx->blocksz = 0;
3118         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
3119         return 0;
3120 }
3121
3122 struct safexcel_alg_template safexcel_alg_ecb_sm4 = {
3123         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
3124         .algo_mask = SAFEXCEL_ALG_SM4,
3125         .alg.skcipher = {
3126                 .setkey = safexcel_skcipher_sm4_setkey,
3127                 .encrypt = safexcel_sm4_blk_encrypt,
3128                 .decrypt = safexcel_sm4_blk_decrypt,
3129                 .min_keysize = SM4_KEY_SIZE,
3130                 .max_keysize = SM4_KEY_SIZE,
3131                 .base = {
3132                         .cra_name = "ecb(sm4)",
3133                         .cra_driver_name = "safexcel-ecb-sm4",
3134                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3135                         .cra_flags = CRYPTO_ALG_ASYNC |
3136                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3137                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3138                         .cra_blocksize = SM4_BLOCK_SIZE,
3139                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3140                         .cra_alignmask = 0,
3141                         .cra_init = safexcel_skcipher_sm4_ecb_cra_init,
3142                         .cra_exit = safexcel_skcipher_cra_exit,
3143                         .cra_module = THIS_MODULE,
3144                 },
3145         },
3146 };
3147
3148 static int safexcel_skcipher_sm4_cbc_cra_init(struct crypto_tfm *tfm)
3149 {
3150         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3151
3152         safexcel_skcipher_cra_init(tfm);
3153         ctx->alg  = SAFEXCEL_SM4;
3154         ctx->blocksz = SM4_BLOCK_SIZE;
3155         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
3156         return 0;
3157 }
3158
3159 struct safexcel_alg_template safexcel_alg_cbc_sm4 = {
3160         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
3161         .algo_mask = SAFEXCEL_ALG_SM4,
3162         .alg.skcipher = {
3163                 .setkey = safexcel_skcipher_sm4_setkey,
3164                 .encrypt = safexcel_sm4_blk_encrypt,
3165                 .decrypt = safexcel_sm4_blk_decrypt,
3166                 .min_keysize = SM4_KEY_SIZE,
3167                 .max_keysize = SM4_KEY_SIZE,
3168                 .ivsize = SM4_BLOCK_SIZE,
3169                 .base = {
3170                         .cra_name = "cbc(sm4)",
3171                         .cra_driver_name = "safexcel-cbc-sm4",
3172                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3173                         .cra_flags = CRYPTO_ALG_ASYNC |
3174                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3175                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3176                         .cra_blocksize = SM4_BLOCK_SIZE,
3177                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3178                         .cra_alignmask = 0,
3179                         .cra_init = safexcel_skcipher_sm4_cbc_cra_init,
3180                         .cra_exit = safexcel_skcipher_cra_exit,
3181                         .cra_module = THIS_MODULE,
3182                 },
3183         },
3184 };
3185
3186 static int safexcel_skcipher_sm4_ofb_cra_init(struct crypto_tfm *tfm)
3187 {
3188         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3189
3190         safexcel_skcipher_cra_init(tfm);
3191         ctx->alg  = SAFEXCEL_SM4;
3192         ctx->blocksz = SM4_BLOCK_SIZE;
3193         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB;
3194         return 0;
3195 }
3196
3197 struct safexcel_alg_template safexcel_alg_ofb_sm4 = {
3198         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
3199         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB,
3200         .alg.skcipher = {
3201                 .setkey = safexcel_skcipher_sm4_setkey,
3202                 .encrypt = safexcel_encrypt,
3203                 .decrypt = safexcel_decrypt,
3204                 .min_keysize = SM4_KEY_SIZE,
3205                 .max_keysize = SM4_KEY_SIZE,
3206                 .ivsize = SM4_BLOCK_SIZE,
3207                 .base = {
3208                         .cra_name = "ofb(sm4)",
3209                         .cra_driver_name = "safexcel-ofb-sm4",
3210                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3211                         .cra_flags = CRYPTO_ALG_ASYNC |
3212                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3213                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3214                         .cra_blocksize = 1,
3215                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3216                         .cra_alignmask = 0,
3217                         .cra_init = safexcel_skcipher_sm4_ofb_cra_init,
3218                         .cra_exit = safexcel_skcipher_cra_exit,
3219                         .cra_module = THIS_MODULE,
3220                 },
3221         },
3222 };
3223
3224 static int safexcel_skcipher_sm4_cfb_cra_init(struct crypto_tfm *tfm)
3225 {
3226         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3227
3228         safexcel_skcipher_cra_init(tfm);
3229         ctx->alg  = SAFEXCEL_SM4;
3230         ctx->blocksz = SM4_BLOCK_SIZE;
3231         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB;
3232         return 0;
3233 }
3234
3235 struct safexcel_alg_template safexcel_alg_cfb_sm4 = {
3236         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
3237         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB,
3238         .alg.skcipher = {
3239                 .setkey = safexcel_skcipher_sm4_setkey,
3240                 .encrypt = safexcel_encrypt,
3241                 .decrypt = safexcel_decrypt,
3242                 .min_keysize = SM4_KEY_SIZE,
3243                 .max_keysize = SM4_KEY_SIZE,
3244                 .ivsize = SM4_BLOCK_SIZE,
3245                 .base = {
3246                         .cra_name = "cfb(sm4)",
3247                         .cra_driver_name = "safexcel-cfb-sm4",
3248                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3249                         .cra_flags = CRYPTO_ALG_ASYNC |
3250                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3251                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3252                         .cra_blocksize = 1,
3253                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3254                         .cra_alignmask = 0,
3255                         .cra_init = safexcel_skcipher_sm4_cfb_cra_init,
3256                         .cra_exit = safexcel_skcipher_cra_exit,
3257                         .cra_module = THIS_MODULE,
3258                 },
3259         },
3260 };
3261
3262 static int safexcel_skcipher_sm4ctr_setkey(struct crypto_skcipher *ctfm,
3263                                            const u8 *key, unsigned int len)
3264 {
3265         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
3266         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3267
3268         /* last 4 bytes of key are the nonce! */
3269         ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE);
3270         /* exclude the nonce here */
3271         len -= CTR_RFC3686_NONCE_SIZE;
3272
3273         return safexcel_skcipher_sm4_setkey(ctfm, key, len);
3274 }
3275
3276 static int safexcel_skcipher_sm4_ctr_cra_init(struct crypto_tfm *tfm)
3277 {
3278         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3279
3280         safexcel_skcipher_cra_init(tfm);
3281         ctx->alg  = SAFEXCEL_SM4;
3282         ctx->blocksz = SM4_BLOCK_SIZE;
3283         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD;
3284         return 0;
3285 }
3286
3287 struct safexcel_alg_template safexcel_alg_ctr_sm4 = {
3288         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
3289         .algo_mask = SAFEXCEL_ALG_SM4,
3290         .alg.skcipher = {
3291                 .setkey = safexcel_skcipher_sm4ctr_setkey,
3292                 .encrypt = safexcel_encrypt,
3293                 .decrypt = safexcel_decrypt,
3294                 /* Add nonce size */
3295                 .min_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
3296                 .max_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
3297                 .ivsize = CTR_RFC3686_IV_SIZE,
3298                 .base = {
3299                         .cra_name = "rfc3686(ctr(sm4))",
3300                         .cra_driver_name = "safexcel-ctr-sm4",
3301                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3302                         .cra_flags = CRYPTO_ALG_ASYNC |
3303                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3304                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3305                         .cra_blocksize = 1,
3306                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3307                         .cra_alignmask = 0,
3308                         .cra_init = safexcel_skcipher_sm4_ctr_cra_init,
3309                         .cra_exit = safexcel_skcipher_cra_exit,
3310                         .cra_module = THIS_MODULE,
3311                 },
3312         },
3313 };
3314
3315 static int safexcel_aead_sm4_blk_encrypt(struct aead_request *req)
3316 {
3317         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3318         if (req->cryptlen & (SM4_BLOCK_SIZE - 1))
3319                 return -EINVAL;
3320
3321         return safexcel_queue_req(&req->base, aead_request_ctx(req),
3322                                   SAFEXCEL_ENCRYPT);
3323 }
3324
3325 static int safexcel_aead_sm4_blk_decrypt(struct aead_request *req)
3326 {
3327         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3328
3329         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3330         if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1))
3331                 return -EINVAL;
3332
3333         return safexcel_queue_req(&req->base, aead_request_ctx(req),
3334                                   SAFEXCEL_DECRYPT);
3335 }
3336
3337 static int safexcel_aead_sm4cbc_sha1_cra_init(struct crypto_tfm *tfm)
3338 {
3339         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3340
3341         safexcel_aead_cra_init(tfm);
3342         ctx->alg = SAFEXCEL_SM4;
3343         ctx->blocksz = SM4_BLOCK_SIZE;
3344         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
3345         ctx->state_sz = SHA1_DIGEST_SIZE;
3346         return 0;
3347 }
3348
3349 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4 = {
3350         .type = SAFEXCEL_ALG_TYPE_AEAD,
3351         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1,
3352         .alg.aead = {
3353                 .setkey = safexcel_aead_setkey,
3354                 .encrypt = safexcel_aead_sm4_blk_encrypt,
3355                 .decrypt = safexcel_aead_sm4_blk_decrypt,
3356                 .ivsize = SM4_BLOCK_SIZE,
3357                 .maxauthsize = SHA1_DIGEST_SIZE,
3358                 .base = {
3359                         .cra_name = "authenc(hmac(sha1),cbc(sm4))",
3360                         .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-sm4",
3361                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3362                         .cra_flags = CRYPTO_ALG_ASYNC |
3363                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3364                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3365                         .cra_blocksize = SM4_BLOCK_SIZE,
3366                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3367                         .cra_alignmask = 0,
3368                         .cra_init = safexcel_aead_sm4cbc_sha1_cra_init,
3369                         .cra_exit = safexcel_aead_cra_exit,
3370                         .cra_module = THIS_MODULE,
3371                 },
3372         },
3373 };
3374
3375 static int safexcel_aead_fallback_setkey(struct crypto_aead *ctfm,
3376                                          const u8 *key, unsigned int len)
3377 {
3378         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
3379         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3380
3381         /* Keep fallback cipher synchronized */
3382         return crypto_aead_setkey(ctx->fback, (u8 *)key, len) ?:
3383                safexcel_aead_setkey(ctfm, key, len);
3384 }
3385
3386 static int safexcel_aead_fallback_setauthsize(struct crypto_aead *ctfm,
3387                                               unsigned int authsize)
3388 {
3389         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
3390         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3391
3392         /* Keep fallback cipher synchronized */
3393         return crypto_aead_setauthsize(ctx->fback, authsize);
3394 }
3395
3396 static int safexcel_aead_fallback_crypt(struct aead_request *req,
3397                                         enum safexcel_cipher_direction dir)
3398 {
3399         struct crypto_aead *aead = crypto_aead_reqtfm(req);
3400         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
3401         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3402         struct aead_request *subreq = aead_request_ctx(req);
3403
3404         aead_request_set_tfm(subreq, ctx->fback);
3405         aead_request_set_callback(subreq, req->base.flags, req->base.complete,
3406                                   req->base.data);
3407         aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
3408                                req->iv);
3409         aead_request_set_ad(subreq, req->assoclen);
3410
3411         return (dir ==  SAFEXCEL_ENCRYPT) ?
3412                 crypto_aead_encrypt(subreq) :
3413                 crypto_aead_decrypt(subreq);
3414 }
3415
3416 static int safexcel_aead_sm4cbc_sm3_encrypt(struct aead_request *req)
3417 {
3418         struct safexcel_cipher_req *creq = aead_request_ctx(req);
3419
3420         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3421         if (req->cryptlen & (SM4_BLOCK_SIZE - 1))
3422                 return -EINVAL;
3423         else if (req->cryptlen || req->assoclen) /* If input length > 0 only */
3424                 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT);
3425
3426         /* HW cannot do full (AAD+payload) zero length, use fallback */
3427         return safexcel_aead_fallback_crypt(req, SAFEXCEL_ENCRYPT);
3428 }
3429
3430 static int safexcel_aead_sm4cbc_sm3_decrypt(struct aead_request *req)
3431 {
3432         struct safexcel_cipher_req *creq = aead_request_ctx(req);
3433         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3434
3435         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3436         if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1))
3437                 return -EINVAL;
3438         else if (req->cryptlen > crypto_aead_authsize(tfm) || req->assoclen)
3439                 /* If input length > 0 only */
3440                 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT);
3441
3442         /* HW cannot do full (AAD+payload) zero length, use fallback */
3443         return safexcel_aead_fallback_crypt(req, SAFEXCEL_DECRYPT);
3444 }
3445
3446 static int safexcel_aead_sm4cbc_sm3_cra_init(struct crypto_tfm *tfm)
3447 {
3448         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3449
3450         safexcel_aead_fallback_cra_init(tfm);
3451         ctx->alg = SAFEXCEL_SM4;
3452         ctx->blocksz = SM4_BLOCK_SIZE;
3453         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
3454         ctx->state_sz = SM3_DIGEST_SIZE;
3455         return 0;
3456 }
3457
3458 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4 = {
3459         .type = SAFEXCEL_ALG_TYPE_AEAD,
3460         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3,
3461         .alg.aead = {
3462                 .setkey = safexcel_aead_fallback_setkey,
3463                 .setauthsize = safexcel_aead_fallback_setauthsize,
3464                 .encrypt = safexcel_aead_sm4cbc_sm3_encrypt,
3465                 .decrypt = safexcel_aead_sm4cbc_sm3_decrypt,
3466                 .ivsize = SM4_BLOCK_SIZE,
3467                 .maxauthsize = SM3_DIGEST_SIZE,
3468                 .base = {
3469                         .cra_name = "authenc(hmac(sm3),cbc(sm4))",
3470                         .cra_driver_name = "safexcel-authenc-hmac-sm3-cbc-sm4",
3471                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3472                         .cra_flags = CRYPTO_ALG_ASYNC |
3473                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3474                                      CRYPTO_ALG_KERN_DRIVER_ONLY |
3475                                      CRYPTO_ALG_NEED_FALLBACK,
3476                         .cra_blocksize = SM4_BLOCK_SIZE,
3477                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3478                         .cra_alignmask = 0,
3479                         .cra_init = safexcel_aead_sm4cbc_sm3_cra_init,
3480                         .cra_exit = safexcel_aead_fallback_cra_exit,
3481                         .cra_module = THIS_MODULE,
3482                 },
3483         },
3484 };
3485
3486 static int safexcel_aead_sm4ctr_sha1_cra_init(struct crypto_tfm *tfm)
3487 {
3488         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3489
3490         safexcel_aead_sm4cbc_sha1_cra_init(tfm);
3491         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD;
3492         return 0;
3493 }
3494
3495 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_sm4 = {
3496         .type = SAFEXCEL_ALG_TYPE_AEAD,
3497         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1,
3498         .alg.aead = {
3499                 .setkey = safexcel_aead_setkey,
3500                 .encrypt = safexcel_aead_encrypt,
3501                 .decrypt = safexcel_aead_decrypt,
3502                 .ivsize = CTR_RFC3686_IV_SIZE,
3503                 .maxauthsize = SHA1_DIGEST_SIZE,
3504                 .base = {
3505                         .cra_name = "authenc(hmac(sha1),rfc3686(ctr(sm4)))",
3506                         .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-sm4",
3507                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3508                         .cra_flags = CRYPTO_ALG_ASYNC |
3509                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3510                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3511                         .cra_blocksize = 1,
3512                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3513                         .cra_alignmask = 0,
3514                         .cra_init = safexcel_aead_sm4ctr_sha1_cra_init,
3515                         .cra_exit = safexcel_aead_cra_exit,
3516                         .cra_module = THIS_MODULE,
3517                 },
3518         },
3519 };
3520
3521 static int safexcel_aead_sm4ctr_sm3_cra_init(struct crypto_tfm *tfm)
3522 {
3523         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3524
3525         safexcel_aead_sm4cbc_sm3_cra_init(tfm);
3526         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD;
3527         return 0;
3528 }
3529
3530 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4 = {
3531         .type = SAFEXCEL_ALG_TYPE_AEAD,
3532         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3,
3533         .alg.aead = {
3534                 .setkey = safexcel_aead_setkey,
3535                 .encrypt = safexcel_aead_encrypt,
3536                 .decrypt = safexcel_aead_decrypt,
3537                 .ivsize = CTR_RFC3686_IV_SIZE,
3538                 .maxauthsize = SM3_DIGEST_SIZE,
3539                 .base = {
3540                         .cra_name = "authenc(hmac(sm3),rfc3686(ctr(sm4)))",
3541                         .cra_driver_name = "safexcel-authenc-hmac-sm3-ctr-sm4",
3542                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3543                         .cra_flags = CRYPTO_ALG_ASYNC |
3544                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3545                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3546                         .cra_blocksize = 1,
3547                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3548                         .cra_alignmask = 0,
3549                         .cra_init = safexcel_aead_sm4ctr_sm3_cra_init,
3550                         .cra_exit = safexcel_aead_cra_exit,
3551                         .cra_module = THIS_MODULE,
3552                 },
3553         },
3554 };
3555
3556 static int safexcel_rfc4106_gcm_setkey(struct crypto_aead *ctfm, const u8 *key,
3557                                        unsigned int len)
3558 {
3559         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
3560         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3561
3562         /* last 4 bytes of key are the nonce! */
3563         ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE);
3564
3565         len -= CTR_RFC3686_NONCE_SIZE;
3566         return safexcel_aead_gcm_setkey(ctfm, key, len);
3567 }
3568
3569 static int safexcel_rfc4106_gcm_setauthsize(struct crypto_aead *tfm,
3570                                             unsigned int authsize)
3571 {
3572         return crypto_rfc4106_check_authsize(authsize);
3573 }
3574
3575 static int safexcel_rfc4106_encrypt(struct aead_request *req)
3576 {
3577         return crypto_ipsec_check_assoclen(req->assoclen) ?:
3578                safexcel_aead_encrypt(req);
3579 }
3580
3581 static int safexcel_rfc4106_decrypt(struct aead_request *req)
3582 {
3583         return crypto_ipsec_check_assoclen(req->assoclen) ?:
3584                safexcel_aead_decrypt(req);
3585 }
3586
3587 static int safexcel_rfc4106_gcm_cra_init(struct crypto_tfm *tfm)
3588 {
3589         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3590         int ret;
3591
3592         ret = safexcel_aead_gcm_cra_init(tfm);
3593         ctx->aead  = EIP197_AEAD_TYPE_IPSEC_ESP;
3594         ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE;
3595         return ret;
3596 }
3597
3598 struct safexcel_alg_template safexcel_alg_rfc4106_gcm = {
3599         .type = SAFEXCEL_ALG_TYPE_AEAD,
3600         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH,
3601         .alg.aead = {
3602                 .setkey = safexcel_rfc4106_gcm_setkey,
3603                 .setauthsize = safexcel_rfc4106_gcm_setauthsize,
3604                 .encrypt = safexcel_rfc4106_encrypt,
3605                 .decrypt = safexcel_rfc4106_decrypt,
3606                 .ivsize = GCM_RFC4106_IV_SIZE,
3607                 .maxauthsize = GHASH_DIGEST_SIZE,
3608                 .base = {
3609                         .cra_name = "rfc4106(gcm(aes))",
3610                         .cra_driver_name = "safexcel-rfc4106-gcm-aes",
3611                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3612                         .cra_flags = CRYPTO_ALG_ASYNC |
3613                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3614                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3615                         .cra_blocksize = 1,
3616                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3617                         .cra_alignmask = 0,
3618                         .cra_init = safexcel_rfc4106_gcm_cra_init,
3619                         .cra_exit = safexcel_aead_gcm_cra_exit,
3620                 },
3621         },
3622 };
3623
3624 static int safexcel_rfc4543_gcm_setauthsize(struct crypto_aead *tfm,
3625                                             unsigned int authsize)
3626 {
3627         if (authsize != GHASH_DIGEST_SIZE)
3628                 return -EINVAL;
3629
3630         return 0;
3631 }
3632
3633 static int safexcel_rfc4543_gcm_cra_init(struct crypto_tfm *tfm)
3634 {
3635         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3636         int ret;
3637
3638         ret = safexcel_aead_gcm_cra_init(tfm);
3639         ctx->aead  = EIP197_AEAD_TYPE_IPSEC_ESP_GMAC;
3640         return ret;
3641 }
3642
3643 struct safexcel_alg_template safexcel_alg_rfc4543_gcm = {
3644         .type = SAFEXCEL_ALG_TYPE_AEAD,
3645         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH,
3646         .alg.aead = {
3647                 .setkey = safexcel_rfc4106_gcm_setkey,
3648                 .setauthsize = safexcel_rfc4543_gcm_setauthsize,
3649                 .encrypt = safexcel_rfc4106_encrypt,
3650                 .decrypt = safexcel_rfc4106_decrypt,
3651                 .ivsize = GCM_RFC4543_IV_SIZE,
3652                 .maxauthsize = GHASH_DIGEST_SIZE,
3653                 .base = {
3654                         .cra_name = "rfc4543(gcm(aes))",
3655                         .cra_driver_name = "safexcel-rfc4543-gcm-aes",
3656                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3657                         .cra_flags = CRYPTO_ALG_ASYNC |
3658                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3659                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3660                         .cra_blocksize = 1,
3661                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3662                         .cra_alignmask = 0,
3663                         .cra_init = safexcel_rfc4543_gcm_cra_init,
3664                         .cra_exit = safexcel_aead_gcm_cra_exit,
3665                 },
3666         },
3667 };
3668
3669 static int safexcel_rfc4309_ccm_setkey(struct crypto_aead *ctfm, const u8 *key,
3670                                        unsigned int len)
3671 {
3672         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
3673         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3674
3675         /* First byte of the nonce = L = always 3 for RFC4309 (4 byte ctr) */
3676         *(u8 *)&ctx->nonce = EIP197_AEAD_IPSEC_COUNTER_SIZE - 1;
3677         /* last 3 bytes of key are the nonce! */
3678         memcpy((u8 *)&ctx->nonce + 1, key + len -
3679                EIP197_AEAD_IPSEC_CCM_NONCE_SIZE,
3680                EIP197_AEAD_IPSEC_CCM_NONCE_SIZE);
3681
3682         len -= EIP197_AEAD_IPSEC_CCM_NONCE_SIZE;
3683         return safexcel_aead_ccm_setkey(ctfm, key, len);
3684 }
3685
3686 static int safexcel_rfc4309_ccm_setauthsize(struct crypto_aead *tfm,
3687                                             unsigned int authsize)
3688 {
3689         /* Borrowed from crypto/ccm.c */
3690         switch (authsize) {
3691         case 8:
3692         case 12:
3693         case 16:
3694                 break;
3695         default:
3696                 return -EINVAL;
3697         }
3698
3699         return 0;
3700 }
3701
3702 static int safexcel_rfc4309_ccm_encrypt(struct aead_request *req)
3703 {
3704         struct safexcel_cipher_req *creq = aead_request_ctx(req);
3705
3706         /* Borrowed from crypto/ccm.c */
3707         if (req->assoclen != 16 && req->assoclen != 20)
3708                 return -EINVAL;
3709
3710         return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT);
3711 }
3712
3713 static int safexcel_rfc4309_ccm_decrypt(struct aead_request *req)
3714 {
3715         struct safexcel_cipher_req *creq = aead_request_ctx(req);
3716
3717         /* Borrowed from crypto/ccm.c */
3718         if (req->assoclen != 16 && req->assoclen != 20)
3719                 return -EINVAL;
3720
3721         return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT);
3722 }
3723
3724 static int safexcel_rfc4309_ccm_cra_init(struct crypto_tfm *tfm)
3725 {
3726         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3727         int ret;
3728
3729         ret = safexcel_aead_ccm_cra_init(tfm);
3730         ctx->aead  = EIP197_AEAD_TYPE_IPSEC_ESP;
3731         ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE;
3732         return ret;
3733 }
3734
3735 struct safexcel_alg_template safexcel_alg_rfc4309_ccm = {
3736         .type = SAFEXCEL_ALG_TYPE_AEAD,
3737         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL,
3738         .alg.aead = {
3739                 .setkey = safexcel_rfc4309_ccm_setkey,
3740                 .setauthsize = safexcel_rfc4309_ccm_setauthsize,
3741                 .encrypt = safexcel_rfc4309_ccm_encrypt,
3742                 .decrypt = safexcel_rfc4309_ccm_decrypt,
3743                 .ivsize = EIP197_AEAD_IPSEC_IV_SIZE,
3744                 .maxauthsize = AES_BLOCK_SIZE,
3745                 .base = {
3746                         .cra_name = "rfc4309(ccm(aes))",
3747                         .cra_driver_name = "safexcel-rfc4309-ccm-aes",
3748                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3749                         .cra_flags = CRYPTO_ALG_ASYNC |
3750                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3751                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3752                         .cra_blocksize = 1,
3753                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3754                         .cra_alignmask = 0,
3755                         .cra_init = safexcel_rfc4309_ccm_cra_init,
3756                         .cra_exit = safexcel_aead_cra_exit,
3757                         .cra_module = THIS_MODULE,
3758                 },
3759         },
3760 };