drivers/crypto/caam/caamalg_qi2.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
   2 /*
   3  * Copyright 2015-2016 Freescale Semiconductor Inc.
   4  * Copyright 2017-2019 NXP
   5  */
   6
   7 #include "compat.h"
   8 #include "regs.h"
   9 #include "caamalg_qi2.h"
  10 #include "dpseci_cmd.h"
  11 #include "desc_constr.h"
  12 #include "error.h"
  13 #include "sg_sw_sec4.h"
  14 #include "sg_sw_qm2.h"
  15 #include "key_gen.h"
  16 #include "caamalg_desc.h"
  17 #include "caamhash_desc.h"
  18 #include "dpseci-debugfs.h"
  19 #include <linux/fsl/mc.h>
  20 #include <soc/fsl/dpaa2-io.h>
  21 #include <soc/fsl/dpaa2-fd.h>
  22 #include <crypto/xts.h>
  23 #include <asm/unaligned.h>
  24
  25 #define CAAM_CRA_PRIORITY       2000
  26
  27 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  28 #define CAAM_MAX_KEY_SIZE       (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \
  29                                  SHA512_DIGEST_SIZE * 2)
  30
  31 /*
  32  * This is a cache of buffers, from which the users of CAAM QI driver
  33  * can allocate short buffers. It's speedier than doing kmalloc on the hotpath.
  34  * NOTE: A more elegant solution would be to have some headroom in the frames
  35  *       being processed. This can be added by the dpaa2-eth driver. This would
  36  *       pose a problem for userspace application processing which cannot
  37  *       know of this limitation. So for now, this will work.
  38  * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
  39  */
  40 static struct kmem_cache *qi_cache;
  41
  42 struct caam_alg_entry {
  43         struct device *dev;
  44         int class1_alg_type;
  45         int class2_alg_type;
  46         bool rfc3686;
  47         bool geniv;
  48         bool nodkp;
  49 };
  50
  51 struct caam_aead_alg {
  52         struct aead_alg aead;
  53         struct caam_alg_entry caam;
  54         bool registered;
  55 };
  56
  57 struct caam_skcipher_alg {
  58         struct skcipher_alg skcipher;
  59         struct caam_alg_entry caam;
  60         bool registered;
  61 };
  62
  63 /**
  64  * struct caam_ctx - per-session context
  65  * @flc: Flow Contexts array
  66  * @key:  [authentication key], encryption key
  67  * @flc_dma: I/O virtual addresses of the Flow Contexts
  68  * @key_dma: I/O virtual address of the key
  69  * @dir: DMA direction for mapping key and Flow Contexts
  70  * @dev: dpseci device
  71  * @adata: authentication algorithm details
  72  * @cdata: encryption algorithm details
  73  * @authsize: authentication tag (a.k.a. ICV / MAC) size
  74  * @xts_key_fallback: true if fallback tfm needs to be used due
  75  *                    to unsupported xts key lengths
  76  * @fallback: xts fallback tfm
  77  */
  78 struct caam_ctx {
  79         struct caam_flc flc[NUM_OP];
  80         u8 key[CAAM_MAX_KEY_SIZE];
  81         dma_addr_t flc_dma[NUM_OP];
  82         dma_addr_t key_dma;
  83         enum dma_data_direction dir;
  84         struct device *dev;
  85         struct alginfo adata;
  86         struct alginfo cdata;
  87         unsigned int authsize;
  88         bool xts_key_fallback;
  89         struct crypto_skcipher *fallback;
  90 };
  91
  92 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv,
  93                                      dma_addr_t iova_addr)
  94 {
  95         phys_addr_t phys_addr;
  96
  97         phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) :
  98                                    iova_addr;
  99
 100         return phys_to_virt(phys_addr);
 101 }
 102
 103 /*
 104  * qi_cache_zalloc - Allocate buffers from CAAM-QI cache
 105  *
 106  * Allocate data on the hotpath. Instead of using kzalloc, one can use the
 107  * services of the CAAM QI memory cache (backed by kmem_cache). The buffers
 108  * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for
 109  * hosting 16 SG entries.
 110  *
 111  * @flags - flags that would be used for the equivalent kmalloc(..) call
 112  *
 113  * Returns a pointer to a retrieved buffer on success or NULL on failure.
 114  */
 115 static inline void *qi_cache_zalloc(gfp_t flags)
 116 {
 117         return kmem_cache_zalloc(qi_cache, flags);
 118 }
 119
 120 /*
 121  * qi_cache_free - Frees buffers allocated from CAAM-QI cache
 122  *
 123  * @obj - buffer previously allocated by qi_cache_zalloc
 124  *
 125  * No checking is being done, the call is a passthrough call to
 126  * kmem_cache_free(...)
 127  */
 128 static inline void qi_cache_free(void *obj)
 129 {
 130         kmem_cache_free(qi_cache, obj);
 131 }
 132
 133 static struct caam_request *to_caam_req(struct crypto_async_request *areq)
 134 {
 135         switch (crypto_tfm_alg_type(areq->tfm)) {
 136         case CRYPTO_ALG_TYPE_SKCIPHER:
 137                 return skcipher_request_ctx_dma(skcipher_request_cast(areq));
 138         case CRYPTO_ALG_TYPE_AEAD:
 139                 return aead_request_ctx_dma(
 140                         container_of(areq, struct aead_request, base));
 141         case CRYPTO_ALG_TYPE_AHASH:
 142                 return ahash_request_ctx_dma(ahash_request_cast(areq));
 143         default:
 144                 return ERR_PTR(-EINVAL);
 145         }
 146 }
 147
 148 static void caam_unmap(struct device *dev, struct scatterlist *src,
 149                        struct scatterlist *dst, int src_nents,
 150                        int dst_nents, dma_addr_t iv_dma, int ivsize,
 151                        enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
 152                        int qm_sg_bytes)
 153 {
 154         if (dst != src) {
 155                 if (src_nents)
 156                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 157                 if (dst_nents)
 158                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 159         } else {
 160                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 161         }
 162
 163         if (iv_dma)
 164                 dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
 165
 166         if (qm_sg_bytes)
 167                 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
 168 }
 169
 170 static int aead_set_sh_desc(struct crypto_aead *aead)
 171 {
 172         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 173                                                  typeof(*alg), aead);
 174         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 175         unsigned int ivsize = crypto_aead_ivsize(aead);
 176         struct device *dev = ctx->dev;
 177         struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
 178         struct caam_flc *flc;
 179         u32 *desc;
 180         u32 ctx1_iv_off = 0;
 181         u32 *nonce = NULL;
 182         unsigned int data_len[2];
 183         u32 inl_mask;
 184         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 185                                OP_ALG_AAI_CTR_MOD128);
 186         const bool is_rfc3686 = alg->caam.rfc3686;
 187
 188         if (!ctx->cdata.keylen || !ctx->authsize)
 189                 return 0;
 190
 191         /*
 192          * AES-CTR needs to load IV in CONTEXT1 reg
 193          * at an offset of 128bits (16bytes)
 194          * CONTEXT1[255:128] = IV
 195          */
 196         if (ctr_mode)
 197                 ctx1_iv_off = 16;
 198
 199         /*
 200          * RFC3686 specific:
 201          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 202          */
 203         if (is_rfc3686) {
 204                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 205                 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
 206                                 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
 207         }
 208
 209         /*
 210          * In case |user key| > |derived key|, using DKP<imm,imm> would result
 211          * in invalid opcodes (last bytes of user key) in the resulting
 212          * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
 213          * addresses are needed.
 214          */
 215         ctx->adata.key_virt = ctx->key;
 216         ctx->adata.key_dma = ctx->key_dma;
 217
 218         ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 219         ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 220
 221         data_len[0] = ctx->adata.keylen_pad;
 222         data_len[1] = ctx->cdata.keylen;
 223
 224         /* aead_encrypt shared descriptor */
 225         if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN :
 226                                                  DESC_QI_AEAD_ENC_LEN) +
 227                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 228                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 229                               ARRAY_SIZE(data_len)) < 0)
 230                 return -EINVAL;
 231
 232         ctx->adata.key_inline = !!(inl_mask & 1);
 233         ctx->cdata.key_inline = !!(inl_mask & 2);
 234
 235         flc = &ctx->flc[ENCRYPT];
 236         desc = flc->sh_desc;
 237
 238         if (alg->caam.geniv)
 239                 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata,
 240                                           ivsize, ctx->authsize, is_rfc3686,
 241                                           nonce, ctx1_iv_off, true,
 242                                           priv->sec_attr.era);
 243         else
 244                 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata,
 245                                        ivsize, ctx->authsize, is_rfc3686, nonce,
 246                                        ctx1_iv_off, true, priv->sec_attr.era);
 247
 248         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 249         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 250                                    sizeof(flc->flc) + desc_bytes(desc),
 251                                    ctx->dir);
 252
 253         /* aead_decrypt shared descriptor */
 254         if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
 255                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 256                               DESC_JOB_IO_LEN, data_len, &inl_mask,
 257                               ARRAY_SIZE(data_len)) < 0)
 258                 return -EINVAL;
 259
 260         ctx->adata.key_inline = !!(inl_mask & 1);
 261         ctx->cdata.key_inline = !!(inl_mask & 2);
 262
 263         flc = &ctx->flc[DECRYPT];
 264         desc = flc->sh_desc;
 265         cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata,
 266                                ivsize, ctx->authsize, alg->caam.geniv,
 267                                is_rfc3686, nonce, ctx1_iv_off, true,
 268                                priv->sec_attr.era);
 269         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 270         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 271                                    sizeof(flc->flc) + desc_bytes(desc),
 272                                    ctx->dir);
 273
 274         return 0;
 275 }
 276
 277 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 278 {
 279         struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
 280
 281         ctx->authsize = authsize;
 282         aead_set_sh_desc(authenc);
 283
 284         return 0;
 285 }
 286
 287 static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 288                        unsigned int keylen)
 289 {
 290         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 291         struct device *dev = ctx->dev;
 292         struct crypto_authenc_keys keys;
 293
 294         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 295                 goto badkey;
 296
 297         dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n",
 298                 keys.authkeylen + keys.enckeylen, keys.enckeylen,
 299                 keys.authkeylen);
 300         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 301                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 302
 303         ctx->adata.keylen = keys.authkeylen;
 304         ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
 305                                               OP_ALG_ALGSEL_MASK);
 306
 307         if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
 308                 goto badkey;
 309
 310         memcpy(ctx->key, keys.authkey, keys.authkeylen);
 311         memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 312         dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad +
 313                                    keys.enckeylen, ctx->dir);
 314         print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
 315                              DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 316                              ctx->adata.keylen_pad + keys.enckeylen, 1);
 317
 318         ctx->cdata.keylen = keys.enckeylen;
 319
 320         memzero_explicit(&keys, sizeof(keys));
 321         return aead_set_sh_desc(aead);
 322 badkey:
 323         memzero_explicit(&keys, sizeof(keys));
 324         return -EINVAL;
 325 }
 326
 327 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
 328                             unsigned int keylen)
 329 {
 330         struct crypto_authenc_keys keys;
 331         int err;
 332
 333         err = crypto_authenc_extractkeys(&keys, key, keylen);
 334         if (unlikely(err))
 335                 goto out;
 336
 337         err = -EINVAL;
 338         if (keys.enckeylen != DES3_EDE_KEY_SIZE)
 339                 goto out;
 340
 341         err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?:
 342               aead_setkey(aead, key, keylen);
 343
 344 out:
 345         memzero_explicit(&keys, sizeof(keys));
 346         return err;
 347 }
 348
 349 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 350                                            bool encrypt)
 351 {
 352         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 353         struct caam_request *req_ctx = aead_request_ctx_dma(req);
 354         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
 355         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
 356         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 357         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 358                                                  typeof(*alg), aead);
 359         struct device *dev = ctx->dev;
 360         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 361                       GFP_KERNEL : GFP_ATOMIC;
 362         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
 363         int src_len, dst_len = 0;
 364         struct aead_edesc *edesc;
 365         dma_addr_t qm_sg_dma, iv_dma = 0;
 366         int ivsize = 0;
 367         unsigned int authsize = ctx->authsize;
 368         int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes;
 369         int in_len, out_len;
 370         struct dpaa2_sg_entry *sg_table;
 371
 372         /* allocate space for base edesc, link tables and IV */
 373         edesc = qi_cache_zalloc(GFP_DMA | flags);
 374         if (unlikely(!edesc)) {
 375                 dev_err(dev, "could not allocate extended descriptor\n");
 376                 return ERR_PTR(-ENOMEM);
 377         }
 378
 379         if (unlikely(req->dst != req->src)) {
 380                 src_len = req->assoclen + req->cryptlen;
 381                 dst_len = src_len + (encrypt ? authsize : (-authsize));
 382
 383                 src_nents = sg_nents_for_len(req->src, src_len);
 384                 if (unlikely(src_nents < 0)) {
 385                         dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
 386                                 src_len);
 387                         qi_cache_free(edesc);
 388                         return ERR_PTR(src_nents);
 389                 }
 390
 391                 dst_nents = sg_nents_for_len(req->dst, dst_len);
 392                 if (unlikely(dst_nents < 0)) {
 393                         dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
 394                                 dst_len);
 395                         qi_cache_free(edesc);
 396                         return ERR_PTR(dst_nents);
 397                 }
 398
 399                 if (src_nents) {
 400                         mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
 401                                                       DMA_TO_DEVICE);
 402                         if (unlikely(!mapped_src_nents)) {
 403                                 dev_err(dev, "unable to map source\n");
 404                                 qi_cache_free(edesc);
 405                                 return ERR_PTR(-ENOMEM);
 406                         }
 407                 } else {
 408                         mapped_src_nents = 0;
 409                 }
 410
 411                 if (dst_nents) {
 412                         mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
 413                                                       DMA_FROM_DEVICE);
 414                         if (unlikely(!mapped_dst_nents)) {
 415                                 dev_err(dev, "unable to map destination\n");
 416                                 dma_unmap_sg(dev, req->src, src_nents,
 417                                              DMA_TO_DEVICE);
 418                                 qi_cache_free(edesc);
 419                                 return ERR_PTR(-ENOMEM);
 420                         }
 421                 } else {
 422                         mapped_dst_nents = 0;
 423                 }
 424         } else {
 425                 src_len = req->assoclen + req->cryptlen +
 426                           (encrypt ? authsize : 0);
 427
 428                 src_nents = sg_nents_for_len(req->src, src_len);
 429                 if (unlikely(src_nents < 0)) {
 430                         dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
 431                                 src_len);
 432                         qi_cache_free(edesc);
 433                         return ERR_PTR(src_nents);
 434                 }
 435
 436                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
 437                                               DMA_BIDIRECTIONAL);
 438                 if (unlikely(!mapped_src_nents)) {
 439                         dev_err(dev, "unable to map source\n");
 440                         qi_cache_free(edesc);
 441                         return ERR_PTR(-ENOMEM);
 442                 }
 443         }
 444
 445         if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
 446                 ivsize = crypto_aead_ivsize(aead);
 447
 448         /*
 449          * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
 450          * Input is not contiguous.
 451          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
 452          * the end of the table by allocating more S/G entries. Logic:
 453          * if (src != dst && output S/G)
 454          *      pad output S/G, if needed
 455          * else if (src == dst && S/G)
 456          *      overlapping S/Gs; pad one of them
 457          * else if (input S/G) ...
 458          *      pad input S/G, if needed
 459          */
 460         qm_sg_nents = 1 + !!ivsize + mapped_src_nents;
 461         if (mapped_dst_nents > 1)
 462                 qm_sg_nents += pad_sg_nents(mapped_dst_nents);
 463         else if ((req->src == req->dst) && (mapped_src_nents > 1))
 464                 qm_sg_nents = max(pad_sg_nents(qm_sg_nents),
 465                                   1 + !!ivsize +
 466                                   pad_sg_nents(mapped_src_nents));
 467         else
 468                 qm_sg_nents = pad_sg_nents(qm_sg_nents);
 469
 470         sg_table = &edesc->sgt[0];
 471         qm_sg_bytes = qm_sg_nents * sizeof(*sg_table);
 472         if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
 473                      CAAM_QI_MEMCACHE_SIZE)) {
 474                 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
 475                         qm_sg_nents, ivsize);
 476                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
 477                            0, DMA_NONE, 0, 0);
 478                 qi_cache_free(edesc);
 479                 return ERR_PTR(-ENOMEM);
 480         }
 481
 482         if (ivsize) {
 483                 u8 *iv = (u8 *)(sg_table + qm_sg_nents);
 484
 485                 /* Make sure IV is located in a DMAable area */
 486                 memcpy(iv, req->iv, ivsize);
 487
 488                 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
 489                 if (dma_mapping_error(dev, iv_dma)) {
 490                         dev_err(dev, "unable to map IV\n");
 491                         caam_unmap(dev, req->src, req->dst, src_nents,
 492                                    dst_nents, 0, 0, DMA_NONE, 0, 0);
 493                         qi_cache_free(edesc);
 494                         return ERR_PTR(-ENOMEM);
 495                 }
 496         }
 497
 498         edesc->src_nents = src_nents;
 499         edesc->dst_nents = dst_nents;
 500         edesc->iv_dma = iv_dma;
 501
 502         if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) ==
 503             OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE)
 504                 /*
 505                  * The associated data comes already with the IV but we need
 506                  * to skip it when we authenticate or encrypt...
 507                  */
 508                 edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize);
 509         else
 510                 edesc->assoclen = cpu_to_caam32(req->assoclen);
 511         edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4,
 512                                              DMA_TO_DEVICE);
 513         if (dma_mapping_error(dev, edesc->assoclen_dma)) {
 514                 dev_err(dev, "unable to map assoclen\n");
 515                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
 516                            iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
 517                 qi_cache_free(edesc);
 518                 return ERR_PTR(-ENOMEM);
 519         }
 520
 521         dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
 522         qm_sg_index++;
 523         if (ivsize) {
 524                 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
 525                 qm_sg_index++;
 526         }
 527         sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
 528         qm_sg_index += mapped_src_nents;
 529
 530         if (mapped_dst_nents > 1)
 531                 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
 532
 533         qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
 534         if (dma_mapping_error(dev, qm_sg_dma)) {
 535                 dev_err(dev, "unable to map S/G table\n");
 536                 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
 537                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
 538                            iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
 539                 qi_cache_free(edesc);
 540                 return ERR_PTR(-ENOMEM);
 541         }
 542
 543         edesc->qm_sg_dma = qm_sg_dma;
 544         edesc->qm_sg_bytes = qm_sg_bytes;
 545
 546         out_len = req->assoclen + req->cryptlen +
 547                   (encrypt ? ctx->authsize : (-ctx->authsize));
 548         in_len = 4 + ivsize + req->assoclen + req->cryptlen;
 549
 550         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
 551         dpaa2_fl_set_final(in_fle, true);
 552         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
 553         dpaa2_fl_set_addr(in_fle, qm_sg_dma);
 554         dpaa2_fl_set_len(in_fle, in_len);
 555
 556         if (req->dst == req->src) {
 557                 if (mapped_src_nents == 1) {
 558                         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
 559                         dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src));
 560                 } else {
 561                         dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
 562                         dpaa2_fl_set_addr(out_fle, qm_sg_dma +
 563                                           (1 + !!ivsize) * sizeof(*sg_table));
 564                 }
 565         } else if (!mapped_dst_nents) {
 566                 /*
 567                  * crypto engine requires the output entry to be present when
 568                  * "frame list" FD is used.
 569                  * Since engine does not support FMT=2'b11 (unused entry type),
 570                  * leaving out_fle zeroized is the best option.
 571                  */
 572                 goto skip_out_fle;
 573         } else if (mapped_dst_nents == 1) {
 574                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
 575                 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
 576         } else {
 577                 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
 578                 dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index *
 579                                   sizeof(*sg_table));
 580         }
 581
 582         dpaa2_fl_set_len(out_fle, out_len);
 583
 584 skip_out_fle:
 585         return edesc;
 586 }
 587
 588 static int chachapoly_set_sh_desc(struct crypto_aead *aead)
 589 {
 590         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 591         unsigned int ivsize = crypto_aead_ivsize(aead);
 592         struct device *dev = ctx->dev;
 593         struct caam_flc *flc;
 594         u32 *desc;
 595
 596         if (!ctx->cdata.keylen || !ctx->authsize)
 597                 return 0;
 598
 599         flc = &ctx->flc[ENCRYPT];
 600         desc = flc->sh_desc;
 601         cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 602                                ctx->authsize, true, true);
 603         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 604         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 605                                    sizeof(flc->flc) + desc_bytes(desc),
 606                                    ctx->dir);
 607
 608         flc = &ctx->flc[DECRYPT];
 609         desc = flc->sh_desc;
 610         cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 611                                ctx->authsize, false, true);
 612         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 613         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 614                                    sizeof(flc->flc) + desc_bytes(desc),
 615                                    ctx->dir);
 616
 617         return 0;
 618 }
 619
 620 static int chachapoly_setauthsize(struct crypto_aead *aead,
 621                                   unsigned int authsize)
 622 {
 623         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 624
 625         if (authsize != POLY1305_DIGEST_SIZE)
 626                 return -EINVAL;
 627
 628         ctx->authsize = authsize;
 629         return chachapoly_set_sh_desc(aead);
 630 }
 631
 632 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
 633                              unsigned int keylen)
 634 {
 635         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 636         unsigned int ivsize = crypto_aead_ivsize(aead);
 637         unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
 638
 639         if (keylen != CHACHA_KEY_SIZE + saltlen)
 640                 return -EINVAL;
 641
 642         ctx->cdata.key_virt = key;
 643         ctx->cdata.keylen = keylen - saltlen;
 644
 645         return chachapoly_set_sh_desc(aead);
 646 }
 647
 648 static int gcm_set_sh_desc(struct crypto_aead *aead)
 649 {
 650         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 651         struct device *dev = ctx->dev;
 652         unsigned int ivsize = crypto_aead_ivsize(aead);
 653         struct caam_flc *flc;
 654         u32 *desc;
 655         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 656                         ctx->cdata.keylen;
 657
 658         if (!ctx->cdata.keylen || !ctx->authsize)
 659                 return 0;
 660
 661         /*
 662          * AES GCM encrypt shared descriptor
 663          * Job Descriptor and Shared Descriptor
 664          * must fit into the 64-word Descriptor h/w Buffer
 665          */
 666         if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
 667                 ctx->cdata.key_inline = true;
 668                 ctx->cdata.key_virt = ctx->key;
 669         } else {
 670                 ctx->cdata.key_inline = false;
 671                 ctx->cdata.key_dma = ctx->key_dma;
 672         }
 673
 674         flc = &ctx->flc[ENCRYPT];
 675         desc = flc->sh_desc;
 676         cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
 677         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 678         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 679                                    sizeof(flc->flc) + desc_bytes(desc),
 680                                    ctx->dir);
 681
 682         /*
 683          * Job Descriptor and Shared Descriptors
 684          * must all fit into the 64-word Descriptor h/w Buffer
 685          */
 686         if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
 687                 ctx->cdata.key_inline = true;
 688                 ctx->cdata.key_virt = ctx->key;
 689         } else {
 690                 ctx->cdata.key_inline = false;
 691                 ctx->cdata.key_dma = ctx->key_dma;
 692         }
 693
 694         flc = &ctx->flc[DECRYPT];
 695         desc = flc->sh_desc;
 696         cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
 697         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 698         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 699                                    sizeof(flc->flc) + desc_bytes(desc),
 700                                    ctx->dir);
 701
 702         return 0;
 703 }
 704
 705 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 706 {
 707         struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
 708         int err;
 709
 710         err = crypto_gcm_check_authsize(authsize);
 711         if (err)
 712                 return err;
 713
 714         ctx->authsize = authsize;
 715         gcm_set_sh_desc(authenc);
 716
 717         return 0;
 718 }
 719
 720 static int gcm_setkey(struct crypto_aead *aead,
 721                       const u8 *key, unsigned int keylen)
 722 {
 723         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 724         struct device *dev = ctx->dev;
 725         int ret;
 726
 727         ret = aes_check_keylen(keylen);
 728         if (ret)
 729                 return ret;
 730         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 731                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 732
 733         memcpy(ctx->key, key, keylen);
 734         dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir);
 735         ctx->cdata.keylen = keylen;
 736
 737         return gcm_set_sh_desc(aead);
 738 }
 739
 740 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 741 {
 742         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 743         struct device *dev = ctx->dev;
 744         unsigned int ivsize = crypto_aead_ivsize(aead);
 745         struct caam_flc *flc;
 746         u32 *desc;
 747         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 748                         ctx->cdata.keylen;
 749
 750         if (!ctx->cdata.keylen || !ctx->authsize)
 751                 return 0;
 752
 753         ctx->cdata.key_virt = ctx->key;
 754
 755         /*
 756          * RFC4106 encrypt shared descriptor
 757          * Job Descriptor and Shared Descriptor
 758          * must fit into the 64-word Descriptor h/w Buffer
 759          */
 760         if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
 761                 ctx->cdata.key_inline = true;
 762         } else {
 763                 ctx->cdata.key_inline = false;
 764                 ctx->cdata.key_dma = ctx->key_dma;
 765         }
 766
 767         flc = &ctx->flc[ENCRYPT];
 768         desc = flc->sh_desc;
 769         cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 770                                   true);
 771         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 772         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 773                                    sizeof(flc->flc) + desc_bytes(desc),
 774                                    ctx->dir);
 775
 776         /*
 777          * Job Descriptor and Shared Descriptors
 778          * must all fit into the 64-word Descriptor h/w Buffer
 779          */
 780         if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
 781                 ctx->cdata.key_inline = true;
 782         } else {
 783                 ctx->cdata.key_inline = false;
 784                 ctx->cdata.key_dma = ctx->key_dma;
 785         }
 786
 787         flc = &ctx->flc[DECRYPT];
 788         desc = flc->sh_desc;
 789         cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 790                                   true);
 791         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 792         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 793                                    sizeof(flc->flc) + desc_bytes(desc),
 794                                    ctx->dir);
 795
 796         return 0;
 797 }
 798
 799 static int rfc4106_setauthsize(struct crypto_aead *authenc,
 800                                unsigned int authsize)
 801 {
 802         struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
 803         int err;
 804
 805         err = crypto_rfc4106_check_authsize(authsize);
 806         if (err)
 807                 return err;
 808
 809         ctx->authsize = authsize;
 810         rfc4106_set_sh_desc(authenc);
 811
 812         return 0;
 813 }
 814
 815 static int rfc4106_setkey(struct crypto_aead *aead,
 816                           const u8 *key, unsigned int keylen)
 817 {
 818         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 819         struct device *dev = ctx->dev;
 820         int ret;
 821
 822         ret = aes_check_keylen(keylen - 4);
 823         if (ret)
 824                 return ret;
 825
 826         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 827                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 828
 829         memcpy(ctx->key, key, keylen);
 830         /*
 831          * The last four bytes of the key material are used as the salt value
 832          * in the nonce. Update the AES key length.
 833          */
 834         ctx->cdata.keylen = keylen - 4;
 835         dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
 836                                    ctx->dir);
 837
 838         return rfc4106_set_sh_desc(aead);
 839 }
 840
 841 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 842 {
 843         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 844         struct device *dev = ctx->dev;
 845         unsigned int ivsize = crypto_aead_ivsize(aead);
 846         struct caam_flc *flc;
 847         u32 *desc;
 848         int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
 849                         ctx->cdata.keylen;
 850
 851         if (!ctx->cdata.keylen || !ctx->authsize)
 852                 return 0;
 853
 854         ctx->cdata.key_virt = ctx->key;
 855
 856         /*
 857          * RFC4543 encrypt shared descriptor
 858          * Job Descriptor and Shared Descriptor
 859          * must fit into the 64-word Descriptor h/w Buffer
 860          */
 861         if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
 862                 ctx->cdata.key_inline = true;
 863         } else {
 864                 ctx->cdata.key_inline = false;
 865                 ctx->cdata.key_dma = ctx->key_dma;
 866         }
 867
 868         flc = &ctx->flc[ENCRYPT];
 869         desc = flc->sh_desc;
 870         cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 871                                   true);
 872         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 873         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 874                                    sizeof(flc->flc) + desc_bytes(desc),
 875                                    ctx->dir);
 876
 877         /*
 878          * Job Descriptor and Shared Descriptors
 879          * must all fit into the 64-word Descriptor h/w Buffer
 880          */
 881         if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
 882                 ctx->cdata.key_inline = true;
 883         } else {
 884                 ctx->cdata.key_inline = false;
 885                 ctx->cdata.key_dma = ctx->key_dma;
 886         }
 887
 888         flc = &ctx->flc[DECRYPT];
 889         desc = flc->sh_desc;
 890         cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 891                                   true);
 892         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 893         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 894                                    sizeof(flc->flc) + desc_bytes(desc),
 895                                    ctx->dir);
 896
 897         return 0;
 898 }
 899
 900 static int rfc4543_setauthsize(struct crypto_aead *authenc,
 901                                unsigned int authsize)
 902 {
 903         struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
 904
 905         if (authsize != 16)
 906                 return -EINVAL;
 907
 908         ctx->authsize = authsize;
 909         rfc4543_set_sh_desc(authenc);
 910
 911         return 0;
 912 }
 913
 914 static int rfc4543_setkey(struct crypto_aead *aead,
 915                           const u8 *key, unsigned int keylen)
 916 {
 917         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
 918         struct device *dev = ctx->dev;
 919         int ret;
 920
 921         ret = aes_check_keylen(keylen - 4);
 922         if (ret)
 923                 return ret;
 924
 925         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 926                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 927
 928         memcpy(ctx->key, key, keylen);
 929         /*
 930          * The last four bytes of the key material are used as the salt value
 931          * in the nonce. Update the AES key length.
 932          */
 933         ctx->cdata.keylen = keylen - 4;
 934         dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
 935                                    ctx->dir);
 936
 937         return rfc4543_set_sh_desc(aead);
 938 }
 939
 940 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 941                            unsigned int keylen, const u32 ctx1_iv_off)
 942 {
 943         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
 944         struct caam_skcipher_alg *alg =
 945                 container_of(crypto_skcipher_alg(skcipher),
 946                              struct caam_skcipher_alg, skcipher);
 947         struct device *dev = ctx->dev;
 948         struct caam_flc *flc;
 949         unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
 950         u32 *desc;
 951         const bool is_rfc3686 = alg->caam.rfc3686;
 952
 953         print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
 954                              DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 955
 956         ctx->cdata.keylen = keylen;
 957         ctx->cdata.key_virt = key;
 958         ctx->cdata.key_inline = true;
 959
 960         /* skcipher_encrypt shared descriptor */
 961         flc = &ctx->flc[ENCRYPT];
 962         desc = flc->sh_desc;
 963         cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
 964                                    ctx1_iv_off);
 965         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 966         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
 967                                    sizeof(flc->flc) + desc_bytes(desc),
 968                                    ctx->dir);
 969
 970         /* skcipher_decrypt shared descriptor */
 971         flc = &ctx->flc[DECRYPT];
 972         desc = flc->sh_desc;
 973         cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
 974                                    ctx1_iv_off);
 975         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
 976         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
 977                                    sizeof(flc->flc) + desc_bytes(desc),
 978                                    ctx->dir);
 979
 980         return 0;
 981 }
 982
 983 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
 984                                const u8 *key, unsigned int keylen)
 985 {
 986         int err;
 987
 988         err = aes_check_keylen(keylen);
 989         if (err)
 990                 return err;
 991
 992         return skcipher_setkey(skcipher, key, keylen, 0);
 993 }
 994
 995 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
 996                                    const u8 *key, unsigned int keylen)
 997 {
 998         u32 ctx1_iv_off;
 999         int err;
1000
1001         /*
1002          * RFC3686 specific:
1003          *      | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1004          *      | *key = {KEY, NONCE}
1005          */
1006         ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1007         keylen -= CTR_RFC3686_NONCE_SIZE;
1008
1009         err = aes_check_keylen(keylen);
1010         if (err)
1011                 return err;
1012
1013         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1014 }
1015
1016 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
1017                                const u8 *key, unsigned int keylen)
1018 {
1019         u32 ctx1_iv_off;
1020         int err;
1021
1022         /*
1023          * AES-CTR needs to load IV in CONTEXT1 reg
1024          * at an offset of 128bits (16bytes)
1025          * CONTEXT1[255:128] = IV
1026          */
1027         ctx1_iv_off = 16;
1028
1029         err = aes_check_keylen(keylen);
1030         if (err)
1031                 return err;
1032
1033         return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1034 }
1035
1036 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher,
1037                                     const u8 *key, unsigned int keylen)
1038 {
1039         if (keylen != CHACHA_KEY_SIZE)
1040                 return -EINVAL;
1041
1042         return skcipher_setkey(skcipher, key, keylen, 0);
1043 }
1044
1045 static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
1046                                const u8 *key, unsigned int keylen)
1047 {
1048         return verify_skcipher_des_key(skcipher, key) ?:
1049                skcipher_setkey(skcipher, key, keylen, 0);
1050 }
1051
1052 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
1053                                 const u8 *key, unsigned int keylen)
1054 {
1055         return verify_skcipher_des3_key(skcipher, key) ?:
1056                skcipher_setkey(skcipher, key, keylen, 0);
1057 }
1058
1059 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
1060                                unsigned int keylen)
1061 {
1062         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1063         struct device *dev = ctx->dev;
1064         struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
1065         struct caam_flc *flc;
1066         u32 *desc;
1067         int err;
1068
1069         err = xts_verify_key(skcipher, key, keylen);
1070         if (err) {
1071                 dev_dbg(dev, "key size mismatch\n");
1072                 return err;
1073         }
1074
1075         if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256)
1076                 ctx->xts_key_fallback = true;
1077
1078         if (priv->sec_attr.era <= 8 || ctx->xts_key_fallback) {
1079                 err = crypto_skcipher_setkey(ctx->fallback, key, keylen);
1080                 if (err)
1081                         return err;
1082         }
1083
1084         ctx->cdata.keylen = keylen;
1085         ctx->cdata.key_virt = key;
1086         ctx->cdata.key_inline = true;
1087
1088         /* xts_skcipher_encrypt shared descriptor */
1089         flc = &ctx->flc[ENCRYPT];
1090         desc = flc->sh_desc;
1091         cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
1092         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1093         dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
1094                                    sizeof(flc->flc) + desc_bytes(desc),
1095                                    ctx->dir);
1096
1097         /* xts_skcipher_decrypt shared descriptor */
1098         flc = &ctx->flc[DECRYPT];
1099         desc = flc->sh_desc;
1100         cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
1101         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1102         dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
1103                                    sizeof(flc->flc) + desc_bytes(desc),
1104                                    ctx->dir);
1105
1106         return 0;
1107 }
1108
1109 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
1110 {
1111         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1112         struct caam_request *req_ctx = skcipher_request_ctx_dma(req);
1113         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
1114         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
1115         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1116         struct device *dev = ctx->dev;
1117         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1118                        GFP_KERNEL : GFP_ATOMIC;
1119         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1120         struct skcipher_edesc *edesc;
1121         dma_addr_t iv_dma;
1122         u8 *iv;
1123         int ivsize = crypto_skcipher_ivsize(skcipher);
1124         int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
1125         struct dpaa2_sg_entry *sg_table;
1126
1127         src_nents = sg_nents_for_len(req->src, req->cryptlen);
1128         if (unlikely(src_nents < 0)) {
1129                 dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
1130                         req->cryptlen);
1131                 return ERR_PTR(src_nents);
1132         }
1133
1134         if (unlikely(req->dst != req->src)) {
1135                 dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1136                 if (unlikely(dst_nents < 0)) {
1137                         dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
1138                                 req->cryptlen);
1139                         return ERR_PTR(dst_nents);
1140                 }
1141
1142                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1143                                               DMA_TO_DEVICE);
1144                 if (unlikely(!mapped_src_nents)) {
1145                         dev_err(dev, "unable to map source\n");
1146                         return ERR_PTR(-ENOMEM);
1147                 }
1148
1149                 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
1150                                               DMA_FROM_DEVICE);
1151                 if (unlikely(!mapped_dst_nents)) {
1152                         dev_err(dev, "unable to map destination\n");
1153                         dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
1154                         return ERR_PTR(-ENOMEM);
1155                 }
1156         } else {
1157                 mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1158                                               DMA_BIDIRECTIONAL);
1159                 if (unlikely(!mapped_src_nents)) {
1160                         dev_err(dev, "unable to map source\n");
1161                         return ERR_PTR(-ENOMEM);
1162                 }
1163         }
1164
1165         qm_sg_ents = 1 + mapped_src_nents;
1166         dst_sg_idx = qm_sg_ents;
1167
1168         /*
1169          * Input, output HW S/G tables: [IV, src][dst, IV]
1170          * IV entries point to the same buffer
1171          * If src == dst, S/G entries are reused (S/G tables overlap)
1172          *
1173          * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1174          * the end of the table by allocating more S/G entries.
1175          */
1176         if (req->src != req->dst)
1177                 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
1178         else
1179                 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
1180
1181         qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry);
1182         if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
1183                      ivsize > CAAM_QI_MEMCACHE_SIZE)) {
1184                 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
1185                         qm_sg_ents, ivsize);
1186                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1187                            0, DMA_NONE, 0, 0);
1188                 return ERR_PTR(-ENOMEM);
1189         }
1190
1191         /* allocate space for base edesc, link tables and IV */
1192         edesc = qi_cache_zalloc(GFP_DMA | flags);
1193         if (unlikely(!edesc)) {
1194                 dev_err(dev, "could not allocate extended descriptor\n");
1195                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1196                            0, DMA_NONE, 0, 0);
1197                 return ERR_PTR(-ENOMEM);
1198         }
1199
1200         /* Make sure IV is located in a DMAable area */
1201         sg_table = &edesc->sgt[0];
1202         iv = (u8 *)(sg_table + qm_sg_ents);
1203         memcpy(iv, req->iv, ivsize);
1204
1205         iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL);
1206         if (dma_mapping_error(dev, iv_dma)) {
1207                 dev_err(dev, "unable to map IV\n");
1208                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1209                            0, DMA_NONE, 0, 0);
1210                 qi_cache_free(edesc);
1211                 return ERR_PTR(-ENOMEM);
1212         }
1213
1214         edesc->src_nents = src_nents;
1215         edesc->dst_nents = dst_nents;
1216         edesc->iv_dma = iv_dma;
1217         edesc->qm_sg_bytes = qm_sg_bytes;
1218
1219         dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
1220         sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
1221
1222         if (req->src != req->dst)
1223                 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
1224
1225         dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
1226                          ivsize, 0);
1227
1228         edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes,
1229                                           DMA_TO_DEVICE);
1230         if (dma_mapping_error(dev, edesc->qm_sg_dma)) {
1231                 dev_err(dev, "unable to map S/G table\n");
1232                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
1233                            iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
1234                 qi_cache_free(edesc);
1235                 return ERR_PTR(-ENOMEM);
1236         }
1237
1238         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
1239         dpaa2_fl_set_final(in_fle, true);
1240         dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize);
1241         dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize);
1242
1243         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
1244         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
1245
1246         dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
1247
1248         if (req->src == req->dst)
1249                 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma +
1250                                   sizeof(*sg_table));
1251         else
1252                 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx *
1253                                   sizeof(*sg_table));
1254
1255         return edesc;
1256 }
1257
1258 static void aead_unmap(struct device *dev, struct aead_edesc *edesc,
1259                        struct aead_request *req)
1260 {
1261         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1262         int ivsize = crypto_aead_ivsize(aead);
1263
1264         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1265                    edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
1266                    edesc->qm_sg_bytes);
1267         dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
1268 }
1269
1270 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
1271                            struct skcipher_request *req)
1272 {
1273         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1274         int ivsize = crypto_skcipher_ivsize(skcipher);
1275
1276         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1277                    edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
1278                    edesc->qm_sg_bytes);
1279 }
1280
1281 static void aead_encrypt_done(void *cbk_ctx, u32 status)
1282 {
1283         struct crypto_async_request *areq = cbk_ctx;
1284         struct aead_request *req = container_of(areq, struct aead_request,
1285                                                 base);
1286         struct caam_request *req_ctx = to_caam_req(areq);
1287         struct aead_edesc *edesc = req_ctx->edesc;
1288         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1289         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1290         int ecode = 0;
1291
1292         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1293
1294         if (unlikely(status))
1295                 ecode = caam_qi2_strstatus(ctx->dev, status);
1296
1297         aead_unmap(ctx->dev, edesc, req);
1298         qi_cache_free(edesc);
1299         aead_request_complete(req, ecode);
1300 }
1301
1302 static void aead_decrypt_done(void *cbk_ctx, u32 status)
1303 {
1304         struct crypto_async_request *areq = cbk_ctx;
1305         struct aead_request *req = container_of(areq, struct aead_request,
1306                                                 base);
1307         struct caam_request *req_ctx = to_caam_req(areq);
1308         struct aead_edesc *edesc = req_ctx->edesc;
1309         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1310         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1311         int ecode = 0;
1312
1313         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1314
1315         if (unlikely(status))
1316                 ecode = caam_qi2_strstatus(ctx->dev, status);
1317
1318         aead_unmap(ctx->dev, edesc, req);
1319         qi_cache_free(edesc);
1320         aead_request_complete(req, ecode);
1321 }
1322
1323 static int aead_encrypt(struct aead_request *req)
1324 {
1325         struct aead_edesc *edesc;
1326         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1327         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1328         struct caam_request *caam_req = aead_request_ctx_dma(req);
1329         int ret;
1330
1331         /* allocate extended descriptor */
1332         edesc = aead_edesc_alloc(req, true);
1333         if (IS_ERR(edesc))
1334                 return PTR_ERR(edesc);
1335
1336         caam_req->flc = &ctx->flc[ENCRYPT];
1337         caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1338         caam_req->cbk = aead_encrypt_done;
1339         caam_req->ctx = &req->base;
1340         caam_req->edesc = edesc;
1341         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1342         if (ret != -EINPROGRESS &&
1343             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1344                 aead_unmap(ctx->dev, edesc, req);
1345                 qi_cache_free(edesc);
1346         }
1347
1348         return ret;
1349 }
1350
1351 static int aead_decrypt(struct aead_request *req)
1352 {
1353         struct aead_edesc *edesc;
1354         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1355         struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1356         struct caam_request *caam_req = aead_request_ctx_dma(req);
1357         int ret;
1358
1359         /* allocate extended descriptor */
1360         edesc = aead_edesc_alloc(req, false);
1361         if (IS_ERR(edesc))
1362                 return PTR_ERR(edesc);
1363
1364         caam_req->flc = &ctx->flc[DECRYPT];
1365         caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1366         caam_req->cbk = aead_decrypt_done;
1367         caam_req->ctx = &req->base;
1368         caam_req->edesc = edesc;
1369         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1370         if (ret != -EINPROGRESS &&
1371             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1372                 aead_unmap(ctx->dev, edesc, req);
1373                 qi_cache_free(edesc);
1374         }
1375
1376         return ret;
1377 }
1378
1379 static int ipsec_gcm_encrypt(struct aead_request *req)
1380 {
1381         return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req);
1382 }
1383
1384 static int ipsec_gcm_decrypt(struct aead_request *req)
1385 {
1386         return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req);
1387 }
1388
1389 static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
1390 {
1391         struct crypto_async_request *areq = cbk_ctx;
1392         struct skcipher_request *req = skcipher_request_cast(areq);
1393         struct caam_request *req_ctx = to_caam_req(areq);
1394         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1395         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1396         struct skcipher_edesc *edesc = req_ctx->edesc;
1397         int ecode = 0;
1398         int ivsize = crypto_skcipher_ivsize(skcipher);
1399
1400         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1401
1402         if (unlikely(status))
1403                 ecode = caam_qi2_strstatus(ctx->dev, status);
1404
1405         print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1406                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1407                              edesc->src_nents > 1 ? 100 : ivsize, 1);
1408         caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1409                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1410                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1411
1412         skcipher_unmap(ctx->dev, edesc, req);
1413
1414         /*
1415          * The crypto API expects us to set the IV (req->iv) to the last
1416          * ciphertext block (CBC mode) or last counter (CTR mode).
1417          * This is used e.g. by the CTS mode.
1418          */
1419         if (!ecode)
1420                 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1421                        ivsize);
1422
1423         qi_cache_free(edesc);
1424         skcipher_request_complete(req, ecode);
1425 }
1426
1427 static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
1428 {
1429         struct crypto_async_request *areq = cbk_ctx;
1430         struct skcipher_request *req = skcipher_request_cast(areq);
1431         struct caam_request *req_ctx = to_caam_req(areq);
1432         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1433         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1434         struct skcipher_edesc *edesc = req_ctx->edesc;
1435         int ecode = 0;
1436         int ivsize = crypto_skcipher_ivsize(skcipher);
1437
1438         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1439
1440         if (unlikely(status))
1441                 ecode = caam_qi2_strstatus(ctx->dev, status);
1442
1443         print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1444                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1445                              edesc->src_nents > 1 ? 100 : ivsize, 1);
1446         caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1447                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1448                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1449
1450         skcipher_unmap(ctx->dev, edesc, req);
1451
1452         /*
1453          * The crypto API expects us to set the IV (req->iv) to the last
1454          * ciphertext block (CBC mode) or last counter (CTR mode).
1455          * This is used e.g. by the CTS mode.
1456          */
1457         if (!ecode)
1458                 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1459                        ivsize);
1460
1461         qi_cache_free(edesc);
1462         skcipher_request_complete(req, ecode);
1463 }
1464
1465 static inline bool xts_skcipher_ivsize(struct skcipher_request *req)
1466 {
1467         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1468         unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
1469
1470         return !!get_unaligned((u64 *)(req->iv + (ivsize / 2)));
1471 }
1472
1473 static int skcipher_encrypt(struct skcipher_request *req)
1474 {
1475         struct skcipher_edesc *edesc;
1476         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1477         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1478         struct caam_request *caam_req = skcipher_request_ctx_dma(req);
1479         struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
1480         int ret;
1481
1482         /*
1483          * XTS is expected to return an error even for input length = 0
1484          * Note that the case input length < block size will be caught during
1485          * HW offloading and return an error.
1486          */
1487         if (!req->cryptlen && !ctx->fallback)
1488                 return 0;
1489
1490         if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) ||
1491                               ctx->xts_key_fallback)) {
1492                 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback);
1493                 skcipher_request_set_callback(&caam_req->fallback_req,
1494                                               req->base.flags,
1495                                               req->base.complete,
1496                                               req->base.data);
1497                 skcipher_request_set_crypt(&caam_req->fallback_req, req->src,
1498                                            req->dst, req->cryptlen, req->iv);
1499
1500                 return crypto_skcipher_encrypt(&caam_req->fallback_req);
1501         }
1502
1503         /* allocate extended descriptor */
1504         edesc = skcipher_edesc_alloc(req);
1505         if (IS_ERR(edesc))
1506                 return PTR_ERR(edesc);
1507
1508         caam_req->flc = &ctx->flc[ENCRYPT];
1509         caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1510         caam_req->cbk = skcipher_encrypt_done;
1511         caam_req->ctx = &req->base;
1512         caam_req->edesc = edesc;
1513         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1514         if (ret != -EINPROGRESS &&
1515             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1516                 skcipher_unmap(ctx->dev, edesc, req);
1517                 qi_cache_free(edesc);
1518         }
1519
1520         return ret;
1521 }
1522
1523 static int skcipher_decrypt(struct skcipher_request *req)
1524 {
1525         struct skcipher_edesc *edesc;
1526         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1527         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1528         struct caam_request *caam_req = skcipher_request_ctx_dma(req);
1529         struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
1530         int ret;
1531
1532         /*
1533          * XTS is expected to return an error even for input length = 0
1534          * Note that the case input length < block size will be caught during
1535          * HW offloading and return an error.
1536          */
1537         if (!req->cryptlen && !ctx->fallback)
1538                 return 0;
1539
1540         if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) ||
1541                               ctx->xts_key_fallback)) {
1542                 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback);
1543                 skcipher_request_set_callback(&caam_req->fallback_req,
1544                                               req->base.flags,
1545                                               req->base.complete,
1546                                               req->base.data);
1547                 skcipher_request_set_crypt(&caam_req->fallback_req, req->src,
1548                                            req->dst, req->cryptlen, req->iv);
1549
1550                 return crypto_skcipher_decrypt(&caam_req->fallback_req);
1551         }
1552
1553         /* allocate extended descriptor */
1554         edesc = skcipher_edesc_alloc(req);
1555         if (IS_ERR(edesc))
1556                 return PTR_ERR(edesc);
1557
1558         caam_req->flc = &ctx->flc[DECRYPT];
1559         caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1560         caam_req->cbk = skcipher_decrypt_done;
1561         caam_req->ctx = &req->base;
1562         caam_req->edesc = edesc;
1563         ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1564         if (ret != -EINPROGRESS &&
1565             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1566                 skcipher_unmap(ctx->dev, edesc, req);
1567                 qi_cache_free(edesc);
1568         }
1569
1570         return ret;
1571 }
1572
1573 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam,
1574                          bool uses_dkp)
1575 {
1576         dma_addr_t dma_addr;
1577         int i;
1578
1579         /* copy descriptor header template value */
1580         ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
1581         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
1582
1583         ctx->dev = caam->dev;
1584         ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
1585
1586         dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc,
1587                                         offsetof(struct caam_ctx, flc_dma),
1588                                         ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
1589         if (dma_mapping_error(ctx->dev, dma_addr)) {
1590                 dev_err(ctx->dev, "unable to map key, shared descriptors\n");
1591                 return -ENOMEM;
1592         }
1593
1594         for (i = 0; i < NUM_OP; i++)
1595                 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
1596         ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]);
1597
1598         return 0;
1599 }
1600
1601 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm)
1602 {
1603         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1604         struct caam_skcipher_alg *caam_alg =
1605                 container_of(alg, typeof(*caam_alg), skcipher);
1606         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
1607         u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
1608         int ret = 0;
1609
1610         if (alg_aai == OP_ALG_AAI_XTS) {
1611                 const char *tfm_name = crypto_tfm_alg_name(&tfm->base);
1612                 struct crypto_skcipher *fallback;
1613
1614                 fallback = crypto_alloc_skcipher(tfm_name, 0,
1615                                                  CRYPTO_ALG_NEED_FALLBACK);
1616                 if (IS_ERR(fallback)) {
1617                         dev_err(caam_alg->caam.dev,
1618                                 "Failed to allocate %s fallback: %ld\n",
1619                                 tfm_name, PTR_ERR(fallback));
1620                         return PTR_ERR(fallback);
1621                 }
1622
1623                 ctx->fallback = fallback;
1624                 crypto_skcipher_set_reqsize_dma(
1625                         tfm, sizeof(struct caam_request) +
1626                              crypto_skcipher_reqsize(fallback));
1627         } else {
1628                 crypto_skcipher_set_reqsize_dma(tfm,
1629                                                 sizeof(struct caam_request));
1630         }
1631
1632         ret = caam_cra_init(ctx, &caam_alg->caam, false);
1633         if (ret && ctx->fallback)
1634                 crypto_free_skcipher(ctx->fallback);
1635
1636         return ret;
1637 }
1638
1639 static int caam_cra_init_aead(struct crypto_aead *tfm)
1640 {
1641         struct aead_alg *alg = crypto_aead_alg(tfm);
1642         struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
1643                                                       aead);
1644
1645         crypto_aead_set_reqsize_dma(tfm, sizeof(struct caam_request));
1646         return caam_cra_init(crypto_aead_ctx_dma(tfm), &caam_alg->caam,
1647                              !caam_alg->caam.nodkp);
1648 }
1649
1650 static void caam_exit_common(struct caam_ctx *ctx)
1651 {
1652         dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0],
1653                                offsetof(struct caam_ctx, flc_dma), ctx->dir,
1654                                DMA_ATTR_SKIP_CPU_SYNC);
1655 }
1656
1657 static void caam_cra_exit(struct crypto_skcipher *tfm)
1658 {
1659         struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
1660
1661         if (ctx->fallback)
1662                 crypto_free_skcipher(ctx->fallback);
1663         caam_exit_common(ctx);
1664 }
1665
1666 static void caam_cra_exit_aead(struct crypto_aead *tfm)
1667 {
1668         caam_exit_common(crypto_aead_ctx_dma(tfm));
1669 }
1670
1671 static struct caam_skcipher_alg driver_algs[] = {
1672         {
1673                 .skcipher = {
1674                         .base = {
1675                                 .cra_name = "cbc(aes)",
1676                                 .cra_driver_name = "cbc-aes-caam-qi2",
1677                                 .cra_blocksize = AES_BLOCK_SIZE,
1678                         },
1679                         .setkey = aes_skcipher_setkey,
1680                         .encrypt = skcipher_encrypt,
1681                         .decrypt = skcipher_decrypt,
1682                         .min_keysize = AES_MIN_KEY_SIZE,
1683                         .max_keysize = AES_MAX_KEY_SIZE,
1684                         .ivsize = AES_BLOCK_SIZE,
1685                 },
1686                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1687         },
1688         {
1689                 .skcipher = {
1690                         .base = {
1691                                 .cra_name = "cbc(des3_ede)",
1692                                 .cra_driver_name = "cbc-3des-caam-qi2",
1693                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1694                         },
1695                         .setkey = des3_skcipher_setkey,
1696                         .encrypt = skcipher_encrypt,
1697                         .decrypt = skcipher_decrypt,
1698                         .min_keysize = DES3_EDE_KEY_SIZE,
1699                         .max_keysize = DES3_EDE_KEY_SIZE,
1700                         .ivsize = DES3_EDE_BLOCK_SIZE,
1701                 },
1702                 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1703         },
1704         {
1705                 .skcipher = {
1706                         .base = {
1707                                 .cra_name = "cbc(des)",
1708                                 .cra_driver_name = "cbc-des-caam-qi2",
1709                                 .cra_blocksize = DES_BLOCK_SIZE,
1710                         },
1711                         .setkey = des_skcipher_setkey,
1712                         .encrypt = skcipher_encrypt,
1713                         .decrypt = skcipher_decrypt,
1714                         .min_keysize = DES_KEY_SIZE,
1715                         .max_keysize = DES_KEY_SIZE,
1716                         .ivsize = DES_BLOCK_SIZE,
1717                 },
1718                 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1719         },
1720         {
1721                 .skcipher = {
1722                         .base = {
1723                                 .cra_name = "ctr(aes)",
1724                                 .cra_driver_name = "ctr-aes-caam-qi2",
1725                                 .cra_blocksize = 1,
1726                         },
1727                         .setkey = ctr_skcipher_setkey,
1728                         .encrypt = skcipher_encrypt,
1729                         .decrypt = skcipher_decrypt,
1730                         .min_keysize = AES_MIN_KEY_SIZE,
1731                         .max_keysize = AES_MAX_KEY_SIZE,
1732                         .ivsize = AES_BLOCK_SIZE,
1733                         .chunksize = AES_BLOCK_SIZE,
1734                 },
1735                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1736                                         OP_ALG_AAI_CTR_MOD128,
1737         },
1738         {
1739                 .skcipher = {
1740                         .base = {
1741                                 .cra_name = "rfc3686(ctr(aes))",
1742                                 .cra_driver_name = "rfc3686-ctr-aes-caam-qi2",
1743                                 .cra_blocksize = 1,
1744                         },
1745                         .setkey = rfc3686_skcipher_setkey,
1746                         .encrypt = skcipher_encrypt,
1747                         .decrypt = skcipher_decrypt,
1748                         .min_keysize = AES_MIN_KEY_SIZE +
1749                                        CTR_RFC3686_NONCE_SIZE,
1750                         .max_keysize = AES_MAX_KEY_SIZE +
1751                                        CTR_RFC3686_NONCE_SIZE,
1752                         .ivsize = CTR_RFC3686_IV_SIZE,
1753                         .chunksize = AES_BLOCK_SIZE,
1754                 },
1755                 .caam = {
1756                         .class1_alg_type = OP_ALG_ALGSEL_AES |
1757                                            OP_ALG_AAI_CTR_MOD128,
1758                         .rfc3686 = true,
1759                 },
1760         },
1761         {
1762                 .skcipher = {
1763                         .base = {
1764                                 .cra_name = "xts(aes)",
1765                                 .cra_driver_name = "xts-aes-caam-qi2",
1766                                 .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
1767                                 .cra_blocksize = AES_BLOCK_SIZE,
1768                         },
1769                         .setkey = xts_skcipher_setkey,
1770                         .encrypt = skcipher_encrypt,
1771                         .decrypt = skcipher_decrypt,
1772                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
1773                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
1774                         .ivsize = AES_BLOCK_SIZE,
1775                 },
1776                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1777         },
1778         {
1779                 .skcipher = {
1780                         .base = {
1781                                 .cra_name = "chacha20",
1782                                 .cra_driver_name = "chacha20-caam-qi2",
1783                                 .cra_blocksize = 1,
1784                         },
1785                         .setkey = chacha20_skcipher_setkey,
1786                         .encrypt = skcipher_encrypt,
1787                         .decrypt = skcipher_decrypt,
1788                         .min_keysize = CHACHA_KEY_SIZE,
1789                         .max_keysize = CHACHA_KEY_SIZE,
1790                         .ivsize = CHACHA_IV_SIZE,
1791                 },
1792                 .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20,
1793         },
1794 };
1795
1796 static struct caam_aead_alg driver_aeads[] = {
1797         {
1798                 .aead = {
1799                         .base = {
1800                                 .cra_name = "rfc4106(gcm(aes))",
1801                                 .cra_driver_name = "rfc4106-gcm-aes-caam-qi2",
1802                                 .cra_blocksize = 1,
1803                         },
1804                         .setkey = rfc4106_setkey,
1805                         .setauthsize = rfc4106_setauthsize,
1806                         .encrypt = ipsec_gcm_encrypt,
1807                         .decrypt = ipsec_gcm_decrypt,
1808                         .ivsize = 8,
1809                         .maxauthsize = AES_BLOCK_SIZE,
1810                 },
1811                 .caam = {
1812                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1813                         .nodkp = true,
1814                 },
1815         },
1816         {
1817                 .aead = {
1818                         .base = {
1819                                 .cra_name = "rfc4543(gcm(aes))",
1820                                 .cra_driver_name = "rfc4543-gcm-aes-caam-qi2",
1821                                 .cra_blocksize = 1,
1822                         },
1823                         .setkey = rfc4543_setkey,
1824                         .setauthsize = rfc4543_setauthsize,
1825                         .encrypt = ipsec_gcm_encrypt,
1826                         .decrypt = ipsec_gcm_decrypt,
1827                         .ivsize = 8,
1828                         .maxauthsize = AES_BLOCK_SIZE,
1829                 },
1830                 .caam = {
1831                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1832                         .nodkp = true,
1833                 },
1834         },
1835         /* Galois Counter Mode */
1836         {
1837                 .aead = {
1838                         .base = {
1839                                 .cra_name = "gcm(aes)",
1840                                 .cra_driver_name = "gcm-aes-caam-qi2",
1841                                 .cra_blocksize = 1,
1842                         },
1843                         .setkey = gcm_setkey,
1844                         .setauthsize = gcm_setauthsize,
1845                         .encrypt = aead_encrypt,
1846                         .decrypt = aead_decrypt,
1847                         .ivsize = 12,
1848                         .maxauthsize = AES_BLOCK_SIZE,
1849                 },
1850                 .caam = {
1851                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1852                         .nodkp = true,
1853                 }
1854         },
1855         /* single-pass ipsec_esp descriptor */
1856         {
1857                 .aead = {
1858                         .base = {
1859                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
1860                                 .cra_driver_name = "authenc-hmac-md5-"
1861                                                    "cbc-aes-caam-qi2",
1862                                 .cra_blocksize = AES_BLOCK_SIZE,
1863                         },
1864                         .setkey = aead_setkey,
1865                         .setauthsize = aead_setauthsize,
1866                         .encrypt = aead_encrypt,
1867                         .decrypt = aead_decrypt,
1868                         .ivsize = AES_BLOCK_SIZE,
1869                         .maxauthsize = MD5_DIGEST_SIZE,
1870                 },
1871                 .caam = {
1872                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1873                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1874                                            OP_ALG_AAI_HMAC_PRECOMP,
1875                 }
1876         },
1877         {
1878                 .aead = {
1879                         .base = {
1880                                 .cra_name = "echainiv(authenc(hmac(md5),"
1881                                             "cbc(aes)))",
1882                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
1883                                                    "cbc-aes-caam-qi2",
1884                                 .cra_blocksize = AES_BLOCK_SIZE,
1885                         },
1886                         .setkey = aead_setkey,
1887                         .setauthsize = aead_setauthsize,
1888                         .encrypt = aead_encrypt,
1889                         .decrypt = aead_decrypt,
1890                         .ivsize = AES_BLOCK_SIZE,
1891                         .maxauthsize = MD5_DIGEST_SIZE,
1892                 },
1893                 .caam = {
1894                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1895                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1896                                            OP_ALG_AAI_HMAC_PRECOMP,
1897                         .geniv = true,
1898                 }
1899         },
1900         {
1901                 .aead = {
1902                         .base = {
1903                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1904                                 .cra_driver_name = "authenc-hmac-sha1-"
1905                                                    "cbc-aes-caam-qi2",
1906                                 .cra_blocksize = AES_BLOCK_SIZE,
1907                         },
1908                         .setkey = aead_setkey,
1909                         .setauthsize = aead_setauthsize,
1910                         .encrypt = aead_encrypt,
1911                         .decrypt = aead_decrypt,
1912                         .ivsize = AES_BLOCK_SIZE,
1913                         .maxauthsize = SHA1_DIGEST_SIZE,
1914                 },
1915                 .caam = {
1916                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1917                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1918                                            OP_ALG_AAI_HMAC_PRECOMP,
1919                 }
1920         },
1921         {
1922                 .aead = {
1923                         .base = {
1924                                 .cra_name = "echainiv(authenc(hmac(sha1),"
1925                                             "cbc(aes)))",
1926                                 .cra_driver_name = "echainiv-authenc-"
1927                                                    "hmac-sha1-cbc-aes-caam-qi2",
1928                                 .cra_blocksize = AES_BLOCK_SIZE,
1929                         },
1930                         .setkey = aead_setkey,
1931                         .setauthsize = aead_setauthsize,
1932                         .encrypt = aead_encrypt,
1933                         .decrypt = aead_decrypt,
1934                         .ivsize = AES_BLOCK_SIZE,
1935                         .maxauthsize = SHA1_DIGEST_SIZE,
1936                 },
1937                 .caam = {
1938                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1939                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1940                                            OP_ALG_AAI_HMAC_PRECOMP,
1941                         .geniv = true,
1942                 },
1943         },
1944         {
1945                 .aead = {
1946                         .base = {
1947                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
1948                                 .cra_driver_name = "authenc-hmac-sha224-"
1949                                                    "cbc-aes-caam-qi2",
1950                                 .cra_blocksize = AES_BLOCK_SIZE,
1951                         },
1952                         .setkey = aead_setkey,
1953                         .setauthsize = aead_setauthsize,
1954                         .encrypt = aead_encrypt,
1955                         .decrypt = aead_decrypt,
1956                         .ivsize = AES_BLOCK_SIZE,
1957                         .maxauthsize = SHA224_DIGEST_SIZE,
1958                 },
1959                 .caam = {
1960                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1961                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1962                                            OP_ALG_AAI_HMAC_PRECOMP,
1963                 }
1964         },
1965         {
1966                 .aead = {
1967                         .base = {
1968                                 .cra_name = "echainiv(authenc(hmac(sha224),"
1969                                             "cbc(aes)))",
1970                                 .cra_driver_name = "echainiv-authenc-"
1971                                                    "hmac-sha224-cbc-aes-caam-qi2",
1972                                 .cra_blocksize = AES_BLOCK_SIZE,
1973                         },
1974                         .setkey = aead_setkey,
1975                         .setauthsize = aead_setauthsize,
1976                         .encrypt = aead_encrypt,
1977                         .decrypt = aead_decrypt,
1978                         .ivsize = AES_BLOCK_SIZE,
1979                         .maxauthsize = SHA224_DIGEST_SIZE,
1980                 },
1981                 .caam = {
1982                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1983                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1984                                            OP_ALG_AAI_HMAC_PRECOMP,
1985                         .geniv = true,
1986                 }
1987         },
1988         {
1989                 .aead = {
1990                         .base = {
1991                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1992                                 .cra_driver_name = "authenc-hmac-sha256-"
1993                                                    "cbc-aes-caam-qi2",
1994                                 .cra_blocksize = AES_BLOCK_SIZE,
1995                         },
1996                         .setkey = aead_setkey,
1997                         .setauthsize = aead_setauthsize,
1998                         .encrypt = aead_encrypt,
1999                         .decrypt = aead_decrypt,
2000                         .ivsize = AES_BLOCK_SIZE,
2001                         .maxauthsize = SHA256_DIGEST_SIZE,
2002                 },
2003                 .caam = {
2004                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2005                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2006                                            OP_ALG_AAI_HMAC_PRECOMP,
2007                 }
2008         },
2009         {
2010                 .aead = {
2011                         .base = {
2012                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2013                                             "cbc(aes)))",
2014                                 .cra_driver_name = "echainiv-authenc-"
2015                                                    "hmac-sha256-cbc-aes-"
2016                                                    "caam-qi2",
2017                                 .cra_blocksize = AES_BLOCK_SIZE,
2018                         },
2019                         .setkey = aead_setkey,
2020                         .setauthsize = aead_setauthsize,
2021                         .encrypt = aead_encrypt,
2022                         .decrypt = aead_decrypt,
2023                         .ivsize = AES_BLOCK_SIZE,
2024                         .maxauthsize = SHA256_DIGEST_SIZE,
2025                 },
2026                 .caam = {
2027                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2028                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2029                                            OP_ALG_AAI_HMAC_PRECOMP,
2030                         .geniv = true,
2031                 }
2032         },
2033         {
2034                 .aead = {
2035                         .base = {
2036                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2037                                 .cra_driver_name = "authenc-hmac-sha384-"
2038                                                    "cbc-aes-caam-qi2",
2039                                 .cra_blocksize = AES_BLOCK_SIZE,
2040                         },
2041                         .setkey = aead_setkey,
2042                         .setauthsize = aead_setauthsize,
2043                         .encrypt = aead_encrypt,
2044                         .decrypt = aead_decrypt,
2045                         .ivsize = AES_BLOCK_SIZE,
2046                         .maxauthsize = SHA384_DIGEST_SIZE,
2047                 },
2048                 .caam = {
2049                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2050                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2051                                            OP_ALG_AAI_HMAC_PRECOMP,
2052                 }
2053         },
2054         {
2055                 .aead = {
2056                         .base = {
2057                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2058                                             "cbc(aes)))",
2059                                 .cra_driver_name = "echainiv-authenc-"
2060                                                    "hmac-sha384-cbc-aes-"
2061                                                    "caam-qi2",
2062                                 .cra_blocksize = AES_BLOCK_SIZE,
2063                         },
2064                         .setkey = aead_setkey,
2065                         .setauthsize = aead_setauthsize,
2066                         .encrypt = aead_encrypt,
2067                         .decrypt = aead_decrypt,
2068                         .ivsize = AES_BLOCK_SIZE,
2069                         .maxauthsize = SHA384_DIGEST_SIZE,
2070                 },
2071                 .caam = {
2072                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2073                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2074                                            OP_ALG_AAI_HMAC_PRECOMP,
2075                         .geniv = true,
2076                 }
2077         },
2078         {
2079                 .aead = {
2080                         .base = {
2081                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2082                                 .cra_driver_name = "authenc-hmac-sha512-"
2083                                                    "cbc-aes-caam-qi2",
2084                                 .cra_blocksize = AES_BLOCK_SIZE,
2085                         },
2086                         .setkey = aead_setkey,
2087                         .setauthsize = aead_setauthsize,
2088                         .encrypt = aead_encrypt,
2089                         .decrypt = aead_decrypt,
2090                         .ivsize = AES_BLOCK_SIZE,
2091                         .maxauthsize = SHA512_DIGEST_SIZE,
2092                 },
2093                 .caam = {
2094                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2095                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2096                                            OP_ALG_AAI_HMAC_PRECOMP,
2097                 }
2098         },
2099         {
2100                 .aead = {
2101                         .base = {
2102                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2103                                             "cbc(aes)))",
2104                                 .cra_driver_name = "echainiv-authenc-"
2105                                                    "hmac-sha512-cbc-aes-"
2106                                                    "caam-qi2",
2107                                 .cra_blocksize = AES_BLOCK_SIZE,
2108                         },
2109                         .setkey = aead_setkey,
2110                         .setauthsize = aead_setauthsize,
2111                         .encrypt = aead_encrypt,
2112                         .decrypt = aead_decrypt,
2113                         .ivsize = AES_BLOCK_SIZE,
2114                         .maxauthsize = SHA512_DIGEST_SIZE,
2115                 },
2116                 .caam = {
2117                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2118                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2119                                            OP_ALG_AAI_HMAC_PRECOMP,
2120                         .geniv = true,
2121                 }
2122         },
2123         {
2124                 .aead = {
2125                         .base = {
2126                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2127                                 .cra_driver_name = "authenc-hmac-md5-"
2128                                                    "cbc-des3_ede-caam-qi2",
2129                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2130                         },
2131                         .setkey = des3_aead_setkey,
2132                         .setauthsize = aead_setauthsize,
2133                         .encrypt = aead_encrypt,
2134                         .decrypt = aead_decrypt,
2135                         .ivsize = DES3_EDE_BLOCK_SIZE,
2136                         .maxauthsize = MD5_DIGEST_SIZE,
2137                 },
2138                 .caam = {
2139                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2140                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2141                                            OP_ALG_AAI_HMAC_PRECOMP,
2142                 }
2143         },
2144         {
2145                 .aead = {
2146                         .base = {
2147                                 .cra_name = "echainiv(authenc(hmac(md5),"
2148                                             "cbc(des3_ede)))",
2149                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2150                                                    "cbc-des3_ede-caam-qi2",
2151                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2152                         },
2153                         .setkey = des3_aead_setkey,
2154                         .setauthsize = aead_setauthsize,
2155                         .encrypt = aead_encrypt,
2156                         .decrypt = aead_decrypt,
2157                         .ivsize = DES3_EDE_BLOCK_SIZE,
2158                         .maxauthsize = MD5_DIGEST_SIZE,
2159                 },
2160                 .caam = {
2161                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2162                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2163                                            OP_ALG_AAI_HMAC_PRECOMP,
2164                         .geniv = true,
2165                 }
2166         },
2167         {
2168                 .aead = {
2169                         .base = {
2170                                 .cra_name = "authenc(hmac(sha1),"
2171                                             "cbc(des3_ede))",
2172                                 .cra_driver_name = "authenc-hmac-sha1-"
2173                                                    "cbc-des3_ede-caam-qi2",
2174                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2175                         },
2176                         .setkey = des3_aead_setkey,
2177                         .setauthsize = aead_setauthsize,
2178                         .encrypt = aead_encrypt,
2179                         .decrypt = aead_decrypt,
2180                         .ivsize = DES3_EDE_BLOCK_SIZE,
2181                         .maxauthsize = SHA1_DIGEST_SIZE,
2182                 },
2183                 .caam = {
2184                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2185                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2186                                            OP_ALG_AAI_HMAC_PRECOMP,
2187                 },
2188         },
2189         {
2190                 .aead = {
2191                         .base = {
2192                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2193                                             "cbc(des3_ede)))",
2194                                 .cra_driver_name = "echainiv-authenc-"
2195                                                    "hmac-sha1-"
2196                                                    "cbc-des3_ede-caam-qi2",
2197                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2198                         },
2199                         .setkey = des3_aead_setkey,
2200                         .setauthsize = aead_setauthsize,
2201                         .encrypt = aead_encrypt,
2202                         .decrypt = aead_decrypt,
2203                         .ivsize = DES3_EDE_BLOCK_SIZE,
2204                         .maxauthsize = SHA1_DIGEST_SIZE,
2205                 },
2206                 .caam = {
2207                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2208                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2209                                            OP_ALG_AAI_HMAC_PRECOMP,
2210                         .geniv = true,
2211                 }
2212         },
2213         {
2214                 .aead = {
2215                         .base = {
2216                                 .cra_name = "authenc(hmac(sha224),"
2217                                             "cbc(des3_ede))",
2218                                 .cra_driver_name = "authenc-hmac-sha224-"
2219                                                    "cbc-des3_ede-caam-qi2",
2220                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2221                         },
2222                         .setkey = des3_aead_setkey,
2223                         .setauthsize = aead_setauthsize,
2224                         .encrypt = aead_encrypt,
2225                         .decrypt = aead_decrypt,
2226                         .ivsize = DES3_EDE_BLOCK_SIZE,
2227                         .maxauthsize = SHA224_DIGEST_SIZE,
2228                 },
2229                 .caam = {
2230                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2231                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2232                                            OP_ALG_AAI_HMAC_PRECOMP,
2233                 },
2234         },
2235         {
2236                 .aead = {
2237                         .base = {
2238                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2239                                             "cbc(des3_ede)))",
2240                                 .cra_driver_name = "echainiv-authenc-"
2241                                                    "hmac-sha224-"
2242                                                    "cbc-des3_ede-caam-qi2",
2243                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2244                         },
2245                         .setkey = des3_aead_setkey,
2246                         .setauthsize = aead_setauthsize,
2247                         .encrypt = aead_encrypt,
2248                         .decrypt = aead_decrypt,
2249                         .ivsize = DES3_EDE_BLOCK_SIZE,
2250                         .maxauthsize = SHA224_DIGEST_SIZE,
2251                 },
2252                 .caam = {
2253                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2254                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2255                                            OP_ALG_AAI_HMAC_PRECOMP,
2256                         .geniv = true,
2257                 }
2258         },
2259         {
2260                 .aead = {
2261                         .base = {
2262                                 .cra_name = "authenc(hmac(sha256),"
2263                                             "cbc(des3_ede))",
2264                                 .cra_driver_name = "authenc-hmac-sha256-"
2265                                                    "cbc-des3_ede-caam-qi2",
2266                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2267                         },
2268                         .setkey = des3_aead_setkey,
2269                         .setauthsize = aead_setauthsize,
2270                         .encrypt = aead_encrypt,
2271                         .decrypt = aead_decrypt,
2272                         .ivsize = DES3_EDE_BLOCK_SIZE,
2273                         .maxauthsize = SHA256_DIGEST_SIZE,
2274                 },
2275                 .caam = {
2276                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2277                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2278                                            OP_ALG_AAI_HMAC_PRECOMP,
2279                 },
2280         },
2281         {
2282                 .aead = {
2283                         .base = {
2284                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2285                                             "cbc(des3_ede)))",
2286                                 .cra_driver_name = "echainiv-authenc-"
2287                                                    "hmac-sha256-"
2288                                                    "cbc-des3_ede-caam-qi2",
2289                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2290                         },
2291                         .setkey = des3_aead_setkey,
2292                         .setauthsize = aead_setauthsize,
2293                         .encrypt = aead_encrypt,
2294                         .decrypt = aead_decrypt,
2295                         .ivsize = DES3_EDE_BLOCK_SIZE,
2296                         .maxauthsize = SHA256_DIGEST_SIZE,
2297                 },
2298                 .caam = {
2299                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2300                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2301                                            OP_ALG_AAI_HMAC_PRECOMP,
2302                         .geniv = true,
2303                 }
2304         },
2305         {
2306                 .aead = {
2307                         .base = {
2308                                 .cra_name = "authenc(hmac(sha384),"
2309                                             "cbc(des3_ede))",
2310                                 .cra_driver_name = "authenc-hmac-sha384-"
2311                                                    "cbc-des3_ede-caam-qi2",
2312                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2313                         },
2314                         .setkey = des3_aead_setkey,
2315                         .setauthsize = aead_setauthsize,
2316                         .encrypt = aead_encrypt,
2317                         .decrypt = aead_decrypt,
2318                         .ivsize = DES3_EDE_BLOCK_SIZE,
2319                         .maxauthsize = SHA384_DIGEST_SIZE,
2320                 },
2321                 .caam = {
2322                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2323                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2324                                            OP_ALG_AAI_HMAC_PRECOMP,
2325                 },
2326         },
2327         {
2328                 .aead = {
2329                         .base = {
2330                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2331                                             "cbc(des3_ede)))",
2332                                 .cra_driver_name = "echainiv-authenc-"
2333                                                    "hmac-sha384-"
2334                                                    "cbc-des3_ede-caam-qi2",
2335                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2336                         },
2337                         .setkey = des3_aead_setkey,
2338                         .setauthsize = aead_setauthsize,
2339                         .encrypt = aead_encrypt,
2340                         .decrypt = aead_decrypt,
2341                         .ivsize = DES3_EDE_BLOCK_SIZE,
2342                         .maxauthsize = SHA384_DIGEST_SIZE,
2343                 },
2344                 .caam = {
2345                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2346                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2347                                            OP_ALG_AAI_HMAC_PRECOMP,
2348                         .geniv = true,
2349                 }
2350         },
2351         {
2352                 .aead = {
2353                         .base = {
2354                                 .cra_name = "authenc(hmac(sha512),"
2355                                             "cbc(des3_ede))",
2356                                 .cra_driver_name = "authenc-hmac-sha512-"
2357                                                    "cbc-des3_ede-caam-qi2",
2358                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2359                         },
2360                         .setkey = des3_aead_setkey,
2361                         .setauthsize = aead_setauthsize,
2362                         .encrypt = aead_encrypt,
2363                         .decrypt = aead_decrypt,
2364                         .ivsize = DES3_EDE_BLOCK_SIZE,
2365                         .maxauthsize = SHA512_DIGEST_SIZE,
2366                 },
2367                 .caam = {
2368                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2369                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2370                                            OP_ALG_AAI_HMAC_PRECOMP,
2371                 },
2372         },
2373         {
2374                 .aead = {
2375                         .base = {
2376                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2377                                             "cbc(des3_ede)))",
2378                                 .cra_driver_name = "echainiv-authenc-"
2379                                                    "hmac-sha512-"
2380                                                    "cbc-des3_ede-caam-qi2",
2381                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2382                         },
2383                         .setkey = des3_aead_setkey,
2384                         .setauthsize = aead_setauthsize,
2385                         .encrypt = aead_encrypt,
2386                         .decrypt = aead_decrypt,
2387                         .ivsize = DES3_EDE_BLOCK_SIZE,
2388                         .maxauthsize = SHA512_DIGEST_SIZE,
2389                 },
2390                 .caam = {
2391                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2392                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2393                                            OP_ALG_AAI_HMAC_PRECOMP,
2394                         .geniv = true,
2395                 }
2396         },
2397         {
2398                 .aead = {
2399                         .base = {
2400                                 .cra_name = "authenc(hmac(md5),cbc(des))",
2401                                 .cra_driver_name = "authenc-hmac-md5-"
2402                                                    "cbc-des-caam-qi2",
2403                                 .cra_blocksize = DES_BLOCK_SIZE,
2404                         },
2405                         .setkey = aead_setkey,
2406                         .setauthsize = aead_setauthsize,
2407                         .encrypt = aead_encrypt,
2408                         .decrypt = aead_decrypt,
2409                         .ivsize = DES_BLOCK_SIZE,
2410                         .maxauthsize = MD5_DIGEST_SIZE,
2411                 },
2412                 .caam = {
2413                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2414                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2415                                            OP_ALG_AAI_HMAC_PRECOMP,
2416                 },
2417         },
2418         {
2419                 .aead = {
2420                         .base = {
2421                                 .cra_name = "echainiv(authenc(hmac(md5),"
2422                                             "cbc(des)))",
2423                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2424                                                    "cbc-des-caam-qi2",
2425                                 .cra_blocksize = DES_BLOCK_SIZE,
2426                         },
2427                         .setkey = aead_setkey,
2428                         .setauthsize = aead_setauthsize,
2429                         .encrypt = aead_encrypt,
2430                         .decrypt = aead_decrypt,
2431                         .ivsize = DES_BLOCK_SIZE,
2432                         .maxauthsize = MD5_DIGEST_SIZE,
2433                 },
2434                 .caam = {
2435                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2436                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2437                                            OP_ALG_AAI_HMAC_PRECOMP,
2438                         .geniv = true,
2439                 }
2440         },
2441         {
2442                 .aead = {
2443                         .base = {
2444                                 .cra_name = "authenc(hmac(sha1),cbc(des))",
2445                                 .cra_driver_name = "authenc-hmac-sha1-"
2446                                                    "cbc-des-caam-qi2",
2447                                 .cra_blocksize = DES_BLOCK_SIZE,
2448                         },
2449                         .setkey = aead_setkey,
2450                         .setauthsize = aead_setauthsize,
2451                         .encrypt = aead_encrypt,
2452                         .decrypt = aead_decrypt,
2453                         .ivsize = DES_BLOCK_SIZE,
2454                         .maxauthsize = SHA1_DIGEST_SIZE,
2455                 },
2456                 .caam = {
2457                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2458                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2459                                            OP_ALG_AAI_HMAC_PRECOMP,
2460                 },
2461         },
2462         {
2463                 .aead = {
2464                         .base = {
2465                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2466                                             "cbc(des)))",
2467                                 .cra_driver_name = "echainiv-authenc-"
2468                                                    "hmac-sha1-cbc-des-caam-qi2",
2469                                 .cra_blocksize = DES_BLOCK_SIZE,
2470                         },
2471                         .setkey = aead_setkey,
2472                         .setauthsize = aead_setauthsize,
2473                         .encrypt = aead_encrypt,
2474                         .decrypt = aead_decrypt,
2475                         .ivsize = DES_BLOCK_SIZE,
2476                         .maxauthsize = SHA1_DIGEST_SIZE,
2477                 },
2478                 .caam = {
2479                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2480                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2481                                            OP_ALG_AAI_HMAC_PRECOMP,
2482                         .geniv = true,
2483                 }
2484         },
2485         {
2486                 .aead = {
2487                         .base = {
2488                                 .cra_name = "authenc(hmac(sha224),cbc(des))",
2489                                 .cra_driver_name = "authenc-hmac-sha224-"
2490                                                    "cbc-des-caam-qi2",
2491                                 .cra_blocksize = DES_BLOCK_SIZE,
2492                         },
2493                         .setkey = aead_setkey,
2494                         .setauthsize = aead_setauthsize,
2495                         .encrypt = aead_encrypt,
2496                         .decrypt = aead_decrypt,
2497                         .ivsize = DES_BLOCK_SIZE,
2498                         .maxauthsize = SHA224_DIGEST_SIZE,
2499                 },
2500                 .caam = {
2501                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2502                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2503                                            OP_ALG_AAI_HMAC_PRECOMP,
2504                 },
2505         },
2506         {
2507                 .aead = {
2508                         .base = {
2509                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2510                                             "cbc(des)))",
2511                                 .cra_driver_name = "echainiv-authenc-"
2512                                                    "hmac-sha224-cbc-des-"
2513                                                    "caam-qi2",
2514                                 .cra_blocksize = DES_BLOCK_SIZE,
2515                         },
2516                         .setkey = aead_setkey,
2517                         .setauthsize = aead_setauthsize,
2518                         .encrypt = aead_encrypt,
2519                         .decrypt = aead_decrypt,
2520                         .ivsize = DES_BLOCK_SIZE,
2521                         .maxauthsize = SHA224_DIGEST_SIZE,
2522                 },
2523                 .caam = {
2524                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2525                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2526                                            OP_ALG_AAI_HMAC_PRECOMP,
2527                         .geniv = true,
2528                 }
2529         },
2530         {
2531                 .aead = {
2532                         .base = {
2533                                 .cra_name = "authenc(hmac(sha256),cbc(des))",
2534                                 .cra_driver_name = "authenc-hmac-sha256-"
2535                                                    "cbc-des-caam-qi2",
2536                                 .cra_blocksize = DES_BLOCK_SIZE,
2537                         },
2538                         .setkey = aead_setkey,
2539                         .setauthsize = aead_setauthsize,
2540                         .encrypt = aead_encrypt,
2541                         .decrypt = aead_decrypt,
2542                         .ivsize = DES_BLOCK_SIZE,
2543                         .maxauthsize = SHA256_DIGEST_SIZE,
2544                 },
2545                 .caam = {
2546                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2547                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2548                                            OP_ALG_AAI_HMAC_PRECOMP,
2549                 },
2550         },
2551         {
2552                 .aead = {
2553                         .base = {
2554                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2555                                             "cbc(des)))",
2556                                 .cra_driver_name = "echainiv-authenc-"
2557                                                    "hmac-sha256-cbc-des-"
2558                                                    "caam-qi2",
2559                                 .cra_blocksize = DES_BLOCK_SIZE,
2560                         },
2561                         .setkey = aead_setkey,
2562                         .setauthsize = aead_setauthsize,
2563                         .encrypt = aead_encrypt,
2564                         .decrypt = aead_decrypt,
2565                         .ivsize = DES_BLOCK_SIZE,
2566                         .maxauthsize = SHA256_DIGEST_SIZE,
2567                 },
2568                 .caam = {
2569                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2570                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2571                                            OP_ALG_AAI_HMAC_PRECOMP,
2572                         .geniv = true,
2573                 },
2574         },
2575         {
2576                 .aead = {
2577                         .base = {
2578                                 .cra_name = "authenc(hmac(sha384),cbc(des))",
2579                                 .cra_driver_name = "authenc-hmac-sha384-"
2580                                                    "cbc-des-caam-qi2",
2581                                 .cra_blocksize = DES_BLOCK_SIZE,
2582                         },
2583                         .setkey = aead_setkey,
2584                         .setauthsize = aead_setauthsize,
2585                         .encrypt = aead_encrypt,
2586                         .decrypt = aead_decrypt,
2587                         .ivsize = DES_BLOCK_SIZE,
2588                         .maxauthsize = SHA384_DIGEST_SIZE,
2589                 },
2590                 .caam = {
2591                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2592                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2593                                            OP_ALG_AAI_HMAC_PRECOMP,
2594                 },
2595         },
2596         {
2597                 .aead = {
2598                         .base = {
2599                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2600                                             "cbc(des)))",
2601                                 .cra_driver_name = "echainiv-authenc-"
2602                                                    "hmac-sha384-cbc-des-"
2603                                                    "caam-qi2",
2604                                 .cra_blocksize = DES_BLOCK_SIZE,
2605                         },
2606                         .setkey = aead_setkey,
2607                         .setauthsize = aead_setauthsize,
2608                         .encrypt = aead_encrypt,
2609                         .decrypt = aead_decrypt,
2610                         .ivsize = DES_BLOCK_SIZE,
2611                         .maxauthsize = SHA384_DIGEST_SIZE,
2612                 },
2613                 .caam = {
2614                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2615                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2616                                            OP_ALG_AAI_HMAC_PRECOMP,
2617                         .geniv = true,
2618                 }
2619         },
2620         {
2621                 .aead = {
2622                         .base = {
2623                                 .cra_name = "authenc(hmac(sha512),cbc(des))",
2624                                 .cra_driver_name = "authenc-hmac-sha512-"
2625                                                    "cbc-des-caam-qi2",
2626                                 .cra_blocksize = DES_BLOCK_SIZE,
2627                         },
2628                         .setkey = aead_setkey,
2629                         .setauthsize = aead_setauthsize,
2630                         .encrypt = aead_encrypt,
2631                         .decrypt = aead_decrypt,
2632                         .ivsize = DES_BLOCK_SIZE,
2633                         .maxauthsize = SHA512_DIGEST_SIZE,
2634                 },
2635                 .caam = {
2636                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2637                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2638                                            OP_ALG_AAI_HMAC_PRECOMP,
2639                 }
2640         },
2641         {
2642                 .aead = {
2643                         .base = {
2644                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2645                                             "cbc(des)))",
2646                                 .cra_driver_name = "echainiv-authenc-"
2647                                                    "hmac-sha512-cbc-des-"
2648                                                    "caam-qi2",
2649                                 .cra_blocksize = DES_BLOCK_SIZE,
2650                         },
2651                         .setkey = aead_setkey,
2652                         .setauthsize = aead_setauthsize,
2653                         .encrypt = aead_encrypt,
2654                         .decrypt = aead_decrypt,
2655                         .ivsize = DES_BLOCK_SIZE,
2656                         .maxauthsize = SHA512_DIGEST_SIZE,
2657                 },
2658                 .caam = {
2659                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2660                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2661                                            OP_ALG_AAI_HMAC_PRECOMP,
2662                         .geniv = true,
2663                 }
2664         },
2665         {
2666                 .aead = {
2667                         .base = {
2668                                 .cra_name = "authenc(hmac(md5),"
2669                                             "rfc3686(ctr(aes)))",
2670                                 .cra_driver_name = "authenc-hmac-md5-"
2671                                                    "rfc3686-ctr-aes-caam-qi2",
2672                                 .cra_blocksize = 1,
2673                         },
2674                         .setkey = aead_setkey,
2675                         .setauthsize = aead_setauthsize,
2676                         .encrypt = aead_encrypt,
2677                         .decrypt = aead_decrypt,
2678                         .ivsize = CTR_RFC3686_IV_SIZE,
2679                         .maxauthsize = MD5_DIGEST_SIZE,
2680                 },
2681                 .caam = {
2682                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2683                                            OP_ALG_AAI_CTR_MOD128,
2684                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2685                                            OP_ALG_AAI_HMAC_PRECOMP,
2686                         .rfc3686 = true,
2687                 },
2688         },
2689         {
2690                 .aead = {
2691                         .base = {
2692                                 .cra_name = "seqiv(authenc("
2693                                             "hmac(md5),rfc3686(ctr(aes))))",
2694                                 .cra_driver_name = "seqiv-authenc-hmac-md5-"
2695                                                    "rfc3686-ctr-aes-caam-qi2",
2696                                 .cra_blocksize = 1,
2697                         },
2698                         .setkey = aead_setkey,
2699                         .setauthsize = aead_setauthsize,
2700                         .encrypt = aead_encrypt,
2701                         .decrypt = aead_decrypt,
2702                         .ivsize = CTR_RFC3686_IV_SIZE,
2703                         .maxauthsize = MD5_DIGEST_SIZE,
2704                 },
2705                 .caam = {
2706                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2707                                            OP_ALG_AAI_CTR_MOD128,
2708                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2709                                            OP_ALG_AAI_HMAC_PRECOMP,
2710                         .rfc3686 = true,
2711                         .geniv = true,
2712                 },
2713         },
2714         {
2715                 .aead = {
2716                         .base = {
2717                                 .cra_name = "authenc(hmac(sha1),"
2718                                             "rfc3686(ctr(aes)))",
2719                                 .cra_driver_name = "authenc-hmac-sha1-"
2720                                                    "rfc3686-ctr-aes-caam-qi2",
2721                                 .cra_blocksize = 1,
2722                         },
2723                         .setkey = aead_setkey,
2724                         .setauthsize = aead_setauthsize,
2725                         .encrypt = aead_encrypt,
2726                         .decrypt = aead_decrypt,
2727                         .ivsize = CTR_RFC3686_IV_SIZE,
2728                         .maxauthsize = SHA1_DIGEST_SIZE,
2729                 },
2730                 .caam = {
2731                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2732                                            OP_ALG_AAI_CTR_MOD128,
2733                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2734                                            OP_ALG_AAI_HMAC_PRECOMP,
2735                         .rfc3686 = true,
2736                 },
2737         },
2738         {
2739                 .aead = {
2740                         .base = {
2741                                 .cra_name = "seqiv(authenc("
2742                                             "hmac(sha1),rfc3686(ctr(aes))))",
2743                                 .cra_driver_name = "seqiv-authenc-hmac-sha1-"
2744                                                    "rfc3686-ctr-aes-caam-qi2",
2745                                 .cra_blocksize = 1,
2746                         },
2747                         .setkey = aead_setkey,
2748                         .setauthsize = aead_setauthsize,
2749                         .encrypt = aead_encrypt,
2750                         .decrypt = aead_decrypt,
2751                         .ivsize = CTR_RFC3686_IV_SIZE,
2752                         .maxauthsize = SHA1_DIGEST_SIZE,
2753                 },
2754                 .caam = {
2755                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2756                                            OP_ALG_AAI_CTR_MOD128,
2757                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2758                                            OP_ALG_AAI_HMAC_PRECOMP,
2759                         .rfc3686 = true,
2760                         .geniv = true,
2761                 },
2762         },
2763         {
2764                 .aead = {
2765                         .base = {
2766                                 .cra_name = "authenc(hmac(sha224),"
2767                                             "rfc3686(ctr(aes)))",
2768                                 .cra_driver_name = "authenc-hmac-sha224-"
2769                                                    "rfc3686-ctr-aes-caam-qi2",
2770                                 .cra_blocksize = 1,
2771                         },
2772                         .setkey = aead_setkey,
2773                         .setauthsize = aead_setauthsize,
2774                         .encrypt = aead_encrypt,
2775                         .decrypt = aead_decrypt,
2776                         .ivsize = CTR_RFC3686_IV_SIZE,
2777                         .maxauthsize = SHA224_DIGEST_SIZE,
2778                 },
2779                 .caam = {
2780                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2781                                            OP_ALG_AAI_CTR_MOD128,
2782                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2783                                            OP_ALG_AAI_HMAC_PRECOMP,
2784                         .rfc3686 = true,
2785                 },
2786         },
2787         {
2788                 .aead = {
2789                         .base = {
2790                                 .cra_name = "seqiv(authenc("
2791                                             "hmac(sha224),rfc3686(ctr(aes))))",
2792                                 .cra_driver_name = "seqiv-authenc-hmac-sha224-"
2793                                                    "rfc3686-ctr-aes-caam-qi2",
2794                                 .cra_blocksize = 1,
2795                         },
2796                         .setkey = aead_setkey,
2797                         .setauthsize = aead_setauthsize,
2798                         .encrypt = aead_encrypt,
2799                         .decrypt = aead_decrypt,
2800                         .ivsize = CTR_RFC3686_IV_SIZE,
2801                         .maxauthsize = SHA224_DIGEST_SIZE,
2802                 },
2803                 .caam = {
2804                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2805                                            OP_ALG_AAI_CTR_MOD128,
2806                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2807                                            OP_ALG_AAI_HMAC_PRECOMP,
2808                         .rfc3686 = true,
2809                         .geniv = true,
2810                 },
2811         },
2812         {
2813                 .aead = {
2814                         .base = {
2815                                 .cra_name = "authenc(hmac(sha256),"
2816                                             "rfc3686(ctr(aes)))",
2817                                 .cra_driver_name = "authenc-hmac-sha256-"
2818                                                    "rfc3686-ctr-aes-caam-qi2",
2819                                 .cra_blocksize = 1,
2820                         },
2821                         .setkey = aead_setkey,
2822                         .setauthsize = aead_setauthsize,
2823                         .encrypt = aead_encrypt,
2824                         .decrypt = aead_decrypt,
2825                         .ivsize = CTR_RFC3686_IV_SIZE,
2826                         .maxauthsize = SHA256_DIGEST_SIZE,
2827                 },
2828                 .caam = {
2829                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2830                                            OP_ALG_AAI_CTR_MOD128,
2831                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2832                                            OP_ALG_AAI_HMAC_PRECOMP,
2833                         .rfc3686 = true,
2834                 },
2835         },
2836         {
2837                 .aead = {
2838                         .base = {
2839                                 .cra_name = "seqiv(authenc(hmac(sha256),"
2840                                             "rfc3686(ctr(aes))))",
2841                                 .cra_driver_name = "seqiv-authenc-hmac-sha256-"
2842                                                    "rfc3686-ctr-aes-caam-qi2",
2843                                 .cra_blocksize = 1,
2844                         },
2845                         .setkey = aead_setkey,
2846                         .setauthsize = aead_setauthsize,
2847                         .encrypt = aead_encrypt,
2848                         .decrypt = aead_decrypt,
2849                         .ivsize = CTR_RFC3686_IV_SIZE,
2850                         .maxauthsize = SHA256_DIGEST_SIZE,
2851                 },
2852                 .caam = {
2853                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2854                                            OP_ALG_AAI_CTR_MOD128,
2855                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2856                                            OP_ALG_AAI_HMAC_PRECOMP,
2857                         .rfc3686 = true,
2858                         .geniv = true,
2859                 },
2860         },
2861         {
2862                 .aead = {
2863                         .base = {
2864                                 .cra_name = "authenc(hmac(sha384),"
2865                                             "rfc3686(ctr(aes)))",
2866                                 .cra_driver_name = "authenc-hmac-sha384-"
2867                                                    "rfc3686-ctr-aes-caam-qi2",
2868                                 .cra_blocksize = 1,
2869                         },
2870                         .setkey = aead_setkey,
2871                         .setauthsize = aead_setauthsize,
2872                         .encrypt = aead_encrypt,
2873                         .decrypt = aead_decrypt,
2874                         .ivsize = CTR_RFC3686_IV_SIZE,
2875                         .maxauthsize = SHA384_DIGEST_SIZE,
2876                 },
2877                 .caam = {
2878                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2879                                            OP_ALG_AAI_CTR_MOD128,
2880                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2881                                            OP_ALG_AAI_HMAC_PRECOMP,
2882                         .rfc3686 = true,
2883                 },
2884         },
2885         {
2886                 .aead = {
2887                         .base = {
2888                                 .cra_name = "seqiv(authenc(hmac(sha384),"
2889                                             "rfc3686(ctr(aes))))",
2890                                 .cra_driver_name = "seqiv-authenc-hmac-sha384-"
2891                                                    "rfc3686-ctr-aes-caam-qi2",
2892                                 .cra_blocksize = 1,
2893                         },
2894                         .setkey = aead_setkey,
2895                         .setauthsize = aead_setauthsize,
2896                         .encrypt = aead_encrypt,
2897                         .decrypt = aead_decrypt,
2898                         .ivsize = CTR_RFC3686_IV_SIZE,
2899                         .maxauthsize = SHA384_DIGEST_SIZE,
2900                 },
2901                 .caam = {
2902                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2903                                            OP_ALG_AAI_CTR_MOD128,
2904                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2905                                            OP_ALG_AAI_HMAC_PRECOMP,
2906                         .rfc3686 = true,
2907                         .geniv = true,
2908                 },
2909         },
2910         {
2911                 .aead = {
2912                         .base = {
2913                                 .cra_name = "rfc7539(chacha20,poly1305)",
2914                                 .cra_driver_name = "rfc7539-chacha20-poly1305-"
2915                                                    "caam-qi2",
2916                                 .cra_blocksize = 1,
2917                         },
2918                         .setkey = chachapoly_setkey,
2919                         .setauthsize = chachapoly_setauthsize,
2920                         .encrypt = aead_encrypt,
2921                         .decrypt = aead_decrypt,
2922                         .ivsize = CHACHAPOLY_IV_SIZE,
2923                         .maxauthsize = POLY1305_DIGEST_SIZE,
2924                 },
2925                 .caam = {
2926                         .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2927                                            OP_ALG_AAI_AEAD,
2928                         .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2929                                            OP_ALG_AAI_AEAD,
2930                         .nodkp = true,
2931                 },
2932         },
2933         {
2934                 .aead = {
2935                         .base = {
2936                                 .cra_name = "rfc7539esp(chacha20,poly1305)",
2937                                 .cra_driver_name = "rfc7539esp-chacha20-"
2938                                                    "poly1305-caam-qi2",
2939                                 .cra_blocksize = 1,
2940                         },
2941                         .setkey = chachapoly_setkey,
2942                         .setauthsize = chachapoly_setauthsize,
2943                         .encrypt = aead_encrypt,
2944                         .decrypt = aead_decrypt,
2945                         .ivsize = 8,
2946                         .maxauthsize = POLY1305_DIGEST_SIZE,
2947                 },
2948                 .caam = {
2949                         .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2950                                            OP_ALG_AAI_AEAD,
2951                         .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2952                                            OP_ALG_AAI_AEAD,
2953                         .nodkp = true,
2954                 },
2955         },
2956         {
2957                 .aead = {
2958                         .base = {
2959                                 .cra_name = "authenc(hmac(sha512),"
2960                                             "rfc3686(ctr(aes)))",
2961                                 .cra_driver_name = "authenc-hmac-sha512-"
2962                                                    "rfc3686-ctr-aes-caam-qi2",
2963                                 .cra_blocksize = 1,
2964                         },
2965                         .setkey = aead_setkey,
2966                         .setauthsize = aead_setauthsize,
2967                         .encrypt = aead_encrypt,
2968                         .decrypt = aead_decrypt,
2969                         .ivsize = CTR_RFC3686_IV_SIZE,
2970                         .maxauthsize = SHA512_DIGEST_SIZE,
2971                 },
2972                 .caam = {
2973                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2974                                            OP_ALG_AAI_CTR_MOD128,
2975                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2976                                            OP_ALG_AAI_HMAC_PRECOMP,
2977                         .rfc3686 = true,
2978                 },
2979         },
2980         {
2981                 .aead = {
2982                         .base = {
2983                                 .cra_name = "seqiv(authenc(hmac(sha512),"
2984                                             "rfc3686(ctr(aes))))",
2985                                 .cra_driver_name = "seqiv-authenc-hmac-sha512-"
2986                                                    "rfc3686-ctr-aes-caam-qi2",
2987                                 .cra_blocksize = 1,
2988                         },
2989                         .setkey = aead_setkey,
2990                         .setauthsize = aead_setauthsize,
2991                         .encrypt = aead_encrypt,
2992                         .decrypt = aead_decrypt,
2993                         .ivsize = CTR_RFC3686_IV_SIZE,
2994                         .maxauthsize = SHA512_DIGEST_SIZE,
2995                 },
2996                 .caam = {
2997                         .class1_alg_type = OP_ALG_ALGSEL_AES |
2998                                            OP_ALG_AAI_CTR_MOD128,
2999                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3000                                            OP_ALG_AAI_HMAC_PRECOMP,
3001                         .rfc3686 = true,
3002                         .geniv = true,
3003                 },
3004         },
3005 };
3006
3007 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
3008 {
3009         struct skcipher_alg *alg = &t_alg->skcipher;
3010
3011         alg->base.cra_module = THIS_MODULE;
3012         alg->base.cra_priority = CAAM_CRA_PRIORITY;
3013         alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding();
3014         alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3015                               CRYPTO_ALG_KERN_DRIVER_ONLY);
3016
3017         alg->init = caam_cra_init_skcipher;
3018         alg->exit = caam_cra_exit;
3019 }
3020
3021 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
3022 {
3023         struct aead_alg *alg = &t_alg->aead;
3024
3025         alg->base.cra_module = THIS_MODULE;
3026         alg->base.cra_priority = CAAM_CRA_PRIORITY;
3027         alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding();
3028         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3029                               CRYPTO_ALG_KERN_DRIVER_ONLY;
3030
3031         alg->init = caam_cra_init_aead;
3032         alg->exit = caam_cra_exit_aead;
3033 }
3034
3035 /* max hash key is max split key size */
3036 #define CAAM_MAX_HASH_KEY_SIZE          (SHA512_DIGEST_SIZE * 2)
3037
3038 #define CAAM_MAX_HASH_BLOCK_SIZE        SHA512_BLOCK_SIZE
3039
3040 /* caam context sizes for hashes: running digest + 8 */
3041 #define HASH_MSG_LEN                    8
3042 #define MAX_CTX_LEN                     (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
3043
3044 enum hash_optype {
3045         UPDATE = 0,
3046         UPDATE_FIRST,
3047         FINALIZE,
3048         DIGEST,
3049         HASH_NUM_OP
3050 };
3051
3052 /**
3053  * struct caam_hash_ctx - ahash per-session context
3054  * @flc: Flow Contexts array
3055  * @key: authentication key
3056  * @flc_dma: I/O virtual addresses of the Flow Contexts
3057  * @dev: dpseci device
3058  * @ctx_len: size of Context Register
3059  * @adata: hashing algorithm details
3060  */
3061 struct caam_hash_ctx {
3062         struct caam_flc flc[HASH_NUM_OP];
3063         u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
3064         dma_addr_t flc_dma[HASH_NUM_OP];
3065         struct device *dev;
3066         int ctx_len;
3067         struct alginfo adata;
3068 };
3069
3070 /* ahash state */
3071 struct caam_hash_state {
3072         struct caam_request caam_req;
3073         dma_addr_t buf_dma;
3074         dma_addr_t ctx_dma;
3075         int ctx_dma_len;
3076         u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
3077         int buflen;
3078         int next_buflen;
3079         u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
3080         int (*update)(struct ahash_request *req);
3081         int (*final)(struct ahash_request *req);
3082         int (*finup)(struct ahash_request *req);
3083 };
3084
3085 struct caam_export_state {
3086         u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
3087         u8 caam_ctx[MAX_CTX_LEN];
3088         int buflen;
3089         int (*update)(struct ahash_request *req);
3090         int (*final)(struct ahash_request *req);
3091         int (*finup)(struct ahash_request *req);
3092 };
3093
3094 /* Map current buffer in state (if length > 0) and put it in link table */
3095 static inline int buf_map_to_qm_sg(struct device *dev,
3096                                    struct dpaa2_sg_entry *qm_sg,
3097                                    struct caam_hash_state *state)
3098 {
3099         int buflen = state->buflen;
3100
3101         if (!buflen)
3102                 return 0;
3103
3104         state->buf_dma = dma_map_single(dev, state->buf, buflen,
3105                                         DMA_TO_DEVICE);
3106         if (dma_mapping_error(dev, state->buf_dma)) {
3107                 dev_err(dev, "unable to map buf\n");
3108                 state->buf_dma = 0;
3109                 return -ENOMEM;
3110         }
3111
3112         dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0);
3113
3114         return 0;
3115 }
3116
3117 /* Map state->caam_ctx, and add it to link table */
3118 static inline int ctx_map_to_qm_sg(struct device *dev,
3119                                    struct caam_hash_state *state, int ctx_len,
3120                                    struct dpaa2_sg_entry *qm_sg, u32 flag)
3121 {
3122         state->ctx_dma_len = ctx_len;
3123         state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag);
3124         if (dma_mapping_error(dev, state->ctx_dma)) {
3125                 dev_err(dev, "unable to map ctx\n");
3126                 state->ctx_dma = 0;
3127                 return -ENOMEM;
3128         }
3129
3130         dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0);
3131
3132         return 0;
3133 }
3134
3135 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
3136 {
3137         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3138         int digestsize = crypto_ahash_digestsize(ahash);
3139         struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
3140         struct caam_flc *flc;
3141         u32 *desc;
3142
3143         /* ahash_update shared descriptor */
3144         flc = &ctx->flc[UPDATE];
3145         desc = flc->sh_desc;
3146         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
3147                           ctx->ctx_len, true, priv->sec_attr.era);
3148         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3149         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE],
3150                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3151         print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ",
3152                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3153                              1);
3154
3155         /* ahash_update_first shared descriptor */
3156         flc = &ctx->flc[UPDATE_FIRST];
3157         desc = flc->sh_desc;
3158         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
3159                           ctx->ctx_len, false, priv->sec_attr.era);
3160         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3161         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST],
3162                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3163         print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ",
3164                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3165                              1);
3166
3167         /* ahash_final shared descriptor */
3168         flc = &ctx->flc[FINALIZE];
3169         desc = flc->sh_desc;
3170         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
3171                           ctx->ctx_len, true, priv->sec_attr.era);
3172         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3173         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE],
3174                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3175         print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ",
3176                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3177                              1);
3178
3179         /* ahash_digest shared descriptor */
3180         flc = &ctx->flc[DIGEST];
3181         desc = flc->sh_desc;
3182         cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
3183                           ctx->ctx_len, false, priv->sec_attr.era);
3184         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3185         dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST],
3186                                    desc_bytes(desc), DMA_BIDIRECTIONAL);
3187         print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ",
3188                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3189                              1);
3190
3191         return 0;
3192 }
3193
3194 struct split_key_sh_result {
3195         struct completion completion;
3196         int err;
3197         struct device *dev;
3198 };
3199
3200 static void split_key_sh_done(void *cbk_ctx, u32 err)
3201 {
3202         struct split_key_sh_result *res = cbk_ctx;
3203
3204         dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
3205
3206         res->err = err ? caam_qi2_strstatus(res->dev, err) : 0;
3207         complete(&res->completion);
3208 }
3209
3210 /* Digest hash size if it is too large */
3211 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
3212                            u32 digestsize)
3213 {
3214         struct caam_request *req_ctx;
3215         u32 *desc;
3216         struct split_key_sh_result result;
3217         dma_addr_t key_dma;
3218         struct caam_flc *flc;
3219         dma_addr_t flc_dma;
3220         int ret = -ENOMEM;
3221         struct dpaa2_fl_entry *in_fle, *out_fle;
3222
3223         req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA);
3224         if (!req_ctx)
3225                 return -ENOMEM;
3226
3227         in_fle = &req_ctx->fd_flt[1];
3228         out_fle = &req_ctx->fd_flt[0];
3229
3230         flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA);
3231         if (!flc)
3232                 goto err_flc;
3233
3234         key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL);
3235         if (dma_mapping_error(ctx->dev, key_dma)) {
3236                 dev_err(ctx->dev, "unable to map key memory\n");
3237                 goto err_key_dma;
3238         }
3239
3240         desc = flc->sh_desc;
3241
3242         init_sh_desc(desc, 0);
3243
3244         /* descriptor to perform unkeyed hash on key_in */
3245         append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
3246                          OP_ALG_AS_INITFINAL);
3247         append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
3248                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
3249         append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
3250                          LDST_SRCDST_BYTE_CONTEXT);
3251
3252         flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3253         flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) +
3254                                  desc_bytes(desc), DMA_TO_DEVICE);
3255         if (dma_mapping_error(ctx->dev, flc_dma)) {
3256                 dev_err(ctx->dev, "unable to map shared descriptor\n");
3257                 goto err_flc_dma;
3258         }
3259
3260         dpaa2_fl_set_final(in_fle, true);
3261         dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3262         dpaa2_fl_set_addr(in_fle, key_dma);
3263         dpaa2_fl_set_len(in_fle, *keylen);
3264         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3265         dpaa2_fl_set_addr(out_fle, key_dma);
3266         dpaa2_fl_set_len(out_fle, digestsize);
3267
3268         print_hex_dump_debug("key_in@" __stringify(__LINE__)": ",
3269                              DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
3270         print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ",
3271                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3272                              1);
3273
3274         result.err = 0;
3275         init_completion(&result.completion);
3276         result.dev = ctx->dev;
3277
3278         req_ctx->flc = flc;
3279         req_ctx->flc_dma = flc_dma;
3280         req_ctx->cbk = split_key_sh_done;
3281         req_ctx->ctx = &result;
3282
3283         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3284         if (ret == -EINPROGRESS) {
3285                 /* in progress */
3286                 wait_for_completion(&result.completion);
3287                 ret = result.err;
3288                 print_hex_dump_debug("digested key@" __stringify(__LINE__)": ",
3289                                      DUMP_PREFIX_ADDRESS, 16, 4, key,
3290                                      digestsize, 1);
3291         }
3292
3293         dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc),
3294                          DMA_TO_DEVICE);
3295 err_flc_dma:
3296         dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL);
3297 err_key_dma:
3298         kfree(flc);
3299 err_flc:
3300         kfree(req_ctx);
3301
3302         *keylen = digestsize;
3303
3304         return ret;
3305 }
3306
3307 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
3308                         unsigned int keylen)
3309 {
3310         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3311         unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
3312         unsigned int digestsize = crypto_ahash_digestsize(ahash);
3313         int ret;
3314         u8 *hashed_key = NULL;
3315
3316         dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize);
3317
3318         if (keylen > blocksize) {
3319                 hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
3320                 if (!hashed_key)
3321                         return -ENOMEM;
3322                 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
3323                 if (ret)
3324                         goto bad_free_key;
3325                 key = hashed_key;
3326         }
3327
3328         ctx->adata.keylen = keylen;
3329         ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
3330                                               OP_ALG_ALGSEL_MASK);
3331         if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
3332                 goto bad_free_key;
3333
3334         ctx->adata.key_virt = key;
3335         ctx->adata.key_inline = true;
3336
3337         /*
3338          * In case |user key| > |derived key|, using DKP<imm,imm> would result
3339          * in invalid opcodes (last bytes of user key) in the resulting
3340          * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
3341          * addresses are needed.
3342          */
3343         if (keylen > ctx->adata.keylen_pad) {
3344                 memcpy(ctx->key, key, keylen);
3345                 dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma,
3346                                            ctx->adata.keylen_pad,
3347                                            DMA_TO_DEVICE);
3348         }
3349
3350         ret = ahash_set_sh_desc(ahash);
3351         kfree(hashed_key);
3352         return ret;
3353 bad_free_key:
3354         kfree(hashed_key);
3355         return -EINVAL;
3356 }
3357
3358 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc,
3359                                struct ahash_request *req)
3360 {
3361         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3362
3363         if (edesc->src_nents)
3364                 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
3365
3366         if (edesc->qm_sg_bytes)
3367                 dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes,
3368                                  DMA_TO_DEVICE);
3369
3370         if (state->buf_dma) {
3371                 dma_unmap_single(dev, state->buf_dma, state->buflen,
3372                                  DMA_TO_DEVICE);
3373                 state->buf_dma = 0;
3374         }
3375 }
3376
3377 static inline void ahash_unmap_ctx(struct device *dev,
3378                                    struct ahash_edesc *edesc,
3379                                    struct ahash_request *req, u32 flag)
3380 {
3381         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3382
3383         if (state->ctx_dma) {
3384                 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag);
3385                 state->ctx_dma = 0;
3386         }
3387         ahash_unmap(dev, edesc, req);
3388 }
3389
3390 static void ahash_done(void *cbk_ctx, u32 status)
3391 {
3392         struct crypto_async_request *areq = cbk_ctx;
3393         struct ahash_request *req = ahash_request_cast(areq);
3394         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3395         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3396         struct ahash_edesc *edesc = state->caam_req.edesc;
3397         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3398         int digestsize = crypto_ahash_digestsize(ahash);
3399         int ecode = 0;
3400
3401         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3402
3403         if (unlikely(status))
3404                 ecode = caam_qi2_strstatus(ctx->dev, status);
3405
3406         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3407         memcpy(req->result, state->caam_ctx, digestsize);
3408         qi_cache_free(edesc);
3409
3410         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3411                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3412                              ctx->ctx_len, 1);
3413
3414         req->base.complete(&req->base, ecode);
3415 }
3416
3417 static void ahash_done_bi(void *cbk_ctx, u32 status)
3418 {
3419         struct crypto_async_request *areq = cbk_ctx;
3420         struct ahash_request *req = ahash_request_cast(areq);
3421         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3422         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3423         struct ahash_edesc *edesc = state->caam_req.edesc;
3424         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3425         int ecode = 0;
3426
3427         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3428
3429         if (unlikely(status))
3430                 ecode = caam_qi2_strstatus(ctx->dev, status);
3431
3432         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3433         qi_cache_free(edesc);
3434
3435         scatterwalk_map_and_copy(state->buf, req->src,
3436                                  req->nbytes - state->next_buflen,
3437                                  state->next_buflen, 0);
3438         state->buflen = state->next_buflen;
3439
3440         print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3441                              DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3442                              state->buflen, 1);
3443
3444         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3445                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3446                              ctx->ctx_len, 1);
3447         if (req->result)
3448                 print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3449                                      DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3450                                      crypto_ahash_digestsize(ahash), 1);
3451
3452         req->base.complete(&req->base, ecode);
3453 }
3454
3455 static void ahash_done_ctx_src(void *cbk_ctx, u32 status)
3456 {
3457         struct crypto_async_request *areq = cbk_ctx;
3458         struct ahash_request *req = ahash_request_cast(areq);
3459         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3460         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3461         struct ahash_edesc *edesc = state->caam_req.edesc;
3462         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3463         int digestsize = crypto_ahash_digestsize(ahash);
3464         int ecode = 0;
3465
3466         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3467
3468         if (unlikely(status))
3469                 ecode = caam_qi2_strstatus(ctx->dev, status);
3470
3471         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3472         memcpy(req->result, state->caam_ctx, digestsize);
3473         qi_cache_free(edesc);
3474
3475         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3476                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3477                              ctx->ctx_len, 1);
3478
3479         req->base.complete(&req->base, ecode);
3480 }
3481
3482 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status)
3483 {
3484         struct crypto_async_request *areq = cbk_ctx;
3485         struct ahash_request *req = ahash_request_cast(areq);
3486         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3487         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3488         struct ahash_edesc *edesc = state->caam_req.edesc;
3489         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3490         int ecode = 0;
3491
3492         dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3493
3494         if (unlikely(status))
3495                 ecode = caam_qi2_strstatus(ctx->dev, status);
3496
3497         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3498         qi_cache_free(edesc);
3499
3500         scatterwalk_map_and_copy(state->buf, req->src,
3501                                  req->nbytes - state->next_buflen,
3502                                  state->next_buflen, 0);
3503         state->buflen = state->next_buflen;
3504
3505         print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3506                              DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3507                              state->buflen, 1);
3508
3509         print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3510                              DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3511                              ctx->ctx_len, 1);
3512         if (req->result)
3513                 print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3514                                      DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3515                                      crypto_ahash_digestsize(ahash), 1);
3516
3517         req->base.complete(&req->base, ecode);
3518 }
3519
3520 static int ahash_update_ctx(struct ahash_request *req)
3521 {
3522         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3523         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3524         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3525         struct caam_request *req_ctx = &state->caam_req;
3526         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3527         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3528         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3529                       GFP_KERNEL : GFP_ATOMIC;
3530         u8 *buf = state->buf;
3531         int *buflen = &state->buflen;
3532         int *next_buflen = &state->next_buflen;
3533         int in_len = *buflen + req->nbytes, to_hash;
3534         int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index;
3535         struct ahash_edesc *edesc;
3536         int ret = 0;
3537
3538         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3539         to_hash = in_len - *next_buflen;
3540
3541         if (to_hash) {
3542                 struct dpaa2_sg_entry *sg_table;
3543                 int src_len = req->nbytes - *next_buflen;
3544
3545                 src_nents = sg_nents_for_len(req->src, src_len);
3546                 if (src_nents < 0) {
3547                         dev_err(ctx->dev, "Invalid number of src SG.\n");
3548                         return src_nents;
3549                 }
3550
3551                 if (src_nents) {
3552                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3553                                                   DMA_TO_DEVICE);
3554                         if (!mapped_nents) {
3555                                 dev_err(ctx->dev, "unable to DMA map source\n");
3556                                 return -ENOMEM;
3557                         }
3558                 } else {
3559                         mapped_nents = 0;
3560                 }
3561
3562                 /* allocate space for base edesc and link tables */
3563                 edesc = qi_cache_zalloc(GFP_DMA | flags);
3564                 if (!edesc) {
3565                         dma_unmap_sg(ctx->dev, req->src, src_nents,
3566                                      DMA_TO_DEVICE);
3567                         return -ENOMEM;
3568                 }
3569
3570                 edesc->src_nents = src_nents;
3571                 qm_sg_src_index = 1 + (*buflen ? 1 : 0);
3572                 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3573                               sizeof(*sg_table);
3574                 sg_table = &edesc->sgt[0];
3575
3576                 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3577                                        DMA_BIDIRECTIONAL);
3578                 if (ret)
3579                         goto unmap_ctx;
3580
3581                 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3582                 if (ret)
3583                         goto unmap_ctx;
3584
3585                 if (mapped_nents) {
3586                         sg_to_qm_sg_last(req->src, src_len,
3587                                          sg_table + qm_sg_src_index, 0);
3588                 } else {
3589                         dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1,
3590                                            true);
3591                 }
3592
3593                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3594                                                   qm_sg_bytes, DMA_TO_DEVICE);
3595                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3596                         dev_err(ctx->dev, "unable to map S/G table\n");
3597                         ret = -ENOMEM;
3598                         goto unmap_ctx;
3599                 }
3600                 edesc->qm_sg_bytes = qm_sg_bytes;
3601
3602                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3603                 dpaa2_fl_set_final(in_fle, true);
3604                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3605                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3606                 dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash);
3607                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3608                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3609                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
3610
3611                 req_ctx->flc = &ctx->flc[UPDATE];
3612                 req_ctx->flc_dma = ctx->flc_dma[UPDATE];
3613                 req_ctx->cbk = ahash_done_bi;
3614                 req_ctx->ctx = &req->base;
3615                 req_ctx->edesc = edesc;
3616
3617                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3618                 if (ret != -EINPROGRESS &&
3619                     !(ret == -EBUSY &&
3620                       req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3621                         goto unmap_ctx;
3622         } else if (*next_buflen) {
3623                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
3624                                          req->nbytes, 0);
3625                 *buflen = *next_buflen;
3626
3627                 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3628                                      DUMP_PREFIX_ADDRESS, 16, 4, buf,
3629                                      *buflen, 1);
3630         }
3631
3632         return ret;
3633 unmap_ctx:
3634         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3635         qi_cache_free(edesc);
3636         return ret;
3637 }
3638
3639 static int ahash_final_ctx(struct ahash_request *req)
3640 {
3641         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3642         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3643         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3644         struct caam_request *req_ctx = &state->caam_req;
3645         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3646         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3647         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3648                       GFP_KERNEL : GFP_ATOMIC;
3649         int buflen = state->buflen;
3650         int qm_sg_bytes;
3651         int digestsize = crypto_ahash_digestsize(ahash);
3652         struct ahash_edesc *edesc;
3653         struct dpaa2_sg_entry *sg_table;
3654         int ret;
3655
3656         /* allocate space for base edesc and link tables */
3657         edesc = qi_cache_zalloc(GFP_DMA | flags);
3658         if (!edesc)
3659                 return -ENOMEM;
3660
3661         qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table);
3662         sg_table = &edesc->sgt[0];
3663
3664         ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3665                                DMA_BIDIRECTIONAL);
3666         if (ret)
3667                 goto unmap_ctx;
3668
3669         ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3670         if (ret)
3671                 goto unmap_ctx;
3672
3673         dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true);
3674
3675         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3676                                           DMA_TO_DEVICE);
3677         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3678                 dev_err(ctx->dev, "unable to map S/G table\n");
3679                 ret = -ENOMEM;
3680                 goto unmap_ctx;
3681         }
3682         edesc->qm_sg_bytes = qm_sg_bytes;
3683
3684         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3685         dpaa2_fl_set_final(in_fle, true);
3686         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3687         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3688         dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen);
3689         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3690         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3691         dpaa2_fl_set_len(out_fle, digestsize);
3692
3693         req_ctx->flc = &ctx->flc[FINALIZE];
3694         req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3695         req_ctx->cbk = ahash_done_ctx_src;
3696         req_ctx->ctx = &req->base;
3697         req_ctx->edesc = edesc;
3698
3699         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3700         if (ret == -EINPROGRESS ||
3701             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3702                 return ret;
3703
3704 unmap_ctx:
3705         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3706         qi_cache_free(edesc);
3707         return ret;
3708 }
3709
3710 static int ahash_finup_ctx(struct ahash_request *req)
3711 {
3712         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3713         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3714         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3715         struct caam_request *req_ctx = &state->caam_req;
3716         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3717         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3718         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3719                       GFP_KERNEL : GFP_ATOMIC;
3720         int buflen = state->buflen;
3721         int qm_sg_bytes, qm_sg_src_index;
3722         int src_nents, mapped_nents;
3723         int digestsize = crypto_ahash_digestsize(ahash);
3724         struct ahash_edesc *edesc;
3725         struct dpaa2_sg_entry *sg_table;
3726         int ret;
3727
3728         src_nents = sg_nents_for_len(req->src, req->nbytes);
3729         if (src_nents < 0) {
3730                 dev_err(ctx->dev, "Invalid number of src SG.\n");
3731                 return src_nents;
3732         }
3733
3734         if (src_nents) {
3735                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3736                                           DMA_TO_DEVICE);
3737                 if (!mapped_nents) {
3738                         dev_err(ctx->dev, "unable to DMA map source\n");
3739                         return -ENOMEM;
3740                 }
3741         } else {
3742                 mapped_nents = 0;
3743         }
3744
3745         /* allocate space for base edesc and link tables */
3746         edesc = qi_cache_zalloc(GFP_DMA | flags);
3747         if (!edesc) {
3748                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3749                 return -ENOMEM;
3750         }
3751
3752         edesc->src_nents = src_nents;
3753         qm_sg_src_index = 1 + (buflen ? 1 : 0);
3754         qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3755                       sizeof(*sg_table);
3756         sg_table = &edesc->sgt[0];
3757
3758         ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3759                                DMA_BIDIRECTIONAL);
3760         if (ret)
3761                 goto unmap_ctx;
3762
3763         ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3764         if (ret)
3765                 goto unmap_ctx;
3766
3767         sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0);
3768
3769         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3770                                           DMA_TO_DEVICE);
3771         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3772                 dev_err(ctx->dev, "unable to map S/G table\n");
3773                 ret = -ENOMEM;
3774                 goto unmap_ctx;
3775         }
3776         edesc->qm_sg_bytes = qm_sg_bytes;
3777
3778         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3779         dpaa2_fl_set_final(in_fle, true);
3780         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3781         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3782         dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes);
3783         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3784         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3785         dpaa2_fl_set_len(out_fle, digestsize);
3786
3787         req_ctx->flc = &ctx->flc[FINALIZE];
3788         req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3789         req_ctx->cbk = ahash_done_ctx_src;
3790         req_ctx->ctx = &req->base;
3791         req_ctx->edesc = edesc;
3792
3793         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3794         if (ret == -EINPROGRESS ||
3795             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3796                 return ret;
3797
3798 unmap_ctx:
3799         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3800         qi_cache_free(edesc);
3801         return ret;
3802 }
3803
3804 static int ahash_digest(struct ahash_request *req)
3805 {
3806         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3807         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3808         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3809         struct caam_request *req_ctx = &state->caam_req;
3810         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3811         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3812         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3813                       GFP_KERNEL : GFP_ATOMIC;
3814         int digestsize = crypto_ahash_digestsize(ahash);
3815         int src_nents, mapped_nents;
3816         struct ahash_edesc *edesc;
3817         int ret = -ENOMEM;
3818
3819         state->buf_dma = 0;
3820
3821         src_nents = sg_nents_for_len(req->src, req->nbytes);
3822         if (src_nents < 0) {
3823                 dev_err(ctx->dev, "Invalid number of src SG.\n");
3824                 return src_nents;
3825         }
3826
3827         if (src_nents) {
3828                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3829                                           DMA_TO_DEVICE);
3830                 if (!mapped_nents) {
3831                         dev_err(ctx->dev, "unable to map source for DMA\n");
3832                         return ret;
3833                 }
3834         } else {
3835                 mapped_nents = 0;
3836         }
3837
3838         /* allocate space for base edesc and link tables */
3839         edesc = qi_cache_zalloc(GFP_DMA | flags);
3840         if (!edesc) {
3841                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3842                 return ret;
3843         }
3844
3845         edesc->src_nents = src_nents;
3846         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3847
3848         if (mapped_nents > 1) {
3849                 int qm_sg_bytes;
3850                 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0];
3851
3852                 qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table);
3853                 sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0);
3854                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3855                                                   qm_sg_bytes, DMA_TO_DEVICE);
3856                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3857                         dev_err(ctx->dev, "unable to map S/G table\n");
3858                         goto unmap;
3859                 }
3860                 edesc->qm_sg_bytes = qm_sg_bytes;
3861                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3862                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3863         } else {
3864                 dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3865                 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
3866         }
3867
3868         state->ctx_dma_len = digestsize;
3869         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3870                                         DMA_FROM_DEVICE);
3871         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3872                 dev_err(ctx->dev, "unable to map ctx\n");
3873                 state->ctx_dma = 0;
3874                 goto unmap;
3875         }
3876
3877         dpaa2_fl_set_final(in_fle, true);
3878         dpaa2_fl_set_len(in_fle, req->nbytes);
3879         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3880         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3881         dpaa2_fl_set_len(out_fle, digestsize);
3882
3883         req_ctx->flc = &ctx->flc[DIGEST];
3884         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3885         req_ctx->cbk = ahash_done;
3886         req_ctx->ctx = &req->base;
3887         req_ctx->edesc = edesc;
3888         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3889         if (ret == -EINPROGRESS ||
3890             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3891                 return ret;
3892
3893 unmap:
3894         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3895         qi_cache_free(edesc);
3896         return ret;
3897 }
3898
3899 static int ahash_final_no_ctx(struct ahash_request *req)
3900 {
3901         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3902         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3903         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3904         struct caam_request *req_ctx = &state->caam_req;
3905         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3906         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3907         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3908                       GFP_KERNEL : GFP_ATOMIC;
3909         u8 *buf = state->buf;
3910         int buflen = state->buflen;
3911         int digestsize = crypto_ahash_digestsize(ahash);
3912         struct ahash_edesc *edesc;
3913         int ret = -ENOMEM;
3914
3915         /* allocate space for base edesc and link tables */
3916         edesc = qi_cache_zalloc(GFP_DMA | flags);
3917         if (!edesc)
3918                 return ret;
3919
3920         if (buflen) {
3921                 state->buf_dma = dma_map_single(ctx->dev, buf, buflen,
3922                                                 DMA_TO_DEVICE);
3923                 if (dma_mapping_error(ctx->dev, state->buf_dma)) {
3924                         dev_err(ctx->dev, "unable to map src\n");
3925                         goto unmap;
3926                 }
3927         }
3928
3929         state->ctx_dma_len = digestsize;
3930         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3931                                         DMA_FROM_DEVICE);
3932         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3933                 dev_err(ctx->dev, "unable to map ctx\n");
3934                 state->ctx_dma = 0;
3935                 goto unmap;
3936         }
3937
3938         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3939         dpaa2_fl_set_final(in_fle, true);
3940         /*
3941          * crypto engine requires the input entry to be present when
3942          * "frame list" FD is used.
3943          * Since engine does not support FMT=2'b11 (unused entry type), leaving
3944          * in_fle zeroized (except for "Final" flag) is the best option.
3945          */
3946         if (buflen) {
3947                 dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3948                 dpaa2_fl_set_addr(in_fle, state->buf_dma);
3949                 dpaa2_fl_set_len(in_fle, buflen);
3950         }
3951         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3952         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3953         dpaa2_fl_set_len(out_fle, digestsize);
3954
3955         req_ctx->flc = &ctx->flc[DIGEST];
3956         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3957         req_ctx->cbk = ahash_done;
3958         req_ctx->ctx = &req->base;
3959         req_ctx->edesc = edesc;
3960
3961         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3962         if (ret == -EINPROGRESS ||
3963             (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3964                 return ret;
3965
3966 unmap:
3967         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3968         qi_cache_free(edesc);
3969         return ret;
3970 }
3971
3972 static int ahash_update_no_ctx(struct ahash_request *req)
3973 {
3974         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3975         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3976         struct caam_hash_state *state = ahash_request_ctx_dma(req);
3977         struct caam_request *req_ctx = &state->caam_req;
3978         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3979         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3980         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3981                       GFP_KERNEL : GFP_ATOMIC;
3982         u8 *buf = state->buf;
3983         int *buflen = &state->buflen;
3984         int *next_buflen = &state->next_buflen;
3985         int in_len = *buflen + req->nbytes, to_hash;
3986         int qm_sg_bytes, src_nents, mapped_nents;
3987         struct ahash_edesc *edesc;
3988         int ret = 0;
3989
3990         *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3991         to_hash = in_len - *next_buflen;
3992
3993         if (to_hash) {
3994                 struct dpaa2_sg_entry *sg_table;
3995                 int src_len = req->nbytes - *next_buflen;
3996
3997                 src_nents = sg_nents_for_len(req->src, src_len);
3998                 if (src_nents < 0) {
3999                         dev_err(ctx->dev, "Invalid number of src SG.\n");
4000                         return src_nents;
4001                 }
4002
4003                 if (src_nents) {
4004                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4005                                                   DMA_TO_DEVICE);
4006                         if (!mapped_nents) {
4007                                 dev_err(ctx->dev, "unable to DMA map source\n");
4008                                 return -ENOMEM;
4009                         }
4010                 } else {
4011                         mapped_nents = 0;
4012                 }
4013
4014                 /* allocate space for base edesc and link tables */
4015                 edesc = qi_cache_zalloc(GFP_DMA | flags);
4016                 if (!edesc) {
4017                         dma_unmap_sg(ctx->dev, req->src, src_nents,
4018                                      DMA_TO_DEVICE);
4019                         return -ENOMEM;
4020                 }
4021
4022                 edesc->src_nents = src_nents;
4023                 qm_sg_bytes = pad_sg_nents(1 + mapped_nents) *
4024                               sizeof(*sg_table);
4025                 sg_table = &edesc->sgt[0];
4026
4027                 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
4028                 if (ret)
4029                         goto unmap_ctx;
4030
4031                 sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0);
4032
4033                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
4034                                                   qm_sg_bytes, DMA_TO_DEVICE);
4035                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4036                         dev_err(ctx->dev, "unable to map S/G table\n");
4037                         ret = -ENOMEM;
4038                         goto unmap_ctx;
4039                 }
4040                 edesc->qm_sg_bytes = qm_sg_bytes;
4041
4042                 state->ctx_dma_len = ctx->ctx_len;
4043                 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
4044                                                 ctx->ctx_len, DMA_FROM_DEVICE);
4045                 if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4046                         dev_err(ctx->dev, "unable to map ctx\n");
4047                         state->ctx_dma = 0;
4048                         ret = -ENOMEM;
4049                         goto unmap_ctx;
4050                 }
4051
4052                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4053                 dpaa2_fl_set_final(in_fle, true);
4054                 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4055                 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4056                 dpaa2_fl_set_len(in_fle, to_hash);
4057                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4058                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4059                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4060
4061                 req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4062                 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4063                 req_ctx->cbk = ahash_done_ctx_dst;
4064                 req_ctx->ctx = &req->base;
4065                 req_ctx->edesc = edesc;
4066
4067                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4068                 if (ret != -EINPROGRESS &&
4069                     !(ret == -EBUSY &&
4070                       req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4071                         goto unmap_ctx;
4072
4073                 state->update = ahash_update_ctx;
4074                 state->finup = ahash_finup_ctx;
4075                 state->final = ahash_final_ctx;
4076         } else if (*next_buflen) {
4077                 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
4078                                          req->nbytes, 0);
4079                 *buflen = *next_buflen;
4080
4081                 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
4082                                      DUMP_PREFIX_ADDRESS, 16, 4, buf,
4083                                      *buflen, 1);
4084         }
4085
4086         return ret;
4087 unmap_ctx:
4088         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4089         qi_cache_free(edesc);
4090         return ret;
4091 }
4092
4093 static int ahash_finup_no_ctx(struct ahash_request *req)
4094 {
4095         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4096         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
4097         struct caam_hash_state *state = ahash_request_ctx_dma(req);
4098         struct caam_request *req_ctx = &state->caam_req;
4099         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4100         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4101         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4102                       GFP_KERNEL : GFP_ATOMIC;
4103         int buflen = state->buflen;
4104         int qm_sg_bytes, src_nents, mapped_nents;
4105         int digestsize = crypto_ahash_digestsize(ahash);
4106         struct ahash_edesc *edesc;
4107         struct dpaa2_sg_entry *sg_table;
4108         int ret = -ENOMEM;
4109
4110         src_nents = sg_nents_for_len(req->src, req->nbytes);
4111         if (src_nents < 0) {
4112                 dev_err(ctx->dev, "Invalid number of src SG.\n");
4113                 return src_nents;
4114         }
4115
4116         if (src_nents) {
4117                 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4118                                           DMA_TO_DEVICE);
4119                 if (!mapped_nents) {
4120                         dev_err(ctx->dev, "unable to DMA map source\n");
4121                         return ret;
4122                 }
4123         } else {
4124                 mapped_nents = 0;
4125         }
4126
4127         /* allocate space for base edesc and link tables */
4128         edesc = qi_cache_zalloc(GFP_DMA | flags);
4129         if (!edesc) {
4130                 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
4131                 return ret;
4132         }
4133
4134         edesc->src_nents = src_nents;
4135         qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table);
4136         sg_table = &edesc->sgt[0];
4137
4138         ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
4139         if (ret)
4140                 goto unmap;
4141
4142         sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0);
4143
4144         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
4145                                           DMA_TO_DEVICE);
4146         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4147                 dev_err(ctx->dev, "unable to map S/G table\n");
4148                 ret = -ENOMEM;
4149                 goto unmap;
4150         }
4151         edesc->qm_sg_bytes = qm_sg_bytes;
4152
4153         state->ctx_dma_len = digestsize;
4154         state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
4155                                         DMA_FROM_DEVICE);
4156         if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4157                 dev_err(ctx->dev, "unable to map ctx\n");
4158                 state->ctx_dma = 0;
4159                 ret = -ENOMEM;
4160                 goto unmap;
4161         }
4162
4163         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4164         dpaa2_fl_set_final(in_fle, true);
4165         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4166         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4167         dpaa2_fl_set_len(in_fle, buflen + req->nbytes);
4168         dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4169         dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4170         dpaa2_fl_set_len(out_fle, digestsize);
4171
4172         req_ctx->flc = &ctx->flc[DIGEST];
4173         req_ctx->flc_dma = ctx->flc_dma[DIGEST];
4174         req_ctx->cbk = ahash_done;
4175         req_ctx->ctx = &req->base;
4176         req_ctx->edesc = edesc;
4177         ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4178         if (ret != -EINPROGRESS &&
4179             !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4180                 goto unmap;
4181
4182         return ret;
4183 unmap:
4184         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
4185         qi_cache_free(edesc);
4186         return ret;
4187 }
4188
4189 static int ahash_update_first(struct ahash_request *req)
4190 {
4191         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4192         struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
4193         struct caam_hash_state *state = ahash_request_ctx_dma(req);
4194         struct caam_request *req_ctx = &state->caam_req;
4195         struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4196         struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4197         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4198                       GFP_KERNEL : GFP_ATOMIC;
4199         u8 *buf = state->buf;
4200         int *buflen = &state->buflen;
4201         int *next_buflen = &state->next_buflen;
4202         int to_hash;
4203         int src_nents, mapped_nents;
4204         struct ahash_edesc *edesc;
4205         int ret = 0;
4206
4207         *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
4208                                       1);
4209         to_hash = req->nbytes - *next_buflen;
4210
4211         if (to_hash) {
4212                 struct dpaa2_sg_entry *sg_table;
4213                 int src_len = req->nbytes - *next_buflen;
4214
4215                 src_nents = sg_nents_for_len(req->src, src_len);
4216                 if (src_nents < 0) {
4217                         dev_err(ctx->dev, "Invalid number of src SG.\n");
4218                         return src_nents;
4219                 }
4220
4221                 if (src_nents) {
4222                         mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4223                                                   DMA_TO_DEVICE);
4224                         if (!mapped_nents) {
4225                                 dev_err(ctx->dev, "unable to map source for DMA\n");
4226                                 return -ENOMEM;
4227                         }
4228                 } else {
4229                         mapped_nents = 0;
4230                 }
4231
4232                 /* allocate space for base edesc and link tables */
4233                 edesc = qi_cache_zalloc(GFP_DMA | flags);
4234                 if (!edesc) {
4235                         dma_unmap_sg(ctx->dev, req->src, src_nents,
4236                                      DMA_TO_DEVICE);
4237                         return -ENOMEM;
4238                 }
4239
4240                 edesc->src_nents = src_nents;
4241                 sg_table = &edesc->sgt[0];
4242
4243                 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4244                 dpaa2_fl_set_final(in_fle, true);
4245                 dpaa2_fl_set_len(in_fle, to_hash);
4246
4247                 if (mapped_nents > 1) {
4248                         int qm_sg_bytes;
4249
4250                         sg_to_qm_sg_last(req->src, src_len, sg_table, 0);
4251                         qm_sg_bytes = pad_sg_nents(mapped_nents) *
4252                                       sizeof(*sg_table);
4253                         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
4254                                                           qm_sg_bytes,
4255                                                           DMA_TO_DEVICE);
4256                         if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4257                                 dev_err(ctx->dev, "unable to map S/G table\n");
4258                                 ret = -ENOMEM;
4259                                 goto unmap_ctx;
4260                         }
4261                         edesc->qm_sg_bytes = qm_sg_bytes;
4262                         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4263                         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4264                 } else {
4265                         dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
4266                         dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
4267                 }
4268
4269                 state->ctx_dma_len = ctx->ctx_len;
4270                 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
4271                                                 ctx->ctx_len, DMA_FROM_DEVICE);
4272                 if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4273                         dev_err(ctx->dev, "unable to map ctx\n");
4274                         state->ctx_dma = 0;
4275                         ret = -ENOMEM;
4276                         goto unmap_ctx;
4277                 }
4278
4279                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4280                 dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4281                 dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4282
4283                 req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4284                 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4285                 req_ctx->cbk = ahash_done_ctx_dst;
4286                 req_ctx->ctx = &req->base;
4287                 req_ctx->edesc = edesc;
4288
4289                 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4290                 if (ret != -EINPROGRESS &&
4291                     !(ret == -EBUSY && req->base.flags &
4292                       CRYPTO_TFM_REQ_MAY_BACKLOG))
4293                         goto unmap_ctx;
4294
4295                 state->update = ahash_update_ctx;
4296                 state->finup = ahash_finup_ctx;
4297                 state->final = ahash_final_ctx;
4298         } else if (*next_buflen) {
4299                 state->update = ahash_update_no_ctx;
4300                 state->finup = ahash_finup_no_ctx;
4301                 state->final = ahash_final_no_ctx;
4302                 scatterwalk_map_and_copy(buf, req->src, 0,
4303                                          req->nbytes, 0);
4304                 *buflen = *next_buflen;
4305
4306                 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
4307                                      DUMP_PREFIX_ADDRESS, 16, 4, buf,
4308                                      *buflen, 1);
4309         }
4310
4311         return ret;
4312 unmap_ctx:
4313         ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4314         qi_cache_free(edesc);
4315         return ret;
4316 }
4317
4318 static int ahash_finup_first(struct ahash_request *req)
4319 {
4320         return ahash_digest(req);
4321 }
4322
4323 static int ahash_init(struct ahash_request *req)
4324 {
4325         struct caam_hash_state *state = ahash_request_ctx_dma(req);
4326
4327         state->update = ahash_update_first;
4328         state->finup = ahash_finup_first;
4329         state->final = ahash_final_no_ctx;
4330
4331         state->ctx_dma = 0;
4332         state->ctx_dma_len = 0;
4333         state->buf_dma = 0;
4334         state->buflen = 0;
4335         state->next_buflen = 0;
4336
4337         return 0;
4338 }
4339
4340 static int ahash_update(struct ahash_request *req)
4341 {
4342         struct caam_hash_state *state = ahash_request_ctx_dma(req);
4343
4344         return state->update(req);
4345 }
4346
4347 static int ahash_finup(struct ahash_request *req)
4348 {
4349         struct caam_hash_state *state = ahash_request_ctx_dma(req);
4350
4351         return state->finup(req);
4352 }
4353
4354 static int ahash_final(struct ahash_request *req)
4355 {
4356         struct caam_hash_state *state = ahash_request_ctx_dma(req);
4357
4358         return state->final(req);
4359 }
4360
4361 static int ahash_export(struct ahash_request *req, void *out)
4362 {
4363         struct caam_hash_state *state = ahash_request_ctx_dma(req);
4364         struct caam_export_state *export = out;
4365         u8 *buf = state->buf;
4366         int len = state->buflen;
4367
4368         memcpy(export->buf, buf, len);
4369         memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
4370         export->buflen = len;
4371         export->update = state->update;
4372         export->final = state->final;
4373         export->finup = state->finup;
4374
4375         return 0;
4376 }
4377
4378 static int ahash_import(struct ahash_request *req, const void *in)
4379 {
4380         struct caam_hash_state *state = ahash_request_ctx_dma(req);
4381         const struct caam_export_state *export = in;
4382
4383         memset(state, 0, sizeof(*state));
4384         memcpy(state->buf, export->buf, export->buflen);
4385         memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
4386         state->buflen = export->buflen;
4387         state->update = export->update;
4388         state->final = export->final;
4389         state->finup = export->finup;
4390
4391         return 0;
4392 }
4393
4394 struct caam_hash_template {
4395         char name[CRYPTO_MAX_ALG_NAME];
4396         char driver_name[CRYPTO_MAX_ALG_NAME];
4397         char hmac_name[CRYPTO_MAX_ALG_NAME];
4398         char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
4399         unsigned int blocksize;
4400         struct ahash_alg template_ahash;
4401         u32 alg_type;
4402 };
4403
4404 /* ahash descriptors */
4405 static struct caam_hash_template driver_hash[] = {
4406         {
4407                 .name = "sha1",
4408                 .driver_name = "sha1-caam-qi2",
4409                 .hmac_name = "hmac(sha1)",
4410                 .hmac_driver_name = "hmac-sha1-caam-qi2",
4411                 .blocksize = SHA1_BLOCK_SIZE,
4412                 .template_ahash = {
4413                         .init = ahash_init,
4414                         .update = ahash_update,
4415                         .final = ahash_final,
4416                         .finup = ahash_finup,
4417                         .digest = ahash_digest,
4418                         .export = ahash_export,
4419                         .import = ahash_import,
4420                         .setkey = ahash_setkey,
4421                         .halg = {
4422                                 .digestsize = SHA1_DIGEST_SIZE,
4423                                 .statesize = sizeof(struct caam_export_state),
4424                         },
4425                 },
4426                 .alg_type = OP_ALG_ALGSEL_SHA1,
4427         }, {
4428                 .name = "sha224",
4429                 .driver_name = "sha224-caam-qi2",
4430                 .hmac_name = "hmac(sha224)",
4431                 .hmac_driver_name = "hmac-sha224-caam-qi2",
4432                 .blocksize = SHA224_BLOCK_SIZE,
4433                 .template_ahash = {
4434                         .init = ahash_init,
4435                         .update = ahash_update,
4436                         .final = ahash_final,
4437                         .finup = ahash_finup,
4438                         .digest = ahash_digest,
4439                         .export = ahash_export,
4440                         .import = ahash_import,
4441                         .setkey = ahash_setkey,
4442                         .halg = {
4443                                 .digestsize = SHA224_DIGEST_SIZE,
4444                                 .statesize = sizeof(struct caam_export_state),
4445                         },
4446                 },
4447                 .alg_type = OP_ALG_ALGSEL_SHA224,
4448         }, {
4449                 .name = "sha256",
4450                 .driver_name = "sha256-caam-qi2",
4451                 .hmac_name = "hmac(sha256)",
4452                 .hmac_driver_name = "hmac-sha256-caam-qi2",
4453                 .blocksize = SHA256_BLOCK_SIZE,
4454                 .template_ahash = {
4455                         .init = ahash_init,
4456                         .update = ahash_update,
4457                         .final = ahash_final,
4458                         .finup = ahash_finup,
4459                         .digest = ahash_digest,
4460                         .export = ahash_export,
4461                         .import = ahash_import,
4462                         .setkey = ahash_setkey,
4463                         .halg = {
4464                                 .digestsize = SHA256_DIGEST_SIZE,
4465                                 .statesize = sizeof(struct caam_export_state),
4466                         },
4467                 },
4468                 .alg_type = OP_ALG_ALGSEL_SHA256,
4469         }, {
4470                 .name = "sha384",
4471                 .driver_name = "sha384-caam-qi2",
4472                 .hmac_name = "hmac(sha384)",
4473                 .hmac_driver_name = "hmac-sha384-caam-qi2",
4474                 .blocksize = SHA384_BLOCK_SIZE,
4475                 .template_ahash = {
4476                         .init = ahash_init,
4477                         .update = ahash_update,
4478                         .final = ahash_final,
4479                         .finup = ahash_finup,
4480                         .digest = ahash_digest,
4481                         .export = ahash_export,
4482                         .import = ahash_import,
4483                         .setkey = ahash_setkey,
4484                         .halg = {
4485                                 .digestsize = SHA384_DIGEST_SIZE,
4486                                 .statesize = sizeof(struct caam_export_state),
4487                         },
4488                 },
4489                 .alg_type = OP_ALG_ALGSEL_SHA384,
4490         }, {
4491                 .name = "sha512",
4492                 .driver_name = "sha512-caam-qi2",
4493                 .hmac_name = "hmac(sha512)",
4494                 .hmac_driver_name = "hmac-sha512-caam-qi2",
4495                 .blocksize = SHA512_BLOCK_SIZE,
4496                 .template_ahash = {
4497                         .init = ahash_init,
4498                         .update = ahash_update,
4499                         .final = ahash_final,
4500                         .finup = ahash_finup,
4501                         .digest = ahash_digest,
4502                         .export = ahash_export,
4503                         .import = ahash_import,
4504                         .setkey = ahash_setkey,
4505                         .halg = {
4506                                 .digestsize = SHA512_DIGEST_SIZE,
4507                                 .statesize = sizeof(struct caam_export_state),
4508                         },
4509                 },
4510                 .alg_type = OP_ALG_ALGSEL_SHA512,
4511         }, {
4512                 .name = "md5",
4513                 .driver_name = "md5-caam-qi2",
4514                 .hmac_name = "hmac(md5)",
4515                 .hmac_driver_name = "hmac-md5-caam-qi2",
4516                 .blocksize = MD5_BLOCK_WORDS * 4,
4517                 .template_ahash = {
4518                         .init = ahash_init,
4519                         .update = ahash_update,
4520                         .final = ahash_final,
4521                         .finup = ahash_finup,
4522                         .digest = ahash_digest,
4523                         .export = ahash_export,
4524                         .import = ahash_import,
4525                         .setkey = ahash_setkey,
4526                         .halg = {
4527                                 .digestsize = MD5_DIGEST_SIZE,
4528                                 .statesize = sizeof(struct caam_export_state),
4529                         },
4530                 },
4531                 .alg_type = OP_ALG_ALGSEL_MD5,
4532         }
4533 };
4534
4535 struct caam_hash_alg {
4536         struct list_head entry;
4537         struct device *dev;
4538         int alg_type;
4539         struct ahash_alg ahash_alg;
4540 };
4541
4542 static int caam_hash_cra_init(struct crypto_tfm *tfm)
4543 {
4544         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
4545         struct crypto_alg *base = tfm->__crt_alg;
4546         struct hash_alg_common *halg =
4547                  container_of(base, struct hash_alg_common, base);
4548         struct ahash_alg *alg =
4549                  container_of(halg, struct ahash_alg, halg);
4550         struct caam_hash_alg *caam_hash =
4551                  container_of(alg, struct caam_hash_alg, ahash_alg);
4552         struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);
4553         /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
4554         static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
4555                                          HASH_MSG_LEN + SHA1_DIGEST_SIZE,
4556                                          HASH_MSG_LEN + 32,
4557                                          HASH_MSG_LEN + SHA256_DIGEST_SIZE,
4558                                          HASH_MSG_LEN + 64,
4559                                          HASH_MSG_LEN + SHA512_DIGEST_SIZE };
4560         dma_addr_t dma_addr;
4561         int i;
4562
4563         ctx->dev = caam_hash->dev;
4564
4565         if (alg->setkey) {
4566                 ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key,
4567                                                           ARRAY_SIZE(ctx->key),
4568                                                           DMA_TO_DEVICE,
4569                                                           DMA_ATTR_SKIP_CPU_SYNC);
4570                 if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) {
4571                         dev_err(ctx->dev, "unable to map key\n");
4572                         return -ENOMEM;
4573                 }
4574         }
4575
4576         dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc),
4577                                         DMA_BIDIRECTIONAL,
4578                                         DMA_ATTR_SKIP_CPU_SYNC);
4579         if (dma_mapping_error(ctx->dev, dma_addr)) {
4580                 dev_err(ctx->dev, "unable to map shared descriptors\n");
4581                 if (ctx->adata.key_dma)
4582                         dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4583                                                ARRAY_SIZE(ctx->key),
4584                                                DMA_TO_DEVICE,
4585                                                DMA_ATTR_SKIP_CPU_SYNC);
4586                 return -ENOMEM;
4587         }
4588
4589         for (i = 0; i < HASH_NUM_OP; i++)
4590                 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
4591
4592         /* copy descriptor header template value */
4593         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
4594
4595         ctx->ctx_len = runninglen[(ctx->adata.algtype &
4596                                    OP_ALG_ALGSEL_SUBMASK) >>
4597                                   OP_ALG_ALGSEL_SHIFT];
4598
4599         crypto_ahash_set_reqsize_dma(ahash, sizeof(struct caam_hash_state));
4600
4601         /*
4602          * For keyed hash algorithms shared descriptors
4603          * will be created later in setkey() callback
4604          */
4605         return alg->setkey ? 0 : ahash_set_sh_desc(ahash);
4606 }
4607
4608 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
4609 {
4610         struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);
4611
4612         dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc),
4613                                DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
4614         if (ctx->adata.key_dma)
4615                 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4616                                        ARRAY_SIZE(ctx->key), DMA_TO_DEVICE,
4617                                        DMA_ATTR_SKIP_CPU_SYNC);
4618 }
4619
4620 static struct caam_hash_alg *caam_hash_alloc(struct device *dev,
4621         struct caam_hash_template *template, bool keyed)
4622 {
4623         struct caam_hash_alg *t_alg;
4624         struct ahash_alg *halg;
4625         struct crypto_alg *alg;
4626
4627         t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
4628         if (!t_alg)
4629                 return ERR_PTR(-ENOMEM);
4630
4631         t_alg->ahash_alg = template->template_ahash;
4632         halg = &t_alg->ahash_alg;
4633         alg = &halg->halg.base;
4634
4635         if (keyed) {
4636                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4637                          template->hmac_name);
4638                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4639                          template->hmac_driver_name);
4640         } else {
4641                 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4642                          template->name);
4643                 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4644                          template->driver_name);
4645                 t_alg->ahash_alg.setkey = NULL;
4646         }
4647         alg->cra_module = THIS_MODULE;
4648         alg->cra_init = caam_hash_cra_init;
4649         alg->cra_exit = caam_hash_cra_exit;
4650         alg->cra_ctxsize = sizeof(struct caam_hash_ctx) + crypto_dma_padding();
4651         alg->cra_priority = CAAM_CRA_PRIORITY;
4652         alg->cra_blocksize = template->blocksize;
4653         alg->cra_alignmask = 0;
4654         alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY;
4655
4656         t_alg->alg_type = template->alg_type;
4657         t_alg->dev = dev;
4658
4659         return t_alg;
4660 }
4661
4662 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx)
4663 {
4664         struct dpaa2_caam_priv_per_cpu *ppriv;
4665
4666         ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx);
4667         napi_schedule_irqoff(&ppriv->napi);
4668 }
4669
4670 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv)
4671 {
4672         struct device *dev = priv->dev;
4673         struct dpaa2_io_notification_ctx *nctx;
4674         struct dpaa2_caam_priv_per_cpu *ppriv;
4675         int err, i = 0, cpu;
4676
4677         for_each_online_cpu(cpu) {
4678                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4679                 ppriv->priv = priv;
4680                 nctx = &ppriv->nctx;
4681                 nctx->is_cdan = 0;
4682                 nctx->id = ppriv->rsp_fqid;
4683                 nctx->desired_cpu = cpu;
4684                 nctx->cb = dpaa2_caam_fqdan_cb;
4685
4686                 /* Register notification callbacks */
4687                 ppriv->dpio = dpaa2_io_service_select(cpu);
4688                 err = dpaa2_io_service_register(ppriv->dpio, nctx, dev);
4689                 if (unlikely(err)) {
4690                         dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu);
4691                         nctx->cb = NULL;
4692                         /*
4693                          * If no affine DPIO for this core, there's probably
4694                          * none available for next cores either. Signal we want
4695                          * to retry later, in case the DPIO devices weren't
4696                          * probed yet.
4697                          */
4698                         err = -EPROBE_DEFER;
4699                         goto err;
4700                 }
4701
4702                 ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE,
4703                                                      dev);
4704                 if (unlikely(!ppriv->store)) {
4705                         dev_err(dev, "dpaa2_io_store_create() failed\n");
4706                         err = -ENOMEM;
4707                         goto err;
4708                 }
4709
4710                 if (++i == priv->num_pairs)
4711                         break;
4712         }
4713
4714         return 0;
4715
4716 err:
4717         for_each_online_cpu(cpu) {
4718                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4719                 if (!ppriv->nctx.cb)
4720                         break;
4721                 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev);
4722         }
4723
4724         for_each_online_cpu(cpu) {
4725                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4726                 if (!ppriv->store)
4727                         break;
4728                 dpaa2_io_store_destroy(ppriv->store);
4729         }
4730
4731         return err;
4732 }
4733
4734 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv)
4735 {
4736         struct dpaa2_caam_priv_per_cpu *ppriv;
4737         int i = 0, cpu;
4738
4739         for_each_online_cpu(cpu) {
4740                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4741                 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx,
4742                                             priv->dev);
4743                 dpaa2_io_store_destroy(ppriv->store);
4744
4745                 if (++i == priv->num_pairs)
4746                         return;
4747         }
4748 }
4749
4750 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv)
4751 {
4752         struct dpseci_rx_queue_cfg rx_queue_cfg;
4753         struct device *dev = priv->dev;
4754         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4755         struct dpaa2_caam_priv_per_cpu *ppriv;
4756         int err = 0, i = 0, cpu;
4757
4758         /* Configure Rx queues */
4759         for_each_online_cpu(cpu) {
4760                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
4761
4762                 rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST |
4763                                        DPSECI_QUEUE_OPT_USER_CTX;
4764                 rx_queue_cfg.order_preservation_en = 0;
4765                 rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO;
4766                 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
4767                 /*
4768                  * Rx priority (WQ) doesn't really matter, since we use
4769                  * pull mode, i.e. volatile dequeues from specific FQs
4770                  */
4771                 rx_queue_cfg.dest_cfg.priority = 0;
4772                 rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
4773
4774                 err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4775                                           &rx_queue_cfg);
4776                 if (err) {
4777                         dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n",
4778                                 err);
4779                         return err;
4780                 }
4781
4782                 if (++i == priv->num_pairs)
4783                         break;
4784         }
4785
4786         return err;
4787 }
4788
4789 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv)
4790 {
4791         struct device *dev = priv->dev;
4792
4793         if (!priv->cscn_mem)
4794                 return;
4795
4796         dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4797         kfree(priv->cscn_mem);
4798 }
4799
4800 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv)
4801 {
4802         struct device *dev = priv->dev;
4803         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4804         int err;
4805
4806         if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) {
4807                 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle);
4808                 if (err)
4809                         dev_err(dev, "dpseci_reset() failed\n");
4810         }
4811
4812         dpaa2_dpseci_congestion_free(priv);
4813         dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
4814 }
4815
4816 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv,
4817                                   const struct dpaa2_fd *fd)
4818 {
4819         struct caam_request *req;
4820         u32 fd_err;
4821
4822         if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) {
4823                 dev_err(priv->dev, "Only Frame List FD format is supported!\n");
4824                 return;
4825         }
4826
4827         fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK;
4828         if (unlikely(fd_err))
4829                 dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err);
4830
4831         /*
4832          * FD[ADDR] is guaranteed to be valid, irrespective of errors reported
4833          * in FD[ERR] or FD[FRC].
4834          */
4835         req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd));
4836         dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt),
4837                          DMA_BIDIRECTIONAL);
4838         req->cbk(req->ctx, dpaa2_fd_get_frc(fd));
4839 }
4840
4841 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv)
4842 {
4843         int err;
4844
4845         /* Retry while portal is busy */
4846         do {
4847                 err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid,
4848                                                ppriv->store);
4849         } while (err == -EBUSY);
4850
4851         if (unlikely(err))
4852                 dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err);
4853
4854         return err;
4855 }
4856
4857 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv)
4858 {
4859         struct dpaa2_dq *dq;
4860         int cleaned = 0, is_last;
4861
4862         do {
4863                 dq = dpaa2_io_store_next(ppriv->store, &is_last);
4864                 if (unlikely(!dq)) {
4865                         if (unlikely(!is_last)) {
4866                                 dev_dbg(ppriv->priv->dev,
4867                                         "FQ %d returned no valid frames\n",
4868                                         ppriv->rsp_fqid);
4869                                 /*
4870                                  * MUST retry until we get some sort of
4871                                  * valid response token (be it "empty dequeue"
4872                                  * or a valid frame).
4873                                  */
4874                                 continue;
4875                         }
4876                         break;
4877                 }
4878
4879                 /* Process FD */
4880                 dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq));
4881                 cleaned++;
4882         } while (!is_last);
4883
4884         return cleaned;
4885 }
4886
4887 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget)
4888 {
4889         struct dpaa2_caam_priv_per_cpu *ppriv;
4890         struct dpaa2_caam_priv *priv;
4891         int err, cleaned = 0, store_cleaned;
4892
4893         ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi);
4894         priv = ppriv->priv;
4895
4896         if (unlikely(dpaa2_caam_pull_fq(ppriv)))
4897                 return 0;
4898
4899         do {
4900                 store_cleaned = dpaa2_caam_store_consume(ppriv);
4901                 cleaned += store_cleaned;
4902
4903                 if (store_cleaned == 0 ||
4904                     cleaned > budget - DPAA2_CAAM_STORE_SIZE)
4905                         break;
4906
4907                 /* Try to dequeue some more */
4908                 err = dpaa2_caam_pull_fq(ppriv);
4909                 if (unlikely(err))
4910                         break;
4911         } while (1);
4912
4913         if (cleaned < budget) {
4914                 napi_complete_done(napi, cleaned);
4915                 err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx);
4916                 if (unlikely(err))
4917                         dev_err(priv->dev, "Notification rearm failed: %d\n",
4918                                 err);
4919         }
4920
4921         return cleaned;
4922 }
4923
4924 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv,
4925                                          u16 token)
4926 {
4927         struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 };
4928         struct device *dev = priv->dev;
4929         int err;
4930
4931         /*
4932          * Congestion group feature supported starting with DPSECI API v5.1
4933          * and only when object has been created with this capability.
4934          */
4935         if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) ||
4936             !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG))
4937                 return 0;
4938
4939         priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN,
4940                                  GFP_KERNEL | GFP_DMA);
4941         if (!priv->cscn_mem)
4942                 return -ENOMEM;
4943
4944         priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN);
4945         priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned,
4946                                         DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4947         if (dma_mapping_error(dev, priv->cscn_dma)) {
4948                 dev_err(dev, "Error mapping CSCN memory area\n");
4949                 err = -ENOMEM;
4950                 goto err_dma_map;
4951         }
4952
4953         cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES;
4954         cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH;
4955         cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH;
4956         cong_notif_cfg.message_ctx = (uintptr_t)priv;
4957         cong_notif_cfg.message_iova = priv->cscn_dma;
4958         cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER |
4959                                         DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT |
4960                                         DPSECI_CGN_MODE_COHERENT_WRITE;
4961
4962         err = dpseci_set_congestion_notification(priv->mc_io, 0, token,
4963                                                  &cong_notif_cfg);
4964         if (err) {
4965                 dev_err(dev, "dpseci_set_congestion_notification failed\n");
4966                 goto err_set_cong;
4967         }
4968
4969         return 0;
4970
4971 err_set_cong:
4972         dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4973 err_dma_map:
4974         kfree(priv->cscn_mem);
4975
4976         return err;
4977 }
4978
4979 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev)
4980 {
4981         struct device *dev = &ls_dev->dev;
4982         struct dpaa2_caam_priv *priv;
4983         struct dpaa2_caam_priv_per_cpu *ppriv;
4984         int err, cpu;
4985         u8 i;
4986
4987         priv = dev_get_drvdata(dev);
4988
4989         priv->dev = dev;
4990         priv->dpsec_id = ls_dev->obj_desc.id;
4991
4992         /* Get a handle for the DPSECI this interface is associate with */
4993         err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle);
4994         if (err) {
4995                 dev_err(dev, "dpseci_open() failed: %d\n", err);
4996                 goto err_open;
4997         }
4998
4999         err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver,
5000                                      &priv->minor_ver);
5001         if (err) {
5002                 dev_err(dev, "dpseci_get_api_version() failed\n");
5003                 goto err_get_vers;
5004         }
5005
5006         dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver);
5007
5008         if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) {
5009                 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle);
5010                 if (err) {
5011                         dev_err(dev, "dpseci_reset() failed\n");
5012                         goto err_get_vers;
5013                 }
5014         }
5015
5016         err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
5017                                     &priv->dpseci_attr);
5018         if (err) {
5019                 dev_err(dev, "dpseci_get_attributes() failed\n");
5020                 goto err_get_vers;
5021         }
5022
5023         err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle,
5024                                   &priv->sec_attr);
5025         if (err) {
5026                 dev_err(dev, "dpseci_get_sec_attr() failed\n");
5027                 goto err_get_vers;
5028         }
5029
5030         err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle);
5031         if (err) {
5032                 dev_err(dev, "setup_congestion() failed\n");
5033                 goto err_get_vers;
5034         }
5035
5036         priv->num_pairs = min(priv->dpseci_attr.num_rx_queues,
5037                               priv->dpseci_attr.num_tx_queues);
5038         if (priv->num_pairs > num_online_cpus()) {
5039                 dev_warn(dev, "%d queues won't be used\n",
5040                          priv->num_pairs - num_online_cpus());
5041                 priv->num_pairs = num_online_cpus();
5042         }
5043
5044         for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) {
5045                 err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
5046                                           &priv->rx_queue_attr[i]);
5047                 if (err) {
5048                         dev_err(dev, "dpseci_get_rx_queue() failed\n");
5049                         goto err_get_rx_queue;
5050                 }
5051         }
5052
5053         for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) {
5054                 err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
5055                                           &priv->tx_queue_attr[i]);
5056                 if (err) {
5057                         dev_err(dev, "dpseci_get_tx_queue() failed\n");
5058                         goto err_get_rx_queue;
5059                 }
5060         }
5061
5062         i = 0;
5063         for_each_online_cpu(cpu) {
5064                 u8 j;
5065
5066                 j = i % priv->num_pairs;
5067
5068                 ppriv = per_cpu_ptr(priv->ppriv, cpu);
5069                 ppriv->req_fqid = priv->tx_queue_attr[j].fqid;
5070
5071                 /*
5072                  * Allow all cores to enqueue, while only some of them
5073                  * will take part in dequeuing.
5074                  */
5075                 if (++i > priv->num_pairs)
5076                         continue;
5077
5078                 ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid;
5079                 ppriv->prio = j;
5080
5081                 dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j,
5082                         priv->rx_queue_attr[j].fqid,
5083                         priv->tx_queue_attr[j].fqid);
5084
5085                 ppriv->net_dev.dev = *dev;
5086                 INIT_LIST_HEAD(&ppriv->net_dev.napi_list);
5087                 netif_napi_add_tx_weight(&ppriv->net_dev, &ppriv->napi,
5088                                          dpaa2_dpseci_poll,
5089                                          DPAA2_CAAM_NAPI_WEIGHT);
5090         }
5091
5092         return 0;
5093
5094 err_get_rx_queue:
5095         dpaa2_dpseci_congestion_free(priv);
5096 err_get_vers:
5097         dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
5098 err_open:
5099         return err;
5100 }
5101
5102 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv)
5103 {
5104         struct device *dev = priv->dev;
5105         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5106         struct dpaa2_caam_priv_per_cpu *ppriv;
5107         int i;
5108
5109         for (i = 0; i < priv->num_pairs; i++) {
5110                 ppriv = per_cpu_ptr(priv->ppriv, i);
5111                 napi_enable(&ppriv->napi);
5112         }
5113
5114         return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle);
5115 }
5116
5117 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv)
5118 {
5119         struct device *dev = priv->dev;
5120         struct dpaa2_caam_priv_per_cpu *ppriv;
5121         struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5122         int i, err = 0, enabled;
5123
5124         err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle);
5125         if (err) {
5126                 dev_err(dev, "dpseci_disable() failed\n");
5127                 return err;
5128         }
5129
5130         err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled);
5131         if (err) {
5132                 dev_err(dev, "dpseci_is_enabled() failed\n");
5133                 return err;
5134         }
5135
5136         dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true");
5137
5138         for (i = 0; i < priv->num_pairs; i++) {
5139                 ppriv = per_cpu_ptr(priv->ppriv, i);
5140                 napi_disable(&ppriv->napi);
5141                 netif_napi_del(&ppriv->napi);
5142         }
5143
5144         return 0;
5145 }
5146
5147 static struct list_head hash_list;
5148
5149 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev)
5150 {
5151         struct device *dev;
5152         struct dpaa2_caam_priv *priv;
5153         int i, err = 0;
5154         bool registered = false;
5155
5156         /*
5157          * There is no way to get CAAM endianness - there is no direct register
5158          * space access and MC f/w does not provide this attribute.
5159          * All DPAA2-based SoCs have little endian CAAM, thus hard-code this
5160          * property.
5161          */
5162         caam_little_end = true;
5163
5164         caam_imx = false;
5165
5166         dev = &dpseci_dev->dev;
5167
5168         priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
5169         if (!priv)
5170                 return -ENOMEM;
5171
5172         dev_set_drvdata(dev, priv);
5173
5174         priv->domain = iommu_get_domain_for_dev(dev);
5175
5176         qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE,
5177                                      0, SLAB_CACHE_DMA, NULL);
5178         if (!qi_cache) {
5179                 dev_err(dev, "Can't allocate SEC cache\n");
5180                 return -ENOMEM;
5181         }
5182
5183         err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49));
5184         if (err) {
5185                 dev_err(dev, "dma_set_mask_and_coherent() failed\n");
5186                 goto err_dma_mask;
5187         }
5188
5189         /* Obtain a MC portal */
5190         err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io);
5191         if (err) {
5192                 if (err == -ENXIO)
5193                         err = -EPROBE_DEFER;
5194                 else
5195                         dev_err(dev, "MC portal allocation failed\n");
5196
5197                 goto err_dma_mask;
5198         }
5199
5200         priv->ppriv = alloc_percpu(*priv->ppriv);
5201         if (!priv->ppriv) {
5202                 dev_err(dev, "alloc_percpu() failed\n");
5203                 err = -ENOMEM;
5204                 goto err_alloc_ppriv;
5205         }
5206
5207         /* DPSECI initialization */
5208         err = dpaa2_dpseci_setup(dpseci_dev);
5209         if (err) {
5210                 dev_err(dev, "dpaa2_dpseci_setup() failed\n");
5211                 goto err_dpseci_setup;
5212         }
5213
5214         /* DPIO */
5215         err = dpaa2_dpseci_dpio_setup(priv);
5216         if (err) {
5217                 dev_err_probe(dev, err, "dpaa2_dpseci_dpio_setup() failed\n");
5218                 goto err_dpio_setup;
5219         }
5220
5221         /* DPSECI binding to DPIO */
5222         err = dpaa2_dpseci_bind(priv);
5223         if (err) {
5224                 dev_err(dev, "dpaa2_dpseci_bind() failed\n");
5225                 goto err_bind;
5226         }
5227
5228         /* DPSECI enable */
5229         err = dpaa2_dpseci_enable(priv);
5230         if (err) {
5231                 dev_err(dev, "dpaa2_dpseci_enable() failed\n");
5232                 goto err_bind;
5233         }
5234
5235         dpaa2_dpseci_debugfs_init(priv);
5236
5237         /* register crypto algorithms the device supports */
5238         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5239                 struct caam_skcipher_alg *t_alg = driver_algs + i;
5240                 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
5241
5242                 /* Skip DES algorithms if not supported by device */
5243                 if (!priv->sec_attr.des_acc_num &&
5244                     (alg_sel == OP_ALG_ALGSEL_3DES ||
5245                      alg_sel == OP_ALG_ALGSEL_DES))
5246                         continue;
5247
5248                 /* Skip AES algorithms if not supported by device */
5249                 if (!priv->sec_attr.aes_acc_num &&
5250                     alg_sel == OP_ALG_ALGSEL_AES)
5251                         continue;
5252
5253                 /* Skip CHACHA20 algorithms if not supported by device */
5254                 if (alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5255                     !priv->sec_attr.ccha_acc_num)
5256                         continue;
5257
5258                 t_alg->caam.dev = dev;
5259                 caam_skcipher_alg_init(t_alg);
5260
5261                 err = crypto_register_skcipher(&t_alg->skcipher);
5262                 if (err) {
5263                         dev_warn(dev, "%s alg registration failed: %d\n",
5264                                  t_alg->skcipher.base.cra_driver_name, err);
5265                         continue;
5266                 }
5267
5268                 t_alg->registered = true;
5269                 registered = true;
5270         }
5271
5272         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5273                 struct caam_aead_alg *t_alg = driver_aeads + i;
5274                 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
5275                                  OP_ALG_ALGSEL_MASK;
5276                 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
5277                                  OP_ALG_ALGSEL_MASK;
5278
5279                 /* Skip DES algorithms if not supported by device */
5280                 if (!priv->sec_attr.des_acc_num &&
5281                     (c1_alg_sel == OP_ALG_ALGSEL_3DES ||
5282                      c1_alg_sel == OP_ALG_ALGSEL_DES))
5283                         continue;
5284
5285                 /* Skip AES algorithms if not supported by device */
5286                 if (!priv->sec_attr.aes_acc_num &&
5287                     c1_alg_sel == OP_ALG_ALGSEL_AES)
5288                         continue;
5289
5290                 /* Skip CHACHA20 algorithms if not supported by device */
5291                 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5292                     !priv->sec_attr.ccha_acc_num)
5293                         continue;
5294
5295                 /* Skip POLY1305 algorithms if not supported by device */
5296                 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 &&
5297                     !priv->sec_attr.ptha_acc_num)
5298                         continue;
5299
5300                 /*
5301                  * Skip algorithms requiring message digests
5302                  * if MD not supported by device.
5303                  */
5304                 if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 &&
5305                     !priv->sec_attr.md_acc_num)
5306                         continue;
5307
5308                 t_alg->caam.dev = dev;
5309                 caam_aead_alg_init(t_alg);
5310
5311                 err = crypto_register_aead(&t_alg->aead);
5312                 if (err) {
5313                         dev_warn(dev, "%s alg registration failed: %d\n",
5314                                  t_alg->aead.base.cra_driver_name, err);
5315                         continue;
5316                 }
5317
5318                 t_alg->registered = true;
5319                 registered = true;
5320         }
5321         if (registered)
5322                 dev_info(dev, "algorithms registered in /proc/crypto\n");
5323
5324         /* register hash algorithms the device supports */
5325         INIT_LIST_HEAD(&hash_list);
5326
5327         /*
5328          * Skip registration of any hashing algorithms if MD block
5329          * is not present.
5330          */
5331         if (!priv->sec_attr.md_acc_num)
5332                 return 0;
5333
5334         for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
5335                 struct caam_hash_alg *t_alg;
5336                 struct caam_hash_template *alg = driver_hash + i;
5337
5338                 /* register hmac version */
5339                 t_alg = caam_hash_alloc(dev, alg, true);
5340                 if (IS_ERR(t_alg)) {
5341                         err = PTR_ERR(t_alg);
5342                         dev_warn(dev, "%s hash alg allocation failed: %d\n",
5343                                  alg->hmac_driver_name, err);
5344                         continue;
5345                 }
5346
5347                 err = crypto_register_ahash(&t_alg->ahash_alg);
5348                 if (err) {
5349                         dev_warn(dev, "%s alg registration failed: %d\n",
5350                                  t_alg->ahash_alg.halg.base.cra_driver_name,
5351                                  err);
5352                         kfree(t_alg);
5353                 } else {
5354                         list_add_tail(&t_alg->entry, &hash_list);
5355                 }
5356
5357                 /* register unkeyed version */
5358                 t_alg = caam_hash_alloc(dev, alg, false);
5359                 if (IS_ERR(t_alg)) {
5360                         err = PTR_ERR(t_alg);
5361                         dev_warn(dev, "%s alg allocation failed: %d\n",
5362                                  alg->driver_name, err);
5363                         continue;
5364                 }
5365
5366                 err = crypto_register_ahash(&t_alg->ahash_alg);
5367                 if (err) {
5368                         dev_warn(dev, "%s alg registration failed: %d\n",
5369                                  t_alg->ahash_alg.halg.base.cra_driver_name,
5370                                  err);
5371                         kfree(t_alg);
5372                 } else {
5373                         list_add_tail(&t_alg->entry, &hash_list);
5374                 }
5375         }
5376         if (!list_empty(&hash_list))
5377                 dev_info(dev, "hash algorithms registered in /proc/crypto\n");
5378
5379         return err;
5380
5381 err_bind:
5382         dpaa2_dpseci_dpio_free(priv);
5383 err_dpio_setup:
5384         dpaa2_dpseci_free(priv);
5385 err_dpseci_setup:
5386         free_percpu(priv->ppriv);
5387 err_alloc_ppriv:
5388         fsl_mc_portal_free(priv->mc_io);
5389 err_dma_mask:
5390         kmem_cache_destroy(qi_cache);
5391
5392         return err;
5393 }
5394
5395 static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev)
5396 {
5397         struct device *dev;
5398         struct dpaa2_caam_priv *priv;
5399         int i;
5400
5401         dev = &ls_dev->dev;
5402         priv = dev_get_drvdata(dev);
5403
5404         dpaa2_dpseci_debugfs_exit(priv);
5405
5406         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5407                 struct caam_aead_alg *t_alg = driver_aeads + i;
5408
5409                 if (t_alg->registered)
5410                         crypto_unregister_aead(&t_alg->aead);
5411         }
5412
5413         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5414                 struct caam_skcipher_alg *t_alg = driver_algs + i;
5415
5416                 if (t_alg->registered)
5417                         crypto_unregister_skcipher(&t_alg->skcipher);
5418         }
5419
5420         if (hash_list.next) {
5421                 struct caam_hash_alg *t_hash_alg, *p;
5422
5423                 list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) {
5424                         crypto_unregister_ahash(&t_hash_alg->ahash_alg);
5425                         list_del(&t_hash_alg->entry);
5426                         kfree(t_hash_alg);
5427                 }
5428         }
5429
5430         dpaa2_dpseci_disable(priv);
5431         dpaa2_dpseci_dpio_free(priv);
5432         dpaa2_dpseci_free(priv);
5433         free_percpu(priv->ppriv);
5434         fsl_mc_portal_free(priv->mc_io);
5435         kmem_cache_destroy(qi_cache);
5436
5437         return 0;
5438 }
5439
5440 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
5441 {
5442         struct dpaa2_fd fd;
5443         struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
5444         struct dpaa2_caam_priv_per_cpu *ppriv;
5445         int err = 0, i;
5446
5447         if (IS_ERR(req))
5448                 return PTR_ERR(req);
5449
5450         if (priv->cscn_mem) {
5451                 dma_sync_single_for_cpu(priv->dev, priv->cscn_dma,
5452                                         DPAA2_CSCN_SIZE,
5453                                         DMA_FROM_DEVICE);
5454                 if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) {
5455                         dev_dbg_ratelimited(dev, "Dropping request\n");
5456                         return -EBUSY;
5457                 }
5458         }
5459
5460         dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma);
5461
5462         req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt),
5463                                          DMA_BIDIRECTIONAL);
5464         if (dma_mapping_error(dev, req->fd_flt_dma)) {
5465                 dev_err(dev, "DMA mapping error for QI enqueue request\n");
5466                 goto err_out;
5467         }
5468
5469         memset(&fd, 0, sizeof(fd));
5470         dpaa2_fd_set_format(&fd, dpaa2_fd_list);
5471         dpaa2_fd_set_addr(&fd, req->fd_flt_dma);
5472         dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
5473         dpaa2_fd_set_flc(&fd, req->flc_dma);
5474
5475         ppriv = raw_cpu_ptr(priv->ppriv);
5476         for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
5477                 err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid,
5478                                                   &fd);
5479                 if (err != -EBUSY)
5480                         break;
5481
5482                 cpu_relax();
5483         }
5484
5485         if (unlikely(err)) {
5486                 dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err);
5487                 goto err_out;
5488         }
5489
5490         return -EINPROGRESS;
5491
5492 err_out:
5493         dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt),
5494                          DMA_BIDIRECTIONAL);
5495         return -EIO;
5496 }
5497 EXPORT_SYMBOL(dpaa2_caam_enqueue);
5498
5499 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = {
5500         {
5501                 .vendor = FSL_MC_VENDOR_FREESCALE,
5502                 .obj_type = "dpseci",
5503         },
5504         { .vendor = 0x0 }
5505 };
5506 MODULE_DEVICE_TABLE(fslmc, dpaa2_caam_match_id_table);
5507
5508 static struct fsl_mc_driver dpaa2_caam_driver = {
5509         .driver = {
5510                 .name           = KBUILD_MODNAME,
5511                 .owner          = THIS_MODULE,
5512         },
5513         .probe          = dpaa2_caam_probe,
5514         .remove         = dpaa2_caam_remove,
5515         .match_id_table = dpaa2_caam_match_id_table
5516 };
5517
5518 MODULE_LICENSE("Dual BSD/GPL");
5519 MODULE_AUTHOR("Freescale Semiconductor, Inc");
5520 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver");
5521
5522 module_fsl_mc_driver(dpaa2_caam_driver);