drivers/crypto/bcm/cipher.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright 2016 Broadcom
   4  */
   5
   6 #include <linux/err.h>
   7 #include <linux/module.h>
   8 #include <linux/init.h>
   9 #include <linux/errno.h>
  10 #include <linux/kernel.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/platform_device.h>
  13 #include <linux/scatterlist.h>
  14 #include <linux/crypto.h>
  15 #include <linux/kthread.h>
  16 #include <linux/rtnetlink.h>
  17 #include <linux/sched.h>
  18 #include <linux/of.h>
  19 #include <linux/io.h>
  20 #include <linux/bitops.h>
  21
  22 #include <crypto/algapi.h>
  23 #include <crypto/aead.h>
  24 #include <crypto/internal/aead.h>
  25 #include <crypto/aes.h>
  26 #include <crypto/internal/des.h>
  27 #include <crypto/hmac.h>
  28 #include <crypto/md5.h>
  29 #include <crypto/authenc.h>
  30 #include <crypto/skcipher.h>
  31 #include <crypto/hash.h>
  32 #include <crypto/sha1.h>
  33 #include <crypto/sha2.h>
  34 #include <crypto/sha3.h>
  35
  36 #include "util.h"
  37 #include "cipher.h"
  38 #include "spu.h"
  39 #include "spum.h"
  40 #include "spu2.h"
  41
  42 /* ================= Device Structure ================== */
  43
  44 struct bcm_device_private iproc_priv;
  45
  46 /* ==================== Parameters ===================== */
  47
  48 int flow_debug_logging;
  49 module_param(flow_debug_logging, int, 0644);
  50 MODULE_PARM_DESC(flow_debug_logging, "Enable Flow Debug Logging");
  51
  52 int packet_debug_logging;
  53 module_param(packet_debug_logging, int, 0644);
  54 MODULE_PARM_DESC(packet_debug_logging, "Enable Packet Debug Logging");
  55
  56 int debug_logging_sleep;
  57 module_param(debug_logging_sleep, int, 0644);
  58 MODULE_PARM_DESC(debug_logging_sleep, "Packet Debug Logging Sleep");
  59
  60 /*
  61  * The value of these module parameters is used to set the priority for each
  62  * algo type when this driver registers algos with the kernel crypto API.
  63  * To use a priority other than the default, set the priority in the insmod or
  64  * modprobe. Changing the module priority after init time has no effect.
  65  *
  66  * The default priorities are chosen to be lower (less preferred) than ARMv8 CE
  67  * algos, but more preferred than generic software algos.
  68  */
  69 static int cipher_pri = 150;
  70 module_param(cipher_pri, int, 0644);
  71 MODULE_PARM_DESC(cipher_pri, "Priority for cipher algos");
  72
  73 static int hash_pri = 100;
  74 module_param(hash_pri, int, 0644);
  75 MODULE_PARM_DESC(hash_pri, "Priority for hash algos");
  76
  77 static int aead_pri = 150;
  78 module_param(aead_pri, int, 0644);
  79 MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos");
  80
  81 /* A type 3 BCM header, expected to precede the SPU header for SPU-M.
  82  * Bits 3 and 4 in the first byte encode the channel number (the dma ringset).
  83  * 0x60 - ring 0
  84  * 0x68 - ring 1
  85  * 0x70 - ring 2
  86  * 0x78 - ring 3
  87  */
  88 static char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 };
  89 /*
  90  * Some SPU hw does not use BCM header on SPU messages. So BCM_HDR_LEN
  91  * is set dynamically after reading SPU type from device tree.
  92  */
  93 #define BCM_HDR_LEN  iproc_priv.bcm_hdr_len
  94
  95 /* min and max time to sleep before retrying when mbox queue is full. usec */
  96 #define MBOX_SLEEP_MIN  800
  97 #define MBOX_SLEEP_MAX 1000
  98
  99 /**
 100  * select_channel() - Select a SPU channel to handle a crypto request. Selects
 101  * channel in round robin order.
 102  *
 103  * Return:  channel index
 104  */
 105 static u8 select_channel(void)
 106 {
 107         u8 chan_idx = atomic_inc_return(&iproc_priv.next_chan);
 108
 109         return chan_idx % iproc_priv.spu.num_chan;
 110 }
 111
 112 /**
 113  * spu_skcipher_rx_sg_create() - Build up the scatterlist of buffers used to
 114  * receive a SPU response message for an skcipher request. Includes buffers to
 115  * catch SPU message headers and the response data.
 116  * @mssg:       mailbox message containing the receive sg
 117  * @rctx:       crypto request context
 118  * @rx_frag_num: number of scatterlist elements required to hold the
 119  *              SPU response message
 120  * @chunksize:  Number of bytes of response data expected
 121  * @stat_pad_len: Number of bytes required to pad the STAT field to
 122  *              a 4-byte boundary
 123  *
 124  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
 125  * when the request completes, whether the request is handled successfully or
 126  * there is an error.
 127  *
 128  * Returns:
 129  *   0 if successful
 130  *   < 0 if an error
 131  */
 132 static int
 133 spu_skcipher_rx_sg_create(struct brcm_message *mssg,
 134                             struct iproc_reqctx_s *rctx,
 135                             u8 rx_frag_num,
 136                             unsigned int chunksize, u32 stat_pad_len)
 137 {
 138         struct spu_hw *spu = &iproc_priv.spu;
 139         struct scatterlist *sg; /* used to build sgs in mbox message */
 140         struct iproc_ctx_s *ctx = rctx->ctx;
 141         u32 datalen;            /* Number of bytes of response data expected */
 142
 143         mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist),
 144                                 rctx->gfp);
 145         if (!mssg->spu.dst)
 146                 return -ENOMEM;
 147
 148         sg = mssg->spu.dst;
 149         sg_init_table(sg, rx_frag_num);
 150         /* Space for SPU message header */
 151         sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
 152
 153         /* If XTS tweak in payload, add buffer to receive encrypted tweak */
 154         if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
 155             spu->spu_xts_tweak_in_payload())
 156                 sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak,
 157                            SPU_XTS_TWEAK_SIZE);
 158
 159         /* Copy in each dst sg entry from request, up to chunksize */
 160         datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip,
 161                                  rctx->dst_nents, chunksize);
 162         if (datalen < chunksize) {
 163                 pr_err("%s(): failed to copy dst sg to mbox msg. chunksize %u, datalen %u",
 164                        __func__, chunksize, datalen);
 165                 return -EFAULT;
 166         }
 167
 168         if (stat_pad_len)
 169                 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
 170
 171         memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
 172         sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
 173
 174         return 0;
 175 }
 176
 177 /**
 178  * spu_skcipher_tx_sg_create() - Build up the scatterlist of buffers used to
 179  * send a SPU request message for an skcipher request. Includes SPU message
 180  * headers and the request data.
 181  * @mssg:       mailbox message containing the transmit sg
 182  * @rctx:       crypto request context
 183  * @tx_frag_num: number of scatterlist elements required to construct the
 184  *              SPU request message
 185  * @chunksize:  Number of bytes of request data
 186  * @pad_len:    Number of pad bytes
 187  *
 188  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
 189  * when the request completes, whether the request is handled successfully or
 190  * there is an error.
 191  *
 192  * Returns:
 193  *   0 if successful
 194  *   < 0 if an error
 195  */
 196 static int
 197 spu_skcipher_tx_sg_create(struct brcm_message *mssg,
 198                             struct iproc_reqctx_s *rctx,
 199                             u8 tx_frag_num, unsigned int chunksize, u32 pad_len)
 200 {
 201         struct spu_hw *spu = &iproc_priv.spu;
 202         struct scatterlist *sg; /* used to build sgs in mbox message */
 203         struct iproc_ctx_s *ctx = rctx->ctx;
 204         u32 datalen;            /* Number of bytes of response data expected */
 205         u32 stat_len;
 206
 207         mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist),
 208                                 rctx->gfp);
 209         if (unlikely(!mssg->spu.src))
 210                 return -ENOMEM;
 211
 212         sg = mssg->spu.src;
 213         sg_init_table(sg, tx_frag_num);
 214
 215         sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
 216                    BCM_HDR_LEN + ctx->spu_req_hdr_len);
 217
 218         /* if XTS tweak in payload, copy from IV (where crypto API puts it) */
 219         if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
 220             spu->spu_xts_tweak_in_payload())
 221                 sg_set_buf(sg++, rctx->msg_buf.iv_ctr, SPU_XTS_TWEAK_SIZE);
 222
 223         /* Copy in each src sg entry from request, up to chunksize */
 224         datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
 225                                  rctx->src_nents, chunksize);
 226         if (unlikely(datalen < chunksize)) {
 227                 pr_err("%s(): failed to copy src sg to mbox msg",
 228                        __func__);
 229                 return -EFAULT;
 230         }
 231
 232         if (pad_len)
 233                 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
 234
 235         stat_len = spu->spu_tx_status_len();
 236         if (stat_len) {
 237                 memset(rctx->msg_buf.tx_stat, 0, stat_len);
 238                 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
 239         }
 240         return 0;
 241 }
 242
 243 static int mailbox_send_message(struct brcm_message *mssg, u32 flags,
 244                                 u8 chan_idx)
 245 {
 246         int err;
 247         int retry_cnt = 0;
 248         struct device *dev = &(iproc_priv.pdev->dev);
 249
 250         err = mbox_send_message(iproc_priv.mbox[chan_idx], mssg);
 251         if (flags & CRYPTO_TFM_REQ_MAY_SLEEP) {
 252                 while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) {
 253                         /*
 254                          * Mailbox queue is full. Since MAY_SLEEP is set, assume
 255                          * not in atomic context and we can wait and try again.
 256                          */
 257                         retry_cnt++;
 258                         usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX);
 259                         err = mbox_send_message(iproc_priv.mbox[chan_idx],
 260                                                 mssg);
 261                         atomic_inc(&iproc_priv.mb_no_spc);
 262                 }
 263         }
 264         if (err < 0) {
 265                 atomic_inc(&iproc_priv.mb_send_fail);
 266                 return err;
 267         }
 268
 269         /* Check error returned by mailbox controller */
 270         err = mssg->error;
 271         if (unlikely(err < 0)) {
 272                 dev_err(dev, "message error %d", err);
 273                 /* Signal txdone for mailbox channel */
 274         }
 275
 276         /* Signal txdone for mailbox channel */
 277         mbox_client_txdone(iproc_priv.mbox[chan_idx], err);
 278         return err;
 279 }
 280
 281 /**
 282  * handle_skcipher_req() - Submit as much of a block cipher request as fits in
 283  * a single SPU request message, starting at the current position in the request
 284  * data.
 285  * @rctx:       Crypto request context
 286  *
 287  * This may be called on the crypto API thread, or, when a request is so large
 288  * it must be broken into multiple SPU messages, on the thread used to invoke
 289  * the response callback. When requests are broken into multiple SPU
 290  * messages, we assume subsequent messages depend on previous results, and
 291  * thus always wait for previous results before submitting the next message.
 292  * Because requests are submitted in lock step like this, there is no need
 293  * to synchronize access to request data structures.
 294  *
 295  * Return: -EINPROGRESS: request has been accepted and result will be returned
 296  *                       asynchronously
 297  *         Any other value indicates an error
 298  */
 299 static int handle_skcipher_req(struct iproc_reqctx_s *rctx)
 300 {
 301         struct spu_hw *spu = &iproc_priv.spu;
 302         struct crypto_async_request *areq = rctx->parent;
 303         struct skcipher_request *req =
 304             container_of(areq, struct skcipher_request, base);
 305         struct iproc_ctx_s *ctx = rctx->ctx;
 306         struct spu_cipher_parms cipher_parms;
 307         int err;
 308         unsigned int chunksize; /* Num bytes of request to submit */
 309         int remaining;  /* Bytes of request still to process */
 310         int chunk_start;        /* Beginning of data for current SPU msg */
 311
 312         /* IV or ctr value to use in this SPU msg */
 313         u8 local_iv_ctr[MAX_IV_SIZE];
 314         u32 stat_pad_len;       /* num bytes to align status field */
 315         u32 pad_len;            /* total length of all padding */
 316         struct brcm_message *mssg;      /* mailbox message */
 317
 318         /* number of entries in src and dst sg in mailbox message. */
 319         u8 rx_frag_num = 2;     /* response header and STATUS */
 320         u8 tx_frag_num = 1;     /* request header */
 321
 322         flow_log("%s\n", __func__);
 323
 324         cipher_parms.alg = ctx->cipher.alg;
 325         cipher_parms.mode = ctx->cipher.mode;
 326         cipher_parms.type = ctx->cipher_type;
 327         cipher_parms.key_len = ctx->enckeylen;
 328         cipher_parms.key_buf = ctx->enckey;
 329         cipher_parms.iv_buf = local_iv_ctr;
 330         cipher_parms.iv_len = rctx->iv_ctr_len;
 331
 332         mssg = &rctx->mb_mssg;
 333         chunk_start = rctx->src_sent;
 334         remaining = rctx->total_todo - chunk_start;
 335
 336         /* determine the chunk we are breaking off and update the indexes */
 337         if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
 338             (remaining > ctx->max_payload))
 339                 chunksize = ctx->max_payload;
 340         else
 341                 chunksize = remaining;
 342
 343         rctx->src_sent += chunksize;
 344         rctx->total_sent = rctx->src_sent;
 345
 346         /* Count number of sg entries to be included in this request */
 347         rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize);
 348         rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize);
 349
 350         if ((ctx->cipher.mode == CIPHER_MODE_CBC) &&
 351             rctx->is_encrypt && chunk_start)
 352                 /*
 353                  * Encrypting non-first first chunk. Copy last block of
 354                  * previous result to IV for this chunk.
 355                  */
 356                 sg_copy_part_to_buf(req->dst, rctx->msg_buf.iv_ctr,
 357                                     rctx->iv_ctr_len,
 358                                     chunk_start - rctx->iv_ctr_len);
 359
 360         if (rctx->iv_ctr_len) {
 361                 /* get our local copy of the iv */
 362                 __builtin_memcpy(local_iv_ctr, rctx->msg_buf.iv_ctr,
 363                                  rctx->iv_ctr_len);
 364
 365                 /* generate the next IV if possible */
 366                 if ((ctx->cipher.mode == CIPHER_MODE_CBC) &&
 367                     !rctx->is_encrypt) {
 368                         /*
 369                          * CBC Decrypt: next IV is the last ciphertext block in
 370                          * this chunk
 371                          */
 372                         sg_copy_part_to_buf(req->src, rctx->msg_buf.iv_ctr,
 373                                             rctx->iv_ctr_len,
 374                                             rctx->src_sent - rctx->iv_ctr_len);
 375                 } else if (ctx->cipher.mode == CIPHER_MODE_CTR) {
 376                         /*
 377                          * The SPU hardware increments the counter once for
 378                          * each AES block of 16 bytes. So update the counter
 379                          * for the next chunk, if there is one. Note that for
 380                          * this chunk, the counter has already been copied to
 381                          * local_iv_ctr. We can assume a block size of 16,
 382                          * because we only support CTR mode for AES, not for
 383                          * any other cipher alg.
 384                          */
 385                         add_to_ctr(rctx->msg_buf.iv_ctr, chunksize >> 4);
 386                 }
 387         }
 388
 389         if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
 390                 flow_log("max_payload infinite\n");
 391         else
 392                 flow_log("max_payload %u\n", ctx->max_payload);
 393
 394         flow_log("sent:%u start:%u remains:%u size:%u\n",
 395                  rctx->src_sent, chunk_start, remaining, chunksize);
 396
 397         /* Copy SPU header template created at setkey time */
 398         memcpy(rctx->msg_buf.bcm_spu_req_hdr, ctx->bcm_spu_req_hdr,
 399                sizeof(rctx->msg_buf.bcm_spu_req_hdr));
 400
 401         spu->spu_cipher_req_finish(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
 402                                    ctx->spu_req_hdr_len, !(rctx->is_encrypt),
 403                                    &cipher_parms, chunksize);
 404
 405         atomic64_add(chunksize, &iproc_priv.bytes_out);
 406
 407         stat_pad_len = spu->spu_wordalign_padlen(chunksize);
 408         if (stat_pad_len)
 409                 rx_frag_num++;
 410         pad_len = stat_pad_len;
 411         if (pad_len) {
 412                 tx_frag_num++;
 413                 spu->spu_request_pad(rctx->msg_buf.spu_req_pad, 0,
 414                                      0, ctx->auth.alg, ctx->auth.mode,
 415                                      rctx->total_sent, stat_pad_len);
 416         }
 417
 418         spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
 419                               ctx->spu_req_hdr_len);
 420         packet_log("payload:\n");
 421         dump_sg(rctx->src_sg, rctx->src_skip, chunksize);
 422         packet_dump("   pad: ", rctx->msg_buf.spu_req_pad, pad_len);
 423
 424         /*
 425          * Build mailbox message containing SPU request msg and rx buffers
 426          * to catch response message
 427          */
 428         memset(mssg, 0, sizeof(*mssg));
 429         mssg->type = BRCM_MESSAGE_SPU;
 430         mssg->ctx = rctx;       /* Will be returned in response */
 431
 432         /* Create rx scatterlist to catch result */
 433         rx_frag_num += rctx->dst_nents;
 434
 435         if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
 436             spu->spu_xts_tweak_in_payload())
 437                 rx_frag_num++;  /* extra sg to insert tweak */
 438
 439         err = spu_skcipher_rx_sg_create(mssg, rctx, rx_frag_num, chunksize,
 440                                           stat_pad_len);
 441         if (err)
 442                 return err;
 443
 444         /* Create tx scatterlist containing SPU request message */
 445         tx_frag_num += rctx->src_nents;
 446         if (spu->spu_tx_status_len())
 447                 tx_frag_num++;
 448
 449         if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
 450             spu->spu_xts_tweak_in_payload())
 451                 tx_frag_num++;  /* extra sg to insert tweak */
 452
 453         err = spu_skcipher_tx_sg_create(mssg, rctx, tx_frag_num, chunksize,
 454                                           pad_len);
 455         if (err)
 456                 return err;
 457
 458         err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx);
 459         if (unlikely(err < 0))
 460                 return err;
 461
 462         return -EINPROGRESS;
 463 }
 464
 465 /**
 466  * handle_skcipher_resp() - Process a block cipher SPU response. Updates the
 467  * total received count for the request and updates global stats.
 468  * @rctx:       Crypto request context
 469  */
 470 static void handle_skcipher_resp(struct iproc_reqctx_s *rctx)
 471 {
 472         struct spu_hw *spu = &iproc_priv.spu;
 473         struct crypto_async_request *areq = rctx->parent;
 474         struct skcipher_request *req = skcipher_request_cast(areq);
 475         struct iproc_ctx_s *ctx = rctx->ctx;
 476         u32 payload_len;
 477
 478         /* See how much data was returned */
 479         payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr);
 480
 481         /*
 482          * In XTS mode, the first SPU_XTS_TWEAK_SIZE bytes may be the
 483          * encrypted tweak ("i") value; we don't count those.
 484          */
 485         if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
 486             spu->spu_xts_tweak_in_payload() &&
 487             (payload_len >= SPU_XTS_TWEAK_SIZE))
 488                 payload_len -= SPU_XTS_TWEAK_SIZE;
 489
 490         atomic64_add(payload_len, &iproc_priv.bytes_in);
 491
 492         flow_log("%s() offset: %u, bd_len: %u BD:\n",
 493                  __func__, rctx->total_received, payload_len);
 494
 495         dump_sg(req->dst, rctx->total_received, payload_len);
 496
 497         rctx->total_received += payload_len;
 498         if (rctx->total_received == rctx->total_todo) {
 499                 atomic_inc(&iproc_priv.op_counts[SPU_OP_CIPHER]);
 500                 atomic_inc(
 501                    &iproc_priv.cipher_cnt[ctx->cipher.alg][ctx->cipher.mode]);
 502         }
 503 }
 504
 505 /**
 506  * spu_ahash_rx_sg_create() - Build up the scatterlist of buffers used to
 507  * receive a SPU response message for an ahash request.
 508  * @mssg:       mailbox message containing the receive sg
 509  * @rctx:       crypto request context
 510  * @rx_frag_num: number of scatterlist elements required to hold the
 511  *              SPU response message
 512  * @digestsize: length of hash digest, in bytes
 513  * @stat_pad_len: Number of bytes required to pad the STAT field to
 514  *              a 4-byte boundary
 515  *
 516  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
 517  * when the request completes, whether the request is handled successfully or
 518  * there is an error.
 519  *
 520  * Return:
 521  *   0 if successful
 522  *   < 0 if an error
 523  */
 524 static int
 525 spu_ahash_rx_sg_create(struct brcm_message *mssg,
 526                        struct iproc_reqctx_s *rctx,
 527                        u8 rx_frag_num, unsigned int digestsize,
 528                        u32 stat_pad_len)
 529 {
 530         struct spu_hw *spu = &iproc_priv.spu;
 531         struct scatterlist *sg; /* used to build sgs in mbox message */
 532         struct iproc_ctx_s *ctx = rctx->ctx;
 533
 534         mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist),
 535                                 rctx->gfp);
 536         if (!mssg->spu.dst)
 537                 return -ENOMEM;
 538
 539         sg = mssg->spu.dst;
 540         sg_init_table(sg, rx_frag_num);
 541         /* Space for SPU message header */
 542         sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
 543
 544         /* Space for digest */
 545         sg_set_buf(sg++, rctx->msg_buf.digest, digestsize);
 546
 547         if (stat_pad_len)
 548                 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
 549
 550         memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
 551         sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
 552         return 0;
 553 }
 554
 555 /**
 556  * spu_ahash_tx_sg_create() -  Build up the scatterlist of buffers used to send
 557  * a SPU request message for an ahash request. Includes SPU message headers and
 558  * the request data.
 559  * @mssg:       mailbox message containing the transmit sg
 560  * @rctx:       crypto request context
 561  * @tx_frag_num: number of scatterlist elements required to construct the
 562  *              SPU request message
 563  * @spu_hdr_len: length in bytes of SPU message header
 564  * @hash_carry_len: Number of bytes of data carried over from previous req
 565  * @new_data_len: Number of bytes of new request data
 566  * @pad_len:    Number of pad bytes
 567  *
 568  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
 569  * when the request completes, whether the request is handled successfully or
 570  * there is an error.
 571  *
 572  * Return:
 573  *   0 if successful
 574  *   < 0 if an error
 575  */
 576 static int
 577 spu_ahash_tx_sg_create(struct brcm_message *mssg,
 578                        struct iproc_reqctx_s *rctx,
 579                        u8 tx_frag_num,
 580                        u32 spu_hdr_len,
 581                        unsigned int hash_carry_len,
 582                        unsigned int new_data_len, u32 pad_len)
 583 {
 584         struct spu_hw *spu = &iproc_priv.spu;
 585         struct scatterlist *sg; /* used to build sgs in mbox message */
 586         u32 datalen;            /* Number of bytes of response data expected */
 587         u32 stat_len;
 588
 589         mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist),
 590                                 rctx->gfp);
 591         if (!mssg->spu.src)
 592                 return -ENOMEM;
 593
 594         sg = mssg->spu.src;
 595         sg_init_table(sg, tx_frag_num);
 596
 597         sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
 598                    BCM_HDR_LEN + spu_hdr_len);
 599
 600         if (hash_carry_len)
 601                 sg_set_buf(sg++, rctx->hash_carry, hash_carry_len);
 602
 603         if (new_data_len) {
 604                 /* Copy in each src sg entry from request, up to chunksize */
 605                 datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
 606                                          rctx->src_nents, new_data_len);
 607                 if (datalen < new_data_len) {
 608                         pr_err("%s(): failed to copy src sg to mbox msg",
 609                                __func__);
 610                         return -EFAULT;
 611                 }
 612         }
 613
 614         if (pad_len)
 615                 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
 616
 617         stat_len = spu->spu_tx_status_len();
 618         if (stat_len) {
 619                 memset(rctx->msg_buf.tx_stat, 0, stat_len);
 620                 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
 621         }
 622
 623         return 0;
 624 }
 625
 626 /**
 627  * handle_ahash_req() - Process an asynchronous hash request from the crypto
 628  * API.
 629  * @rctx:  Crypto request context
 630  *
 631  * Builds a SPU request message embedded in a mailbox message and submits the
 632  * mailbox message on a selected mailbox channel. The SPU request message is
 633  * constructed as a scatterlist, including entries from the crypto API's
 634  * src scatterlist to avoid copying the data to be hashed. This function is
 635  * called either on the thread from the crypto API, or, in the case that the
 636  * crypto API request is too large to fit in a single SPU request message,
 637  * on the thread that invokes the receive callback with a response message.
 638  * Because some operations require the response from one chunk before the next
 639  * chunk can be submitted, we always wait for the response for the previous
 640  * chunk before submitting the next chunk. Because requests are submitted in
 641  * lock step like this, there is no need to synchronize access to request data
 642  * structures.
 643  *
 644  * Return:
 645  *   -EINPROGRESS: request has been submitted to SPU and response will be
 646  *                 returned asynchronously
 647  *   -EAGAIN:      non-final request included a small amount of data, which for
 648  *                 efficiency we did not submit to the SPU, but instead stored
 649  *                 to be submitted to the SPU with the next part of the request
 650  *   other:        an error code
 651  */
 652 static int handle_ahash_req(struct iproc_reqctx_s *rctx)
 653 {
 654         struct spu_hw *spu = &iproc_priv.spu;
 655         struct crypto_async_request *areq = rctx->parent;
 656         struct ahash_request *req = ahash_request_cast(areq);
 657         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 658         struct crypto_tfm *tfm = crypto_ahash_tfm(ahash);
 659         unsigned int blocksize = crypto_tfm_alg_blocksize(tfm);
 660         struct iproc_ctx_s *ctx = rctx->ctx;
 661
 662         /* number of bytes still to be hashed in this req */
 663         unsigned int nbytes_to_hash = 0;
 664         int err;
 665         unsigned int chunksize = 0;     /* length of hash carry + new data */
 666         /*
 667          * length of new data, not from hash carry, to be submitted in
 668          * this hw request
 669          */
 670         unsigned int new_data_len;
 671
 672         unsigned int __maybe_unused chunk_start = 0;
 673         u32 db_size;     /* Length of data field, incl gcm and hash padding */
 674         int pad_len = 0; /* total pad len, including gcm, hash, stat padding */
 675         u32 data_pad_len = 0;   /* length of GCM/CCM padding */
 676         u32 stat_pad_len = 0;   /* length of padding to align STATUS word */
 677         struct brcm_message *mssg;      /* mailbox message */
 678         struct spu_request_opts req_opts;
 679         struct spu_cipher_parms cipher_parms;
 680         struct spu_hash_parms hash_parms;
 681         struct spu_aead_parms aead_parms;
 682         unsigned int local_nbuf;
 683         u32 spu_hdr_len;
 684         unsigned int digestsize;
 685         u16 rem = 0;
 686
 687         /*
 688          * number of entries in src and dst sg. Always includes SPU msg header.
 689          * rx always includes a buffer to catch digest and STATUS.
 690          */
 691         u8 rx_frag_num = 3;
 692         u8 tx_frag_num = 1;
 693
 694         flow_log("total_todo %u, total_sent %u\n",
 695                  rctx->total_todo, rctx->total_sent);
 696
 697         memset(&req_opts, 0, sizeof(req_opts));
 698         memset(&cipher_parms, 0, sizeof(cipher_parms));
 699         memset(&hash_parms, 0, sizeof(hash_parms));
 700         memset(&aead_parms, 0, sizeof(aead_parms));
 701
 702         req_opts.bd_suppress = true;
 703         hash_parms.alg = ctx->auth.alg;
 704         hash_parms.mode = ctx->auth.mode;
 705         hash_parms.type = HASH_TYPE_NONE;
 706         hash_parms.key_buf = (u8 *)ctx->authkey;
 707         hash_parms.key_len = ctx->authkeylen;
 708
 709         /*
 710          * For hash algorithms below assignment looks bit odd but
 711          * it's needed for AES-XCBC and AES-CMAC hash algorithms
 712          * to differentiate between 128, 192, 256 bit key values.
 713          * Based on the key values, hash algorithm is selected.
 714          * For example for 128 bit key, hash algorithm is AES-128.
 715          */
 716         cipher_parms.type = ctx->cipher_type;
 717
 718         mssg = &rctx->mb_mssg;
 719         chunk_start = rctx->src_sent;
 720
 721         /*
 722          * Compute the amount remaining to hash. This may include data
 723          * carried over from previous requests.
 724          */
 725         nbytes_to_hash = rctx->total_todo - rctx->total_sent;
 726         chunksize = nbytes_to_hash;
 727         if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
 728             (chunksize > ctx->max_payload))
 729                 chunksize = ctx->max_payload;
 730
 731         /*
 732          * If this is not a final request and the request data is not a multiple
 733          * of a full block, then simply park the extra data and prefix it to the
 734          * data for the next request.
 735          */
 736         if (!rctx->is_final) {
 737                 u8 *dest = rctx->hash_carry + rctx->hash_carry_len;
 738                 u16 new_len;  /* len of data to add to hash carry */
 739
 740                 rem = chunksize % blocksize;   /* remainder */
 741                 if (rem) {
 742                         /* chunksize not a multiple of blocksize */
 743                         chunksize -= rem;
 744                         if (chunksize == 0) {
 745                                 /* Don't have a full block to submit to hw */
 746                                 new_len = rem - rctx->hash_carry_len;
 747                                 sg_copy_part_to_buf(req->src, dest, new_len,
 748                                                     rctx->src_sent);
 749                                 rctx->hash_carry_len = rem;
 750                                 flow_log("Exiting with hash carry len: %u\n",
 751                                          rctx->hash_carry_len);
 752                                 packet_dump("  buf: ",
 753                                             rctx->hash_carry,
 754                                             rctx->hash_carry_len);
 755                                 return -EAGAIN;
 756                         }
 757                 }
 758         }
 759
 760         /* if we have hash carry, then prefix it to the data in this request */
 761         local_nbuf = rctx->hash_carry_len;
 762         rctx->hash_carry_len = 0;
 763         if (local_nbuf)
 764                 tx_frag_num++;
 765         new_data_len = chunksize - local_nbuf;
 766
 767         /* Count number of sg entries to be used in this request */
 768         rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip,
 769                                        new_data_len);
 770
 771         /* AES hashing keeps key size in type field, so need to copy it here */
 772         if (hash_parms.alg == HASH_ALG_AES)
 773                 hash_parms.type = (enum hash_type)cipher_parms.type;
 774         else
 775                 hash_parms.type = spu->spu_hash_type(rctx->total_sent);
 776
 777         digestsize = spu->spu_digest_size(ctx->digestsize, ctx->auth.alg,
 778                                           hash_parms.type);
 779         hash_parms.digestsize = digestsize;
 780
 781         /* update the indexes */
 782         rctx->total_sent += chunksize;
 783         /* if you sent a prebuf then that wasn't from this req->src */
 784         rctx->src_sent += new_data_len;
 785
 786         if ((rctx->total_sent == rctx->total_todo) && rctx->is_final)
 787                 hash_parms.pad_len = spu->spu_hash_pad_len(hash_parms.alg,
 788                                                            hash_parms.mode,
 789                                                            chunksize,
 790                                                            blocksize);
 791
 792         /*
 793          * If a non-first chunk, then include the digest returned from the
 794          * previous chunk so that hw can add to it (except for AES types).
 795          */
 796         if ((hash_parms.type == HASH_TYPE_UPDT) &&
 797             (hash_parms.alg != HASH_ALG_AES)) {
 798                 hash_parms.key_buf = rctx->incr_hash;
 799                 hash_parms.key_len = digestsize;
 800         }
 801
 802         atomic64_add(chunksize, &iproc_priv.bytes_out);
 803
 804         flow_log("%s() final: %u nbuf: %u ",
 805                  __func__, rctx->is_final, local_nbuf);
 806
 807         if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
 808                 flow_log("max_payload infinite\n");
 809         else
 810                 flow_log("max_payload %u\n", ctx->max_payload);
 811
 812         flow_log("chunk_start: %u chunk_size: %u\n", chunk_start, chunksize);
 813
 814         /* Prepend SPU header with type 3 BCM header */
 815         memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
 816
 817         hash_parms.prebuf_len = local_nbuf;
 818         spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr +
 819                                               BCM_HDR_LEN,
 820                                               &req_opts, &cipher_parms,
 821                                               &hash_parms, &aead_parms,
 822                                               new_data_len);
 823
 824         if (spu_hdr_len == 0) {
 825                 pr_err("Failed to create SPU request header\n");
 826                 return -EFAULT;
 827         }
 828
 829         /*
 830          * Determine total length of padding required. Put all padding in one
 831          * buffer.
 832          */
 833         data_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode, chunksize);
 834         db_size = spu_real_db_size(0, 0, local_nbuf, new_data_len,
 835                                    0, 0, hash_parms.pad_len);
 836         if (spu->spu_tx_status_len())
 837                 stat_pad_len = spu->spu_wordalign_padlen(db_size);
 838         if (stat_pad_len)
 839                 rx_frag_num++;
 840         pad_len = hash_parms.pad_len + data_pad_len + stat_pad_len;
 841         if (pad_len) {
 842                 tx_frag_num++;
 843                 spu->spu_request_pad(rctx->msg_buf.spu_req_pad, data_pad_len,
 844                                      hash_parms.pad_len, ctx->auth.alg,
 845                                      ctx->auth.mode, rctx->total_sent,
 846                                      stat_pad_len);
 847         }
 848
 849         spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
 850                               spu_hdr_len);
 851         packet_dump("    prebuf: ", rctx->hash_carry, local_nbuf);
 852         flow_log("Data:\n");
 853         dump_sg(rctx->src_sg, rctx->src_skip, new_data_len);
 854         packet_dump("   pad: ", rctx->msg_buf.spu_req_pad, pad_len);
 855
 856         /*
 857          * Build mailbox message containing SPU request msg and rx buffers
 858          * to catch response message
 859          */
 860         memset(mssg, 0, sizeof(*mssg));
 861         mssg->type = BRCM_MESSAGE_SPU;
 862         mssg->ctx = rctx;       /* Will be returned in response */
 863
 864         /* Create rx scatterlist to catch result */
 865         err = spu_ahash_rx_sg_create(mssg, rctx, rx_frag_num, digestsize,
 866                                      stat_pad_len);
 867         if (err)
 868                 return err;
 869
 870         /* Create tx scatterlist containing SPU request message */
 871         tx_frag_num += rctx->src_nents;
 872         if (spu->spu_tx_status_len())
 873                 tx_frag_num++;
 874         err = spu_ahash_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len,
 875                                      local_nbuf, new_data_len, pad_len);
 876         if (err)
 877                 return err;
 878
 879         err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx);
 880         if (unlikely(err < 0))
 881                 return err;
 882
 883         return -EINPROGRESS;
 884 }
 885
 886 /**
 887  * spu_hmac_outer_hash() - Request synchonous software compute of the outer hash
 888  * for an HMAC request.
 889  * @req:  The HMAC request from the crypto API
 890  * @ctx:  The session context
 891  *
 892  * Return: 0 if synchronous hash operation successful
 893  *         -EINVAL if the hash algo is unrecognized
 894  *         any other value indicates an error
 895  */
 896 static int spu_hmac_outer_hash(struct ahash_request *req,
 897                                struct iproc_ctx_s *ctx)
 898 {
 899         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 900         unsigned int blocksize =
 901                 crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
 902         int rc;
 903
 904         switch (ctx->auth.alg) {
 905         case HASH_ALG_MD5:
 906                 rc = do_shash("md5", req->result, ctx->opad, blocksize,
 907                               req->result, ctx->digestsize, NULL, 0);
 908                 break;
 909         case HASH_ALG_SHA1:
 910                 rc = do_shash("sha1", req->result, ctx->opad, blocksize,
 911                               req->result, ctx->digestsize, NULL, 0);
 912                 break;
 913         case HASH_ALG_SHA224:
 914                 rc = do_shash("sha224", req->result, ctx->opad, blocksize,
 915                               req->result, ctx->digestsize, NULL, 0);
 916                 break;
 917         case HASH_ALG_SHA256:
 918                 rc = do_shash("sha256", req->result, ctx->opad, blocksize,
 919                               req->result, ctx->digestsize, NULL, 0);
 920                 break;
 921         case HASH_ALG_SHA384:
 922                 rc = do_shash("sha384", req->result, ctx->opad, blocksize,
 923                               req->result, ctx->digestsize, NULL, 0);
 924                 break;
 925         case HASH_ALG_SHA512:
 926                 rc = do_shash("sha512", req->result, ctx->opad, blocksize,
 927                               req->result, ctx->digestsize, NULL, 0);
 928                 break;
 929         default:
 930                 pr_err("%s() Error : unknown hmac type\n", __func__);
 931                 rc = -EINVAL;
 932         }
 933         return rc;
 934 }
 935
 936 /**
 937  * ahash_req_done() - Process a hash result from the SPU hardware.
 938  * @rctx: Crypto request context
 939  *
 940  * Return: 0 if successful
 941  *         < 0 if an error
 942  */
 943 static int ahash_req_done(struct iproc_reqctx_s *rctx)
 944 {
 945         struct spu_hw *spu = &iproc_priv.spu;
 946         struct crypto_async_request *areq = rctx->parent;
 947         struct ahash_request *req = ahash_request_cast(areq);
 948         struct iproc_ctx_s *ctx = rctx->ctx;
 949         int err;
 950
 951         memcpy(req->result, rctx->msg_buf.digest, ctx->digestsize);
 952
 953         if (spu->spu_type == SPU_TYPE_SPUM) {
 954                 /* byte swap the output from the UPDT function to network byte
 955                  * order
 956                  */
 957                 if (ctx->auth.alg == HASH_ALG_MD5) {
 958                         __swab32s((u32 *)req->result);
 959                         __swab32s(((u32 *)req->result) + 1);
 960                         __swab32s(((u32 *)req->result) + 2);
 961                         __swab32s(((u32 *)req->result) + 3);
 962                         __swab32s(((u32 *)req->result) + 4);
 963                 }
 964         }
 965
 966         flow_dump("  digest ", req->result, ctx->digestsize);
 967
 968         /* if this an HMAC then do the outer hash */
 969         if (rctx->is_sw_hmac) {
 970                 err = spu_hmac_outer_hash(req, ctx);
 971                 if (err < 0)
 972                         return err;
 973                 flow_dump("  hmac: ", req->result, ctx->digestsize);
 974         }
 975
 976         if (rctx->is_sw_hmac || ctx->auth.mode == HASH_MODE_HMAC) {
 977                 atomic_inc(&iproc_priv.op_counts[SPU_OP_HMAC]);
 978                 atomic_inc(&iproc_priv.hmac_cnt[ctx->auth.alg]);
 979         } else {
 980                 atomic_inc(&iproc_priv.op_counts[SPU_OP_HASH]);
 981                 atomic_inc(&iproc_priv.hash_cnt[ctx->auth.alg]);
 982         }
 983
 984         return 0;
 985 }
 986
 987 /**
 988  * handle_ahash_resp() - Process a SPU response message for a hash request.
 989  * Checks if the entire crypto API request has been processed, and if so,
 990  * invokes post processing on the result.
 991  * @rctx: Crypto request context
 992  */
 993 static void handle_ahash_resp(struct iproc_reqctx_s *rctx)
 994 {
 995         struct iproc_ctx_s *ctx = rctx->ctx;
 996         struct crypto_async_request *areq = rctx->parent;
 997         struct ahash_request *req = ahash_request_cast(areq);
 998         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 999         unsigned int blocksize =
1000                 crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
1001         /*
1002          * Save hash to use as input to next op if incremental. Might be copying
1003          * too much, but that's easier than figuring out actual digest size here
1004          */
1005         memcpy(rctx->incr_hash, rctx->msg_buf.digest, MAX_DIGEST_SIZE);
1006
1007         flow_log("%s() blocksize:%u digestsize:%u\n",
1008                  __func__, blocksize, ctx->digestsize);
1009
1010         atomic64_add(ctx->digestsize, &iproc_priv.bytes_in);
1011
1012         if (rctx->is_final && (rctx->total_sent == rctx->total_todo))
1013                 ahash_req_done(rctx);
1014 }
1015
1016 /**
1017  * spu_aead_rx_sg_create() - Build up the scatterlist of buffers used to receive
1018  * a SPU response message for an AEAD request. Includes buffers to catch SPU
1019  * message headers and the response data.
1020  * @mssg:       mailbox message containing the receive sg
1021  * @req:        Crypto API request
1022  * @rctx:       crypto request context
1023  * @rx_frag_num: number of scatterlist elements required to hold the
1024  *              SPU response message
1025  * @assoc_len:  Length of associated data included in the crypto request
1026  * @ret_iv_len: Length of IV returned in response
1027  * @resp_len:   Number of bytes of response data expected to be written to
1028  *              dst buffer from crypto API
1029  * @digestsize: Length of hash digest, in bytes
1030  * @stat_pad_len: Number of bytes required to pad the STAT field to
1031  *              a 4-byte boundary
1032  *
1033  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
1034  * when the request completes, whether the request is handled successfully or
1035  * there is an error.
1036  *
1037  * Returns:
1038  *   0 if successful
1039  *   < 0 if an error
1040  */
1041 static int spu_aead_rx_sg_create(struct brcm_message *mssg,
1042                                  struct aead_request *req,
1043                                  struct iproc_reqctx_s *rctx,
1044                                  u8 rx_frag_num,
1045                                  unsigned int assoc_len,
1046                                  u32 ret_iv_len, unsigned int resp_len,
1047                                  unsigned int digestsize, u32 stat_pad_len)
1048 {
1049         struct spu_hw *spu = &iproc_priv.spu;
1050         struct scatterlist *sg; /* used to build sgs in mbox message */
1051         struct iproc_ctx_s *ctx = rctx->ctx;
1052         u32 datalen;            /* Number of bytes of response data expected */
1053         u32 assoc_buf_len;
1054         u8 data_padlen = 0;
1055
1056         if (ctx->is_rfc4543) {
1057                 /* RFC4543: only pad after data, not after AAD */
1058                 data_padlen = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1059                                                           assoc_len + resp_len);
1060                 assoc_buf_len = assoc_len;
1061         } else {
1062                 data_padlen = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1063                                                           resp_len);
1064                 assoc_buf_len = spu->spu_assoc_resp_len(ctx->cipher.mode,
1065                                                 assoc_len, ret_iv_len,
1066                                                 rctx->is_encrypt);
1067         }
1068
1069         if (ctx->cipher.mode == CIPHER_MODE_CCM)
1070                 /* ICV (after data) must be in the next 32-bit word for CCM */
1071                 data_padlen += spu->spu_wordalign_padlen(assoc_buf_len +
1072                                                          resp_len +
1073                                                          data_padlen);
1074
1075         if (data_padlen)
1076                 /* have to catch gcm pad in separate buffer */
1077                 rx_frag_num++;
1078
1079         mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist),
1080                                 rctx->gfp);
1081         if (!mssg->spu.dst)
1082                 return -ENOMEM;
1083
1084         sg = mssg->spu.dst;
1085         sg_init_table(sg, rx_frag_num);
1086
1087         /* Space for SPU message header */
1088         sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
1089
1090         if (assoc_buf_len) {
1091                 /*
1092                  * Don't write directly to req->dst, because SPU may pad the
1093                  * assoc data in the response
1094                  */
1095                 memset(rctx->msg_buf.a.resp_aad, 0, assoc_buf_len);
1096                 sg_set_buf(sg++, rctx->msg_buf.a.resp_aad, assoc_buf_len);
1097         }
1098
1099         if (resp_len) {
1100                 /*
1101                  * Copy in each dst sg entry from request, up to chunksize.
1102                  * dst sg catches just the data. digest caught in separate buf.
1103                  */
1104                 datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip,
1105                                          rctx->dst_nents, resp_len);
1106                 if (datalen < (resp_len)) {
1107                         pr_err("%s(): failed to copy dst sg to mbox msg. expected len %u, datalen %u",
1108                                __func__, resp_len, datalen);
1109                         return -EFAULT;
1110                 }
1111         }
1112
1113         /* If GCM/CCM data is padded, catch padding in separate buffer */
1114         if (data_padlen) {
1115                 memset(rctx->msg_buf.a.gcmpad, 0, data_padlen);
1116                 sg_set_buf(sg++, rctx->msg_buf.a.gcmpad, data_padlen);
1117         }
1118
1119         /* Always catch ICV in separate buffer */
1120         sg_set_buf(sg++, rctx->msg_buf.digest, digestsize);
1121
1122         flow_log("stat_pad_len %u\n", stat_pad_len);
1123         if (stat_pad_len) {
1124                 memset(rctx->msg_buf.rx_stat_pad, 0, stat_pad_len);
1125                 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
1126         }
1127
1128         memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
1129         sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
1130
1131         return 0;
1132 }
1133
1134 /**
1135  * spu_aead_tx_sg_create() - Build up the scatterlist of buffers used to send a
1136  * SPU request message for an AEAD request. Includes SPU message headers and the
1137  * request data.
1138  * @mssg:       mailbox message containing the transmit sg
1139  * @rctx:       crypto request context
1140  * @tx_frag_num: number of scatterlist elements required to construct the
1141  *              SPU request message
1142  * @spu_hdr_len: length of SPU message header in bytes
1143  * @assoc:      crypto API associated data scatterlist
1144  * @assoc_len:  length of associated data
1145  * @assoc_nents: number of scatterlist entries containing assoc data
1146  * @aead_iv_len: length of AEAD IV, if included
1147  * @chunksize:  Number of bytes of request data
1148  * @aad_pad_len: Number of bytes of padding at end of AAD. For GCM/CCM.
1149  * @pad_len:    Number of pad bytes
1150  * @incl_icv:   If true, write separate ICV buffer after data and
1151  *              any padding
1152  *
1153  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
1154  * when the request completes, whether the request is handled successfully or
1155  * there is an error.
1156  *
1157  * Return:
1158  *   0 if successful
1159  *   < 0 if an error
1160  */
1161 static int spu_aead_tx_sg_create(struct brcm_message *mssg,
1162                                  struct iproc_reqctx_s *rctx,
1163                                  u8 tx_frag_num,
1164                                  u32 spu_hdr_len,
1165                                  struct scatterlist *assoc,
1166                                  unsigned int assoc_len,
1167                                  int assoc_nents,
1168                                  unsigned int aead_iv_len,
1169                                  unsigned int chunksize,
1170                                  u32 aad_pad_len, u32 pad_len, bool incl_icv)
1171 {
1172         struct spu_hw *spu = &iproc_priv.spu;
1173         struct scatterlist *sg; /* used to build sgs in mbox message */
1174         struct scatterlist *assoc_sg = assoc;
1175         struct iproc_ctx_s *ctx = rctx->ctx;
1176         u32 datalen;            /* Number of bytes of data to write */
1177         u32 written;            /* Number of bytes of data written */
1178         u32 assoc_offset = 0;
1179         u32 stat_len;
1180
1181         mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist),
1182                                 rctx->gfp);
1183         if (!mssg->spu.src)
1184                 return -ENOMEM;
1185
1186         sg = mssg->spu.src;
1187         sg_init_table(sg, tx_frag_num);
1188
1189         sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
1190                    BCM_HDR_LEN + spu_hdr_len);
1191
1192         if (assoc_len) {
1193                 /* Copy in each associated data sg entry from request */
1194                 written = spu_msg_sg_add(&sg, &assoc_sg, &assoc_offset,
1195                                          assoc_nents, assoc_len);
1196                 if (written < assoc_len) {
1197                         pr_err("%s(): failed to copy assoc sg to mbox msg",
1198                                __func__);
1199                         return -EFAULT;
1200                 }
1201         }
1202
1203         if (aead_iv_len)
1204                 sg_set_buf(sg++, rctx->msg_buf.iv_ctr, aead_iv_len);
1205
1206         if (aad_pad_len) {
1207                 memset(rctx->msg_buf.a.req_aad_pad, 0, aad_pad_len);
1208                 sg_set_buf(sg++, rctx->msg_buf.a.req_aad_pad, aad_pad_len);
1209         }
1210
1211         datalen = chunksize;
1212         if ((chunksize > ctx->digestsize) && incl_icv)
1213                 datalen -= ctx->digestsize;
1214         if (datalen) {
1215                 /* For aead, a single msg should consume the entire src sg */
1216                 written = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
1217                                          rctx->src_nents, datalen);
1218                 if (written < datalen) {
1219                         pr_err("%s(): failed to copy src sg to mbox msg",
1220                                __func__);
1221                         return -EFAULT;
1222                 }
1223         }
1224
1225         if (pad_len) {
1226                 memset(rctx->msg_buf.spu_req_pad, 0, pad_len);
1227                 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
1228         }
1229
1230         if (incl_icv)
1231                 sg_set_buf(sg++, rctx->msg_buf.digest, ctx->digestsize);
1232
1233         stat_len = spu->spu_tx_status_len();
1234         if (stat_len) {
1235                 memset(rctx->msg_buf.tx_stat, 0, stat_len);
1236                 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
1237         }
1238         return 0;
1239 }
1240
1241 /**
1242  * handle_aead_req() - Submit a SPU request message for the next chunk of the
1243  * current AEAD request.
1244  * @rctx:  Crypto request context
1245  *
1246  * Unlike other operation types, we assume the length of the request fits in
1247  * a single SPU request message. aead_enqueue() makes sure this is true.
1248  * Comments for other op types regarding threads applies here as well.
1249  *
1250  * Unlike incremental hash ops, where the spu returns the entire hash for
1251  * truncated algs like sha-224, the SPU returns just the truncated hash in
1252  * response to aead requests. So digestsize is always ctx->digestsize here.
1253  *
1254  * Return: -EINPROGRESS: crypto request has been accepted and result will be
1255  *                       returned asynchronously
1256  *         Any other value indicates an error
1257  */
1258 static int handle_aead_req(struct iproc_reqctx_s *rctx)
1259 {
1260         struct spu_hw *spu = &iproc_priv.spu;
1261         struct crypto_async_request *areq = rctx->parent;
1262         struct aead_request *req = container_of(areq,
1263                                                 struct aead_request, base);
1264         struct iproc_ctx_s *ctx = rctx->ctx;
1265         int err;
1266         unsigned int chunksize;
1267         unsigned int resp_len;
1268         u32 spu_hdr_len;
1269         u32 db_size;
1270         u32 stat_pad_len;
1271         u32 pad_len;
1272         struct brcm_message *mssg;      /* mailbox message */
1273         struct spu_request_opts req_opts;
1274         struct spu_cipher_parms cipher_parms;
1275         struct spu_hash_parms hash_parms;
1276         struct spu_aead_parms aead_parms;
1277         int assoc_nents = 0;
1278         bool incl_icv = false;
1279         unsigned int digestsize = ctx->digestsize;
1280
1281         /* number of entries in src and dst sg. Always includes SPU msg header.
1282          */
1283         u8 rx_frag_num = 2;     /* and STATUS */
1284         u8 tx_frag_num = 1;
1285
1286         /* doing the whole thing at once */
1287         chunksize = rctx->total_todo;
1288
1289         flow_log("%s: chunksize %u\n", __func__, chunksize);
1290
1291         memset(&req_opts, 0, sizeof(req_opts));
1292         memset(&hash_parms, 0, sizeof(hash_parms));
1293         memset(&aead_parms, 0, sizeof(aead_parms));
1294
1295         req_opts.is_inbound = !(rctx->is_encrypt);
1296         req_opts.auth_first = ctx->auth_first;
1297         req_opts.is_aead = true;
1298         req_opts.is_esp = ctx->is_esp;
1299
1300         cipher_parms.alg = ctx->cipher.alg;
1301         cipher_parms.mode = ctx->cipher.mode;
1302         cipher_parms.type = ctx->cipher_type;
1303         cipher_parms.key_buf = ctx->enckey;
1304         cipher_parms.key_len = ctx->enckeylen;
1305         cipher_parms.iv_buf = rctx->msg_buf.iv_ctr;
1306         cipher_parms.iv_len = rctx->iv_ctr_len;
1307
1308         hash_parms.alg = ctx->auth.alg;
1309         hash_parms.mode = ctx->auth.mode;
1310         hash_parms.type = HASH_TYPE_NONE;
1311         hash_parms.key_buf = (u8 *)ctx->authkey;
1312         hash_parms.key_len = ctx->authkeylen;
1313         hash_parms.digestsize = digestsize;
1314
1315         if ((ctx->auth.alg == HASH_ALG_SHA224) &&
1316             (ctx->authkeylen < SHA224_DIGEST_SIZE))
1317                 hash_parms.key_len = SHA224_DIGEST_SIZE;
1318
1319         aead_parms.assoc_size = req->assoclen;
1320         if (ctx->is_esp && !ctx->is_rfc4543) {
1321                 /*
1322                  * 8-byte IV is included assoc data in request. SPU2
1323                  * expects AAD to include just SPI and seqno. So
1324                  * subtract off the IV len.
1325                  */
1326                 aead_parms.assoc_size -= GCM_RFC4106_IV_SIZE;
1327
1328                 if (rctx->is_encrypt) {
1329                         aead_parms.return_iv = true;
1330                         aead_parms.ret_iv_len = GCM_RFC4106_IV_SIZE;
1331                         aead_parms.ret_iv_off = GCM_ESP_SALT_SIZE;
1332                 }
1333         } else {
1334                 aead_parms.ret_iv_len = 0;
1335         }
1336
1337         /*
1338          * Count number of sg entries from the crypto API request that are to
1339          * be included in this mailbox message. For dst sg, don't count space
1340          * for digest. Digest gets caught in a separate buffer and copied back
1341          * to dst sg when processing response.
1342          */
1343         rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize);
1344         rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize);
1345         if (aead_parms.assoc_size)
1346                 assoc_nents = spu_sg_count(rctx->assoc, 0,
1347                                            aead_parms.assoc_size);
1348
1349         mssg = &rctx->mb_mssg;
1350
1351         rctx->total_sent = chunksize;
1352         rctx->src_sent = chunksize;
1353         if (spu->spu_assoc_resp_len(ctx->cipher.mode,
1354                                     aead_parms.assoc_size,
1355                                     aead_parms.ret_iv_len,
1356                                     rctx->is_encrypt))
1357                 rx_frag_num++;
1358
1359         aead_parms.iv_len = spu->spu_aead_ivlen(ctx->cipher.mode,
1360                                                 rctx->iv_ctr_len);
1361
1362         if (ctx->auth.alg == HASH_ALG_AES)
1363                 hash_parms.type = (enum hash_type)ctx->cipher_type;
1364
1365         /* General case AAD padding (CCM and RFC4543 special cases below) */
1366         aead_parms.aad_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1367                                                  aead_parms.assoc_size);
1368
1369         /* General case data padding (CCM decrypt special case below) */
1370         aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1371                                                            chunksize);
1372
1373         if (ctx->cipher.mode == CIPHER_MODE_CCM) {
1374                 /*
1375                  * for CCM, AAD len + 2 (rather than AAD len) needs to be
1376                  * 128-bit aligned
1377                  */
1378                 aead_parms.aad_pad_len = spu->spu_gcm_ccm_pad_len(
1379                                          ctx->cipher.mode,
1380                                          aead_parms.assoc_size + 2);
1381
1382                 /*
1383                  * And when decrypting CCM, need to pad without including
1384                  * size of ICV which is tacked on to end of chunk
1385                  */
1386                 if (!rctx->is_encrypt)
1387                         aead_parms.data_pad_len =
1388                                 spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1389                                                         chunksize - digestsize);
1390
1391                 /* CCM also requires software to rewrite portions of IV: */
1392                 spu->spu_ccm_update_iv(digestsize, &cipher_parms, req->assoclen,
1393                                        chunksize, rctx->is_encrypt,
1394                                        ctx->is_esp);
1395         }
1396
1397         if (ctx->is_rfc4543) {
1398                 /*
1399                  * RFC4543: data is included in AAD, so don't pad after AAD
1400                  * and pad data based on both AAD + data size
1401                  */
1402                 aead_parms.aad_pad_len = 0;
1403                 if (!rctx->is_encrypt)
1404                         aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(
1405                                         ctx->cipher.mode,
1406                                         aead_parms.assoc_size + chunksize -
1407                                         digestsize);
1408                 else
1409                         aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(
1410                                         ctx->cipher.mode,
1411                                         aead_parms.assoc_size + chunksize);
1412
1413                 req_opts.is_rfc4543 = true;
1414         }
1415
1416         if (spu_req_incl_icv(ctx->cipher.mode, rctx->is_encrypt)) {
1417                 incl_icv = true;
1418                 tx_frag_num++;
1419                 /* Copy ICV from end of src scatterlist to digest buf */
1420                 sg_copy_part_to_buf(req->src, rctx->msg_buf.digest, digestsize,
1421                                     req->assoclen + rctx->total_sent -
1422                                     digestsize);
1423         }
1424
1425         atomic64_add(chunksize, &iproc_priv.bytes_out);
1426
1427         flow_log("%s()-sent chunksize:%u\n", __func__, chunksize);
1428
1429         /* Prepend SPU header with type 3 BCM header */
1430         memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
1431
1432         spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr +
1433                                               BCM_HDR_LEN, &req_opts,
1434                                               &cipher_parms, &hash_parms,
1435                                               &aead_parms, chunksize);
1436
1437         /* Determine total length of padding. Put all padding in one buffer. */
1438         db_size = spu_real_db_size(aead_parms.assoc_size, aead_parms.iv_len, 0,
1439                                    chunksize, aead_parms.aad_pad_len,
1440                                    aead_parms.data_pad_len, 0);
1441
1442         stat_pad_len = spu->spu_wordalign_padlen(db_size);
1443
1444         if (stat_pad_len)
1445                 rx_frag_num++;
1446         pad_len = aead_parms.data_pad_len + stat_pad_len;
1447         if (pad_len) {
1448                 tx_frag_num++;
1449                 spu->spu_request_pad(rctx->msg_buf.spu_req_pad,
1450                                      aead_parms.data_pad_len, 0,
1451                                      ctx->auth.alg, ctx->auth.mode,
1452                                      rctx->total_sent, stat_pad_len);
1453         }
1454
1455         spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
1456                               spu_hdr_len);
1457         dump_sg(rctx->assoc, 0, aead_parms.assoc_size);
1458         packet_dump("    aead iv: ", rctx->msg_buf.iv_ctr, aead_parms.iv_len);
1459         packet_log("BD:\n");
1460         dump_sg(rctx->src_sg, rctx->src_skip, chunksize);
1461         packet_dump("   pad: ", rctx->msg_buf.spu_req_pad, pad_len);
1462
1463         /*
1464          * Build mailbox message containing SPU request msg and rx buffers
1465          * to catch response message
1466          */
1467         memset(mssg, 0, sizeof(*mssg));
1468         mssg->type = BRCM_MESSAGE_SPU;
1469         mssg->ctx = rctx;       /* Will be returned in response */
1470
1471         /* Create rx scatterlist to catch result */
1472         rx_frag_num += rctx->dst_nents;
1473         resp_len = chunksize;
1474
1475         /*
1476          * Always catch ICV in separate buffer. Have to for GCM/CCM because of
1477          * padding. Have to for SHA-224 and other truncated SHAs because SPU
1478          * sends entire digest back.
1479          */
1480         rx_frag_num++;
1481
1482         if (((ctx->cipher.mode == CIPHER_MODE_GCM) ||
1483              (ctx->cipher.mode == CIPHER_MODE_CCM)) && !rctx->is_encrypt) {
1484                 /*
1485                  * Input is ciphertxt plus ICV, but ICV not incl
1486                  * in output.
1487                  */
1488                 resp_len -= ctx->digestsize;
1489                 if (resp_len == 0)
1490                         /* no rx frags to catch output data */
1491                         rx_frag_num -= rctx->dst_nents;
1492         }
1493
1494         err = spu_aead_rx_sg_create(mssg, req, rctx, rx_frag_num,
1495                                     aead_parms.assoc_size,
1496                                     aead_parms.ret_iv_len, resp_len, digestsize,
1497                                     stat_pad_len);
1498         if (err)
1499                 return err;
1500
1501         /* Create tx scatterlist containing SPU request message */
1502         tx_frag_num += rctx->src_nents;
1503         tx_frag_num += assoc_nents;
1504         if (aead_parms.aad_pad_len)
1505                 tx_frag_num++;
1506         if (aead_parms.iv_len)
1507                 tx_frag_num++;
1508         if (spu->spu_tx_status_len())
1509                 tx_frag_num++;
1510         err = spu_aead_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len,
1511                                     rctx->assoc, aead_parms.assoc_size,
1512                                     assoc_nents, aead_parms.iv_len, chunksize,
1513                                     aead_parms.aad_pad_len, pad_len, incl_icv);
1514         if (err)
1515                 return err;
1516
1517         err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx);
1518         if (unlikely(err < 0))
1519                 return err;
1520
1521         return -EINPROGRESS;
1522 }
1523
1524 /**
1525  * handle_aead_resp() - Process a SPU response message for an AEAD request.
1526  * @rctx:  Crypto request context
1527  */
1528 static void handle_aead_resp(struct iproc_reqctx_s *rctx)
1529 {
1530         struct spu_hw *spu = &iproc_priv.spu;
1531         struct crypto_async_request *areq = rctx->parent;
1532         struct aead_request *req = container_of(areq,
1533                                                 struct aead_request, base);
1534         struct iproc_ctx_s *ctx = rctx->ctx;
1535         u32 payload_len;
1536         unsigned int icv_offset;
1537         u32 result_len;
1538
1539         /* See how much data was returned */
1540         payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr);
1541         flow_log("payload_len %u\n", payload_len);
1542
1543         /* only count payload */
1544         atomic64_add(payload_len, &iproc_priv.bytes_in);
1545
1546         if (req->assoclen)
1547                 packet_dump("  assoc_data ", rctx->msg_buf.a.resp_aad,
1548                             req->assoclen);
1549
1550         /*
1551          * Copy the ICV back to the destination
1552          * buffer. In decrypt case, SPU gives us back the digest, but crypto
1553          * API doesn't expect ICV in dst buffer.
1554          */
1555         result_len = req->cryptlen;
1556         if (rctx->is_encrypt) {
1557                 icv_offset = req->assoclen + rctx->total_sent;
1558                 packet_dump("  ICV: ", rctx->msg_buf.digest, ctx->digestsize);
1559                 flow_log("copying ICV to dst sg at offset %u\n", icv_offset);
1560                 sg_copy_part_from_buf(req->dst, rctx->msg_buf.digest,
1561                                       ctx->digestsize, icv_offset);
1562                 result_len += ctx->digestsize;
1563         }
1564
1565         packet_log("response data:  ");
1566         dump_sg(req->dst, req->assoclen, result_len);
1567
1568         atomic_inc(&iproc_priv.op_counts[SPU_OP_AEAD]);
1569         if (ctx->cipher.alg == CIPHER_ALG_AES) {
1570                 if (ctx->cipher.mode == CIPHER_MODE_CCM)
1571                         atomic_inc(&iproc_priv.aead_cnt[AES_CCM]);
1572                 else if (ctx->cipher.mode == CIPHER_MODE_GCM)
1573                         atomic_inc(&iproc_priv.aead_cnt[AES_GCM]);
1574                 else
1575                         atomic_inc(&iproc_priv.aead_cnt[AUTHENC]);
1576         } else {
1577                 atomic_inc(&iproc_priv.aead_cnt[AUTHENC]);
1578         }
1579 }
1580
1581 /**
1582  * spu_chunk_cleanup() - Do cleanup after processing one chunk of a request
1583  * @rctx:  request context
1584  *
1585  * Mailbox scatterlists are allocated for each chunk. So free them after
1586  * processing each chunk.
1587  */
1588 static void spu_chunk_cleanup(struct iproc_reqctx_s *rctx)
1589 {
1590         /* mailbox message used to tx request */
1591         struct brcm_message *mssg = &rctx->mb_mssg;
1592
1593         kfree(mssg->spu.src);
1594         kfree(mssg->spu.dst);
1595         memset(mssg, 0, sizeof(struct brcm_message));
1596 }
1597
1598 /**
1599  * finish_req() - Used to invoke the complete callback from the requester when
1600  * a request has been handled asynchronously.
1601  * @rctx:  Request context
1602  * @err:   Indicates whether the request was successful or not
1603  *
1604  * Ensures that cleanup has been done for request
1605  */
1606 static void finish_req(struct iproc_reqctx_s *rctx, int err)
1607 {
1608         struct crypto_async_request *areq = rctx->parent;
1609
1610         flow_log("%s() err:%d\n\n", __func__, err);
1611
1612         /* No harm done if already called */
1613         spu_chunk_cleanup(rctx);
1614
1615         if (areq)
1616                 crypto_request_complete(areq, err);
1617 }
1618
1619 /**
1620  * spu_rx_callback() - Callback from mailbox framework with a SPU response.
1621  * @cl:         mailbox client structure for SPU driver
1622  * @msg:        mailbox message containing SPU response
1623  */
1624 static void spu_rx_callback(struct mbox_client *cl, void *msg)
1625 {
1626         struct spu_hw *spu = &iproc_priv.spu;
1627         struct brcm_message *mssg = msg;
1628         struct iproc_reqctx_s *rctx;
1629         int err;
1630
1631         rctx = mssg->ctx;
1632         if (unlikely(!rctx)) {
1633                 /* This is fatal */
1634                 pr_err("%s(): no request context", __func__);
1635                 err = -EFAULT;
1636                 goto cb_finish;
1637         }
1638
1639         /* process the SPU status */
1640         err = spu->spu_status_process(rctx->msg_buf.rx_stat);
1641         if (err != 0) {
1642                 if (err == SPU_INVALID_ICV)
1643                         atomic_inc(&iproc_priv.bad_icv);
1644                 err = -EBADMSG;
1645                 goto cb_finish;
1646         }
1647
1648         /* Process the SPU response message */
1649         switch (rctx->ctx->alg->type) {
1650         case CRYPTO_ALG_TYPE_SKCIPHER:
1651                 handle_skcipher_resp(rctx);
1652                 break;
1653         case CRYPTO_ALG_TYPE_AHASH:
1654                 handle_ahash_resp(rctx);
1655                 break;
1656         case CRYPTO_ALG_TYPE_AEAD:
1657                 handle_aead_resp(rctx);
1658                 break;
1659         default:
1660                 err = -EINVAL;
1661                 goto cb_finish;
1662         }
1663
1664         /*
1665          * If this response does not complete the request, then send the next
1666          * request chunk.
1667          */
1668         if (rctx->total_sent < rctx->total_todo) {
1669                 /* Deallocate anything specific to previous chunk */
1670                 spu_chunk_cleanup(rctx);
1671
1672                 switch (rctx->ctx->alg->type) {
1673                 case CRYPTO_ALG_TYPE_SKCIPHER:
1674                         err = handle_skcipher_req(rctx);
1675                         break;
1676                 case CRYPTO_ALG_TYPE_AHASH:
1677                         err = handle_ahash_req(rctx);
1678                         if (err == -EAGAIN)
1679                                 /*
1680                                  * we saved data in hash carry, but tell crypto
1681                                  * API we successfully completed request.
1682                                  */
1683                                 err = 0;
1684                         break;
1685                 case CRYPTO_ALG_TYPE_AEAD:
1686                         err = handle_aead_req(rctx);
1687                         break;
1688                 default:
1689                         err = -EINVAL;
1690                 }
1691
1692                 if (err == -EINPROGRESS)
1693                         /* Successfully submitted request for next chunk */
1694                         return;
1695         }
1696
1697 cb_finish:
1698         finish_req(rctx, err);
1699 }
1700
1701 /* ==================== Kernel Cryptographic API ==================== */
1702
1703 /**
1704  * skcipher_enqueue() - Handle skcipher encrypt or decrypt request.
1705  * @req:        Crypto API request
1706  * @encrypt:    true if encrypting; false if decrypting
1707  *
1708  * Return: -EINPROGRESS if request accepted and result will be returned
1709  *                      asynchronously
1710  *         < 0 if an error
1711  */
1712 static int skcipher_enqueue(struct skcipher_request *req, bool encrypt)
1713 {
1714         struct iproc_reqctx_s *rctx = skcipher_request_ctx(req);
1715         struct iproc_ctx_s *ctx =
1716             crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
1717         int err;
1718
1719         flow_log("%s() enc:%u\n", __func__, encrypt);
1720
1721         rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1722                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1723         rctx->parent = &req->base;
1724         rctx->is_encrypt = encrypt;
1725         rctx->bd_suppress = false;
1726         rctx->total_todo = req->cryptlen;
1727         rctx->src_sent = 0;
1728         rctx->total_sent = 0;
1729         rctx->total_received = 0;
1730         rctx->ctx = ctx;
1731
1732         /* Initialize current position in src and dst scatterlists */
1733         rctx->src_sg = req->src;
1734         rctx->src_nents = 0;
1735         rctx->src_skip = 0;
1736         rctx->dst_sg = req->dst;
1737         rctx->dst_nents = 0;
1738         rctx->dst_skip = 0;
1739
1740         if (ctx->cipher.mode == CIPHER_MODE_CBC ||
1741             ctx->cipher.mode == CIPHER_MODE_CTR ||
1742             ctx->cipher.mode == CIPHER_MODE_OFB ||
1743             ctx->cipher.mode == CIPHER_MODE_XTS ||
1744             ctx->cipher.mode == CIPHER_MODE_GCM ||
1745             ctx->cipher.mode == CIPHER_MODE_CCM) {
1746                 rctx->iv_ctr_len =
1747                     crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req));
1748                 memcpy(rctx->msg_buf.iv_ctr, req->iv, rctx->iv_ctr_len);
1749         } else {
1750                 rctx->iv_ctr_len = 0;
1751         }
1752
1753         /* Choose a SPU to process this request */
1754         rctx->chan_idx = select_channel();
1755         err = handle_skcipher_req(rctx);
1756         if (err != -EINPROGRESS)
1757                 /* synchronous result */
1758                 spu_chunk_cleanup(rctx);
1759
1760         return err;
1761 }
1762
1763 static int des_setkey(struct crypto_skcipher *cipher, const u8 *key,
1764                       unsigned int keylen)
1765 {
1766         struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1767         int err;
1768
1769         err = verify_skcipher_des_key(cipher, key);
1770         if (err)
1771                 return err;
1772
1773         ctx->cipher_type = CIPHER_TYPE_DES;
1774         return 0;
1775 }
1776
1777 static int threedes_setkey(struct crypto_skcipher *cipher, const u8 *key,
1778                            unsigned int keylen)
1779 {
1780         struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1781         int err;
1782
1783         err = verify_skcipher_des3_key(cipher, key);
1784         if (err)
1785                 return err;
1786
1787         ctx->cipher_type = CIPHER_TYPE_3DES;
1788         return 0;
1789 }
1790
1791 static int aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
1792                       unsigned int keylen)
1793 {
1794         struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1795
1796         if (ctx->cipher.mode == CIPHER_MODE_XTS)
1797                 /* XTS includes two keys of equal length */
1798                 keylen = keylen / 2;
1799
1800         switch (keylen) {
1801         case AES_KEYSIZE_128:
1802                 ctx->cipher_type = CIPHER_TYPE_AES128;
1803                 break;
1804         case AES_KEYSIZE_192:
1805                 ctx->cipher_type = CIPHER_TYPE_AES192;
1806                 break;
1807         case AES_KEYSIZE_256:
1808                 ctx->cipher_type = CIPHER_TYPE_AES256;
1809                 break;
1810         default:
1811                 return -EINVAL;
1812         }
1813         WARN_ON((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
1814                 ((ctx->max_payload % AES_BLOCK_SIZE) != 0));
1815         return 0;
1816 }
1817
1818 static int skcipher_setkey(struct crypto_skcipher *cipher, const u8 *key,
1819                              unsigned int keylen)
1820 {
1821         struct spu_hw *spu = &iproc_priv.spu;
1822         struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1823         struct spu_cipher_parms cipher_parms;
1824         u32 alloc_len = 0;
1825         int err;
1826
1827         flow_log("skcipher_setkey() keylen: %d\n", keylen);
1828         flow_dump("  key: ", key, keylen);
1829
1830         switch (ctx->cipher.alg) {
1831         case CIPHER_ALG_DES:
1832                 err = des_setkey(cipher, key, keylen);
1833                 break;
1834         case CIPHER_ALG_3DES:
1835                 err = threedes_setkey(cipher, key, keylen);
1836                 break;
1837         case CIPHER_ALG_AES:
1838                 err = aes_setkey(cipher, key, keylen);
1839                 break;
1840         default:
1841                 pr_err("%s() Error: unknown cipher alg\n", __func__);
1842                 err = -EINVAL;
1843         }
1844         if (err)
1845                 return err;
1846
1847         memcpy(ctx->enckey, key, keylen);
1848         ctx->enckeylen = keylen;
1849
1850         /* SPU needs XTS keys in the reverse order the crypto API presents */
1851         if ((ctx->cipher.alg == CIPHER_ALG_AES) &&
1852             (ctx->cipher.mode == CIPHER_MODE_XTS)) {
1853                 unsigned int xts_keylen = keylen / 2;
1854
1855                 memcpy(ctx->enckey, key + xts_keylen, xts_keylen);
1856                 memcpy(ctx->enckey + xts_keylen, key, xts_keylen);
1857         }
1858
1859         if (spu->spu_type == SPU_TYPE_SPUM)
1860                 alloc_len = BCM_HDR_LEN + SPU_HEADER_ALLOC_LEN;
1861         else if (spu->spu_type == SPU_TYPE_SPU2)
1862                 alloc_len = BCM_HDR_LEN + SPU2_HEADER_ALLOC_LEN;
1863         memset(ctx->bcm_spu_req_hdr, 0, alloc_len);
1864         cipher_parms.iv_buf = NULL;
1865         cipher_parms.iv_len = crypto_skcipher_ivsize(cipher);
1866         flow_log("%s: iv_len %u\n", __func__, cipher_parms.iv_len);
1867
1868         cipher_parms.alg = ctx->cipher.alg;
1869         cipher_parms.mode = ctx->cipher.mode;
1870         cipher_parms.type = ctx->cipher_type;
1871         cipher_parms.key_buf = ctx->enckey;
1872         cipher_parms.key_len = ctx->enckeylen;
1873
1874         /* Prepend SPU request message with BCM header */
1875         memcpy(ctx->bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
1876         ctx->spu_req_hdr_len =
1877             spu->spu_cipher_req_init(ctx->bcm_spu_req_hdr + BCM_HDR_LEN,
1878                                      &cipher_parms);
1879
1880         ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
1881                                                           ctx->enckeylen,
1882                                                           false);
1883
1884         atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_CIPHER]);
1885
1886         return 0;
1887 }
1888
1889 static int skcipher_encrypt(struct skcipher_request *req)
1890 {
1891         flow_log("skcipher_encrypt() nbytes:%u\n", req->cryptlen);
1892
1893         return skcipher_enqueue(req, true);
1894 }
1895
1896 static int skcipher_decrypt(struct skcipher_request *req)
1897 {
1898         flow_log("skcipher_decrypt() nbytes:%u\n", req->cryptlen);
1899         return skcipher_enqueue(req, false);
1900 }
1901
1902 static int ahash_enqueue(struct ahash_request *req)
1903 {
1904         struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
1905         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1906         struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
1907         int err;
1908         const char *alg_name;
1909
1910         flow_log("ahash_enqueue() nbytes:%u\n", req->nbytes);
1911
1912         rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1913                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1914         rctx->parent = &req->base;
1915         rctx->ctx = ctx;
1916         rctx->bd_suppress = true;
1917         memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message));
1918
1919         /* Initialize position in src scatterlist */
1920         rctx->src_sg = req->src;
1921         rctx->src_skip = 0;
1922         rctx->src_nents = 0;
1923         rctx->dst_sg = NULL;
1924         rctx->dst_skip = 0;
1925         rctx->dst_nents = 0;
1926
1927         /* SPU2 hardware does not compute hash of zero length data */
1928         if ((rctx->is_final == 1) && (rctx->total_todo == 0) &&
1929             (iproc_priv.spu.spu_type == SPU_TYPE_SPU2)) {
1930                 alg_name = crypto_ahash_alg_name(tfm);
1931                 flow_log("Doing %sfinal %s zero-len hash request in software\n",
1932                          rctx->is_final ? "" : "non-", alg_name);
1933                 err = do_shash((unsigned char *)alg_name, req->result,
1934                                NULL, 0, NULL, 0, ctx->authkey,
1935                                ctx->authkeylen);
1936                 if (err < 0)
1937                         flow_log("Hash request failed with error %d\n", err);
1938                 return err;
1939         }
1940         /* Choose a SPU to process this request */
1941         rctx->chan_idx = select_channel();
1942
1943         err = handle_ahash_req(rctx);
1944         if (err != -EINPROGRESS)
1945                 /* synchronous result */
1946                 spu_chunk_cleanup(rctx);
1947
1948         if (err == -EAGAIN)
1949                 /*
1950                  * we saved data in hash carry, but tell crypto API
1951                  * we successfully completed request.
1952                  */
1953                 err = 0;
1954
1955         return err;
1956 }
1957
1958 static int __ahash_init(struct ahash_request *req)
1959 {
1960         struct spu_hw *spu = &iproc_priv.spu;
1961         struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
1962         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1963         struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
1964
1965         flow_log("%s()\n", __func__);
1966
1967         /* Initialize the context */
1968         rctx->hash_carry_len = 0;
1969         rctx->is_final = 0;
1970
1971         rctx->total_todo = 0;
1972         rctx->src_sent = 0;
1973         rctx->total_sent = 0;
1974         rctx->total_received = 0;
1975
1976         ctx->digestsize = crypto_ahash_digestsize(tfm);
1977         /* If we add a hash whose digest is larger, catch it here. */
1978         WARN_ON(ctx->digestsize > MAX_DIGEST_SIZE);
1979
1980         rctx->is_sw_hmac = false;
1981
1982         ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen, 0,
1983                                                           true);
1984
1985         return 0;
1986 }
1987
1988 /**
1989  * spu_no_incr_hash() - Determine whether incremental hashing is supported.
1990  * @ctx:  Crypto session context
1991  *
1992  * SPU-2 does not support incremental hashing (we'll have to revisit and
1993  * condition based on chip revision or device tree entry if future versions do
1994  * support incremental hash)
1995  *
1996  * SPU-M also doesn't support incremental hashing of AES-XCBC
1997  *
1998  * Return: true if incremental hashing is not supported
1999  *         false otherwise
2000  */
2001 static bool spu_no_incr_hash(struct iproc_ctx_s *ctx)
2002 {
2003         struct spu_hw *spu = &iproc_priv.spu;
2004
2005         if (spu->spu_type == SPU_TYPE_SPU2)
2006                 return true;
2007
2008         if ((ctx->auth.alg == HASH_ALG_AES) &&
2009             (ctx->auth.mode == HASH_MODE_XCBC))
2010                 return true;
2011
2012         /* Otherwise, incremental hashing is supported */
2013         return false;
2014 }
2015
2016 static int ahash_init(struct ahash_request *req)
2017 {
2018         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2019         struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2020         const char *alg_name;
2021         struct crypto_shash *hash;
2022         int ret;
2023         gfp_t gfp;
2024
2025         if (spu_no_incr_hash(ctx)) {
2026                 /*
2027                  * If we get an incremental hashing request and it's not
2028                  * supported by the hardware, we need to handle it in software
2029                  * by calling synchronous hash functions.
2030                  */
2031                 alg_name = crypto_ahash_alg_name(tfm);
2032                 hash = crypto_alloc_shash(alg_name, 0, 0);
2033                 if (IS_ERR(hash)) {
2034                         ret = PTR_ERR(hash);
2035                         goto err;
2036                 }
2037
2038                 gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2039                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2040                 ctx->shash = kmalloc(sizeof(*ctx->shash) +
2041                                      crypto_shash_descsize(hash), gfp);
2042                 if (!ctx->shash) {
2043                         ret = -ENOMEM;
2044                         goto err_hash;
2045                 }
2046                 ctx->shash->tfm = hash;
2047
2048                 /* Set the key using data we already have from setkey */
2049                 if (ctx->authkeylen > 0) {
2050                         ret = crypto_shash_setkey(hash, ctx->authkey,
2051                                                   ctx->authkeylen);
2052                         if (ret)
2053                                 goto err_shash;
2054                 }
2055
2056                 /* Initialize hash w/ this key and other params */
2057                 ret = crypto_shash_init(ctx->shash);
2058                 if (ret)
2059                         goto err_shash;
2060         } else {
2061                 /* Otherwise call the internal function which uses SPU hw */
2062                 ret = __ahash_init(req);
2063         }
2064
2065         return ret;
2066
2067 err_shash:
2068         kfree(ctx->shash);
2069 err_hash:
2070         crypto_free_shash(hash);
2071 err:
2072         return ret;
2073 }
2074
2075 static int __ahash_update(struct ahash_request *req)
2076 {
2077         struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2078
2079         flow_log("ahash_update() nbytes:%u\n", req->nbytes);
2080
2081         if (!req->nbytes)
2082                 return 0;
2083         rctx->total_todo += req->nbytes;
2084         rctx->src_sent = 0;
2085
2086         return ahash_enqueue(req);
2087 }
2088
2089 static int ahash_update(struct ahash_request *req)
2090 {
2091         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2092         struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2093         u8 *tmpbuf;
2094         int ret;
2095         int nents;
2096         gfp_t gfp;
2097
2098         if (spu_no_incr_hash(ctx)) {
2099                 /*
2100                  * If we get an incremental hashing request and it's not
2101                  * supported by the hardware, we need to handle it in software
2102                  * by calling synchronous hash functions.
2103                  */
2104                 if (req->src)
2105                         nents = sg_nents(req->src);
2106                 else
2107                         return -EINVAL;
2108
2109                 /* Copy data from req scatterlist to tmp buffer */
2110                 gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2111                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2112                 tmpbuf = kmalloc(req->nbytes, gfp);
2113                 if (!tmpbuf)
2114                         return -ENOMEM;
2115
2116                 if (sg_copy_to_buffer(req->src, nents, tmpbuf, req->nbytes) !=
2117                                 req->nbytes) {
2118                         kfree(tmpbuf);
2119                         return -EINVAL;
2120                 }
2121
2122                 /* Call synchronous update */
2123                 ret = crypto_shash_update(ctx->shash, tmpbuf, req->nbytes);
2124                 kfree(tmpbuf);
2125         } else {
2126                 /* Otherwise call the internal function which uses SPU hw */
2127                 ret = __ahash_update(req);
2128         }
2129
2130         return ret;
2131 }
2132
2133 static int __ahash_final(struct ahash_request *req)
2134 {
2135         struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2136
2137         flow_log("ahash_final() nbytes:%u\n", req->nbytes);
2138
2139         rctx->is_final = 1;
2140
2141         return ahash_enqueue(req);
2142 }
2143
2144 static int ahash_final(struct ahash_request *req)
2145 {
2146         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2147         struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2148         int ret;
2149
2150         if (spu_no_incr_hash(ctx)) {
2151                 /*
2152                  * If we get an incremental hashing request and it's not
2153                  * supported by the hardware, we need to handle it in software
2154                  * by calling synchronous hash functions.
2155                  */
2156                 ret = crypto_shash_final(ctx->shash, req->result);
2157
2158                 /* Done with hash, can deallocate it now */
2159                 crypto_free_shash(ctx->shash->tfm);
2160                 kfree(ctx->shash);
2161
2162         } else {
2163                 /* Otherwise call the internal function which uses SPU hw */
2164                 ret = __ahash_final(req);
2165         }
2166
2167         return ret;
2168 }
2169
2170 static int __ahash_finup(struct ahash_request *req)
2171 {
2172         struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2173
2174         flow_log("ahash_finup() nbytes:%u\n", req->nbytes);
2175
2176         rctx->total_todo += req->nbytes;
2177         rctx->src_sent = 0;
2178         rctx->is_final = 1;
2179
2180         return ahash_enqueue(req);
2181 }
2182
2183 static int ahash_finup(struct ahash_request *req)
2184 {
2185         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2186         struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2187         u8 *tmpbuf;
2188         int ret;
2189         int nents;
2190         gfp_t gfp;
2191
2192         if (spu_no_incr_hash(ctx)) {
2193                 /*
2194                  * If we get an incremental hashing request and it's not
2195                  * supported by the hardware, we need to handle it in software
2196                  * by calling synchronous hash functions.
2197                  */
2198                 if (req->src) {
2199                         nents = sg_nents(req->src);
2200                 } else {
2201                         ret = -EINVAL;
2202                         goto ahash_finup_exit;
2203                 }
2204
2205                 /* Copy data from req scatterlist to tmp buffer */
2206                 gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2207                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2208                 tmpbuf = kmalloc(req->nbytes, gfp);
2209                 if (!tmpbuf) {
2210                         ret = -ENOMEM;
2211                         goto ahash_finup_exit;
2212                 }
2213
2214                 if (sg_copy_to_buffer(req->src, nents, tmpbuf, req->nbytes) !=
2215                                 req->nbytes) {
2216                         ret = -EINVAL;
2217                         goto ahash_finup_free;
2218                 }
2219
2220                 /* Call synchronous update */
2221                 ret = crypto_shash_finup(ctx->shash, tmpbuf, req->nbytes,
2222                                          req->result);
2223         } else {
2224                 /* Otherwise call the internal function which uses SPU hw */
2225                 return __ahash_finup(req);
2226         }
2227 ahash_finup_free:
2228         kfree(tmpbuf);
2229
2230 ahash_finup_exit:
2231         /* Done with hash, can deallocate it now */
2232         crypto_free_shash(ctx->shash->tfm);
2233         kfree(ctx->shash);
2234         return ret;
2235 }
2236
2237 static int ahash_digest(struct ahash_request *req)
2238 {
2239         int err;
2240
2241         flow_log("ahash_digest() nbytes:%u\n", req->nbytes);
2242
2243         /* whole thing at once */
2244         err = __ahash_init(req);
2245         if (!err)
2246                 err = __ahash_finup(req);
2247
2248         return err;
2249 }
2250
2251 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
2252                         unsigned int keylen)
2253 {
2254         struct iproc_ctx_s *ctx = crypto_ahash_ctx(ahash);
2255
2256         flow_log("%s() ahash:%p key:%p keylen:%u\n",
2257                  __func__, ahash, key, keylen);
2258         flow_dump("  key: ", key, keylen);
2259
2260         if (ctx->auth.alg == HASH_ALG_AES) {
2261                 switch (keylen) {
2262                 case AES_KEYSIZE_128:
2263                         ctx->cipher_type = CIPHER_TYPE_AES128;
2264                         break;
2265                 case AES_KEYSIZE_192:
2266                         ctx->cipher_type = CIPHER_TYPE_AES192;
2267                         break;
2268                 case AES_KEYSIZE_256:
2269                         ctx->cipher_type = CIPHER_TYPE_AES256;
2270                         break;
2271                 default:
2272                         pr_err("%s() Error: Invalid key length\n", __func__);
2273                         return -EINVAL;
2274                 }
2275         } else {
2276                 pr_err("%s() Error: unknown hash alg\n", __func__);
2277                 return -EINVAL;
2278         }
2279         memcpy(ctx->authkey, key, keylen);
2280         ctx->authkeylen = keylen;
2281
2282         return 0;
2283 }
2284
2285 static int ahash_export(struct ahash_request *req, void *out)
2286 {
2287         const struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2288         struct spu_hash_export_s *spu_exp = (struct spu_hash_export_s *)out;
2289
2290         spu_exp->total_todo = rctx->total_todo;
2291         spu_exp->total_sent = rctx->total_sent;
2292         spu_exp->is_sw_hmac = rctx->is_sw_hmac;
2293         memcpy(spu_exp->hash_carry, rctx->hash_carry, sizeof(rctx->hash_carry));
2294         spu_exp->hash_carry_len = rctx->hash_carry_len;
2295         memcpy(spu_exp->incr_hash, rctx->incr_hash, sizeof(rctx->incr_hash));
2296
2297         return 0;
2298 }
2299
2300 static int ahash_import(struct ahash_request *req, const void *in)
2301 {
2302         struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2303         struct spu_hash_export_s *spu_exp = (struct spu_hash_export_s *)in;
2304
2305         rctx->total_todo = spu_exp->total_todo;
2306         rctx->total_sent = spu_exp->total_sent;
2307         rctx->is_sw_hmac = spu_exp->is_sw_hmac;
2308         memcpy(rctx->hash_carry, spu_exp->hash_carry, sizeof(rctx->hash_carry));
2309         rctx->hash_carry_len = spu_exp->hash_carry_len;
2310         memcpy(rctx->incr_hash, spu_exp->incr_hash, sizeof(rctx->incr_hash));
2311
2312         return 0;
2313 }
2314
2315 static int ahash_hmac_setkey(struct crypto_ahash *ahash, const u8 *key,
2316                              unsigned int keylen)
2317 {
2318         struct iproc_ctx_s *ctx = crypto_ahash_ctx(ahash);
2319         unsigned int blocksize =
2320                 crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
2321         unsigned int digestsize = crypto_ahash_digestsize(ahash);
2322         unsigned int index;
2323         int rc;
2324
2325         flow_log("%s() ahash:%p key:%p keylen:%u blksz:%u digestsz:%u\n",
2326                  __func__, ahash, key, keylen, blocksize, digestsize);
2327         flow_dump("  key: ", key, keylen);
2328
2329         if (keylen > blocksize) {
2330                 switch (ctx->auth.alg) {
2331                 case HASH_ALG_MD5:
2332                         rc = do_shash("md5", ctx->authkey, key, keylen, NULL,
2333                                       0, NULL, 0);
2334                         break;
2335                 case HASH_ALG_SHA1:
2336                         rc = do_shash("sha1", ctx->authkey, key, keylen, NULL,
2337                                       0, NULL, 0);
2338                         break;
2339                 case HASH_ALG_SHA224:
2340                         rc = do_shash("sha224", ctx->authkey, key, keylen, NULL,
2341                                       0, NULL, 0);
2342                         break;
2343                 case HASH_ALG_SHA256:
2344                         rc = do_shash("sha256", ctx->authkey, key, keylen, NULL,
2345                                       0, NULL, 0);
2346                         break;
2347                 case HASH_ALG_SHA384:
2348                         rc = do_shash("sha384", ctx->authkey, key, keylen, NULL,
2349                                       0, NULL, 0);
2350                         break;
2351                 case HASH_ALG_SHA512:
2352                         rc = do_shash("sha512", ctx->authkey, key, keylen, NULL,
2353                                       0, NULL, 0);
2354                         break;
2355                 case HASH_ALG_SHA3_224:
2356                         rc = do_shash("sha3-224", ctx->authkey, key, keylen,
2357                                       NULL, 0, NULL, 0);
2358                         break;
2359                 case HASH_ALG_SHA3_256:
2360                         rc = do_shash("sha3-256", ctx->authkey, key, keylen,
2361                                       NULL, 0, NULL, 0);
2362                         break;
2363                 case HASH_ALG_SHA3_384:
2364                         rc = do_shash("sha3-384", ctx->authkey, key, keylen,
2365                                       NULL, 0, NULL, 0);
2366                         break;
2367                 case HASH_ALG_SHA3_512:
2368                         rc = do_shash("sha3-512", ctx->authkey, key, keylen,
2369                                       NULL, 0, NULL, 0);
2370                         break;
2371                 default:
2372                         pr_err("%s() Error: unknown hash alg\n", __func__);
2373                         return -EINVAL;
2374                 }
2375                 if (rc < 0) {
2376                         pr_err("%s() Error %d computing shash for %s\n",
2377                                __func__, rc, hash_alg_name[ctx->auth.alg]);
2378                         return rc;
2379                 }
2380                 ctx->authkeylen = digestsize;
2381
2382                 flow_log("  keylen > digestsize... hashed\n");
2383                 flow_dump("  newkey: ", ctx->authkey, ctx->authkeylen);
2384         } else {
2385                 memcpy(ctx->authkey, key, keylen);
2386                 ctx->authkeylen = keylen;
2387         }
2388
2389         /*
2390          * Full HMAC operation in SPUM is not verified,
2391          * So keeping the generation of IPAD, OPAD and
2392          * outer hashing in software.
2393          */
2394         if (iproc_priv.spu.spu_type == SPU_TYPE_SPUM) {
2395                 memcpy(ctx->ipad, ctx->authkey, ctx->authkeylen);
2396                 memset(ctx->ipad + ctx->authkeylen, 0,
2397                        blocksize - ctx->authkeylen);
2398                 ctx->authkeylen = 0;
2399                 unsafe_memcpy(ctx->opad, ctx->ipad, blocksize,
2400                               "fortified memcpy causes -Wrestrict warning");
2401
2402                 for (index = 0; index < blocksize; index++) {
2403                         ctx->ipad[index] ^= HMAC_IPAD_VALUE;
2404                         ctx->opad[index] ^= HMAC_OPAD_VALUE;
2405                 }
2406
2407                 flow_dump("  ipad: ", ctx->ipad, blocksize);
2408                 flow_dump("  opad: ", ctx->opad, blocksize);
2409         }
2410         ctx->digestsize = digestsize;
2411         atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_HMAC]);
2412
2413         return 0;
2414 }
2415
2416 static int ahash_hmac_init(struct ahash_request *req)
2417 {
2418         struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2419         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2420         struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2421         unsigned int blocksize =
2422                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2423
2424         flow_log("ahash_hmac_init()\n");
2425
2426         /* init the context as a hash */
2427         ahash_init(req);
2428
2429         if (!spu_no_incr_hash(ctx)) {
2430                 /* SPU-M can do incr hashing but needs sw for outer HMAC */
2431                 rctx->is_sw_hmac = true;
2432                 ctx->auth.mode = HASH_MODE_HASH;
2433                 /* start with a prepended ipad */
2434                 memcpy(rctx->hash_carry, ctx->ipad, blocksize);
2435                 rctx->hash_carry_len = blocksize;
2436                 rctx->total_todo += blocksize;
2437         }
2438
2439         return 0;
2440 }
2441
2442 static int ahash_hmac_update(struct ahash_request *req)
2443 {
2444         flow_log("ahash_hmac_update() nbytes:%u\n", req->nbytes);
2445
2446         if (!req->nbytes)
2447                 return 0;
2448
2449         return ahash_update(req);
2450 }
2451
2452 static int ahash_hmac_final(struct ahash_request *req)
2453 {
2454         flow_log("ahash_hmac_final() nbytes:%u\n", req->nbytes);
2455
2456         return ahash_final(req);
2457 }
2458
2459 static int ahash_hmac_finup(struct ahash_request *req)
2460 {
2461         flow_log("ahash_hmac_finupl() nbytes:%u\n", req->nbytes);
2462
2463         return ahash_finup(req);
2464 }
2465
2466 static int ahash_hmac_digest(struct ahash_request *req)
2467 {
2468         struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2469         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2470         struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2471         unsigned int blocksize =
2472                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2473
2474         flow_log("ahash_hmac_digest() nbytes:%u\n", req->nbytes);
2475
2476         /* Perform initialization and then call finup */
2477         __ahash_init(req);
2478
2479         if (iproc_priv.spu.spu_type == SPU_TYPE_SPU2) {
2480                 /*
2481                  * SPU2 supports full HMAC implementation in the
2482                  * hardware, need not to generate IPAD, OPAD and
2483                  * outer hash in software.
2484                  * Only for hash key len > hash block size, SPU2
2485                  * expects to perform hashing on the key, shorten
2486                  * it to digest size and feed it as hash key.
2487                  */
2488                 rctx->is_sw_hmac = false;
2489                 ctx->auth.mode = HASH_MODE_HMAC;
2490         } else {
2491                 rctx->is_sw_hmac = true;
2492                 ctx->auth.mode = HASH_MODE_HASH;
2493                 /* start with a prepended ipad */
2494                 memcpy(rctx->hash_carry, ctx->ipad, blocksize);
2495                 rctx->hash_carry_len = blocksize;
2496                 rctx->total_todo += blocksize;
2497         }
2498
2499         return __ahash_finup(req);
2500 }
2501
2502 /* aead helpers */
2503
2504 static int aead_need_fallback(struct aead_request *req)
2505 {
2506         struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2507         struct spu_hw *spu = &iproc_priv.spu;
2508         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2509         struct iproc_ctx_s *ctx = crypto_aead_ctx(aead);
2510         u32 payload_len;
2511
2512         /*
2513          * SPU hardware cannot handle the AES-GCM/CCM case where plaintext
2514          * and AAD are both 0 bytes long. So use fallback in this case.
2515          */
2516         if (((ctx->cipher.mode == CIPHER_MODE_GCM) ||
2517              (ctx->cipher.mode == CIPHER_MODE_CCM)) &&
2518             (req->assoclen == 0)) {
2519                 if ((rctx->is_encrypt && (req->cryptlen == 0)) ||
2520                     (!rctx->is_encrypt && (req->cryptlen == ctx->digestsize))) {
2521                         flow_log("AES GCM/CCM needs fallback for 0 len req\n");
2522                         return 1;
2523                 }
2524         }
2525
2526         /* SPU-M hardware only supports CCM digest size of 8, 12, or 16 bytes */
2527         if ((ctx->cipher.mode == CIPHER_MODE_CCM) &&
2528             (spu->spu_type == SPU_TYPE_SPUM) &&
2529             (ctx->digestsize != 8) && (ctx->digestsize != 12) &&
2530             (ctx->digestsize != 16)) {
2531                 flow_log("%s() AES CCM needs fallback for digest size %d\n",
2532                          __func__, ctx->digestsize);
2533                 return 1;
2534         }
2535
2536         /*
2537          * SPU-M on NSP has an issue where AES-CCM hash is not correct
2538          * when AAD size is 0
2539          */
2540         if ((ctx->cipher.mode == CIPHER_MODE_CCM) &&
2541             (spu->spu_subtype == SPU_SUBTYPE_SPUM_NSP) &&
2542             (req->assoclen == 0)) {
2543                 flow_log("%s() AES_CCM needs fallback for 0 len AAD on NSP\n",
2544                          __func__);
2545                 return 1;
2546         }
2547
2548         /*
2549          * RFC4106 and RFC4543 cannot handle the case where AAD is other than
2550          * 16 or 20 bytes long. So use fallback in this case.
2551          */
2552         if (ctx->cipher.mode == CIPHER_MODE_GCM &&
2553             ctx->cipher.alg == CIPHER_ALG_AES &&
2554             rctx->iv_ctr_len == GCM_RFC4106_IV_SIZE &&
2555             req->assoclen != 16 && req->assoclen != 20) {
2556                 flow_log("RFC4106/RFC4543 needs fallback for assoclen"
2557                          " other than 16 or 20 bytes\n");
2558                 return 1;
2559         }
2560
2561         payload_len = req->cryptlen;
2562         if (spu->spu_type == SPU_TYPE_SPUM)
2563                 payload_len += req->assoclen;
2564
2565         flow_log("%s() payload len: %u\n", __func__, payload_len);
2566
2567         if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
2568                 return 0;
2569         else
2570                 return payload_len > ctx->max_payload;
2571 }
2572
2573 static int aead_do_fallback(struct aead_request *req, bool is_encrypt)
2574 {
2575         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2576         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
2577         struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2578         struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
2579         struct aead_request *subreq;
2580
2581         flow_log("%s() enc:%u\n", __func__, is_encrypt);
2582
2583         if (!ctx->fallback_cipher)
2584                 return -EINVAL;
2585
2586         subreq = &rctx->req;
2587         aead_request_set_tfm(subreq, ctx->fallback_cipher);
2588         aead_request_set_callback(subreq, aead_request_flags(req),
2589                                   req->base.complete, req->base.data);
2590         aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
2591                                req->iv);
2592         aead_request_set_ad(subreq, req->assoclen);
2593
2594         return is_encrypt ? crypto_aead_encrypt(req) :
2595                             crypto_aead_decrypt(req);
2596 }
2597
2598 static int aead_enqueue(struct aead_request *req, bool is_encrypt)
2599 {
2600         struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2601         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2602         struct iproc_ctx_s *ctx = crypto_aead_ctx(aead);
2603         int err;
2604
2605         flow_log("%s() enc:%u\n", __func__, is_encrypt);
2606
2607         if (req->assoclen > MAX_ASSOC_SIZE) {
2608                 pr_err
2609                     ("%s() Error: associated data too long. (%u > %u bytes)\n",
2610                      __func__, req->assoclen, MAX_ASSOC_SIZE);
2611                 return -EINVAL;
2612         }
2613
2614         rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2615                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2616         rctx->parent = &req->base;
2617         rctx->is_encrypt = is_encrypt;
2618         rctx->bd_suppress = false;
2619         rctx->total_todo = req->cryptlen;
2620         rctx->src_sent = 0;
2621         rctx->total_sent = 0;
2622         rctx->total_received = 0;
2623         rctx->is_sw_hmac = false;
2624         rctx->ctx = ctx;
2625         memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message));
2626
2627         /* assoc data is at start of src sg */
2628         rctx->assoc = req->src;
2629
2630         /*
2631          * Init current position in src scatterlist to be after assoc data.
2632          * src_skip set to buffer offset where data begins. (Assoc data could
2633          * end in the middle of a buffer.)
2634          */
2635         if (spu_sg_at_offset(req->src, req->assoclen, &rctx->src_sg,
2636                              &rctx->src_skip) < 0) {
2637                 pr_err("%s() Error: Unable to find start of src data\n",
2638                        __func__);
2639                 return -EINVAL;
2640         }
2641
2642         rctx->src_nents = 0;
2643         rctx->dst_nents = 0;
2644         if (req->dst == req->src) {
2645                 rctx->dst_sg = rctx->src_sg;
2646                 rctx->dst_skip = rctx->src_skip;
2647         } else {
2648                 /*
2649                  * Expect req->dst to have room for assoc data followed by
2650                  * output data and ICV, if encrypt. So initialize dst_sg
2651                  * to point beyond assoc len offset.
2652                  */
2653                 if (spu_sg_at_offset(req->dst, req->assoclen, &rctx->dst_sg,
2654                                      &rctx->dst_skip) < 0) {
2655                         pr_err("%s() Error: Unable to find start of dst data\n",
2656                                __func__);
2657                         return -EINVAL;
2658                 }
2659         }
2660
2661         if (ctx->cipher.mode == CIPHER_MODE_CBC ||
2662             ctx->cipher.mode == CIPHER_MODE_CTR ||
2663             ctx->cipher.mode == CIPHER_MODE_OFB ||
2664             ctx->cipher.mode == CIPHER_MODE_XTS ||
2665             ctx->cipher.mode == CIPHER_MODE_GCM) {
2666                 rctx->iv_ctr_len =
2667                         ctx->salt_len +
2668                         crypto_aead_ivsize(crypto_aead_reqtfm(req));
2669         } else if (ctx->cipher.mode == CIPHER_MODE_CCM) {
2670                 rctx->iv_ctr_len = CCM_AES_IV_SIZE;
2671         } else {
2672                 rctx->iv_ctr_len = 0;
2673         }
2674
2675         rctx->hash_carry_len = 0;
2676
2677         flow_log("  src sg: %p\n", req->src);
2678         flow_log("  rctx->src_sg: %p, src_skip %u\n",
2679                  rctx->src_sg, rctx->src_skip);
2680         flow_log("  assoc:  %p, assoclen %u\n", rctx->assoc, req->assoclen);
2681         flow_log("  dst sg: %p\n", req->dst);
2682         flow_log("  rctx->dst_sg: %p, dst_skip %u\n",
2683                  rctx->dst_sg, rctx->dst_skip);
2684         flow_log("  iv_ctr_len:%u\n", rctx->iv_ctr_len);
2685         flow_dump("  iv: ", req->iv, rctx->iv_ctr_len);
2686         flow_log("  authkeylen:%u\n", ctx->authkeylen);
2687         flow_log("  is_esp: %s\n", ctx->is_esp ? "yes" : "no");
2688
2689         if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
2690                 flow_log("  max_payload infinite");
2691         else
2692                 flow_log("  max_payload: %u\n", ctx->max_payload);
2693
2694         if (unlikely(aead_need_fallback(req)))
2695                 return aead_do_fallback(req, is_encrypt);
2696
2697         /*
2698          * Do memory allocations for request after fallback check, because if we
2699          * do fallback, we won't call finish_req() to dealloc.
2700          */
2701         if (rctx->iv_ctr_len) {
2702                 if (ctx->salt_len)
2703                         memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset,
2704                                ctx->salt, ctx->salt_len);
2705                 memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset + ctx->salt_len,
2706                        req->iv,
2707                        rctx->iv_ctr_len - ctx->salt_len - ctx->salt_offset);
2708         }
2709
2710         rctx->chan_idx = select_channel();
2711         err = handle_aead_req(rctx);
2712         if (err != -EINPROGRESS)
2713                 /* synchronous result */
2714                 spu_chunk_cleanup(rctx);
2715
2716         return err;
2717 }
2718
2719 static int aead_authenc_setkey(struct crypto_aead *cipher,
2720                                const u8 *key, unsigned int keylen)
2721 {
2722         struct spu_hw *spu = &iproc_priv.spu;
2723         struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2724         struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
2725         struct crypto_authenc_keys keys;
2726         int ret;
2727
2728         flow_log("%s() aead:%p key:%p keylen:%u\n", __func__, cipher, key,
2729                  keylen);
2730         flow_dump("  key: ", key, keylen);
2731
2732         ret = crypto_authenc_extractkeys(&keys, key, keylen);
2733         if (ret)
2734                 goto badkey;
2735
2736         if (keys.enckeylen > MAX_KEY_SIZE ||
2737             keys.authkeylen > MAX_KEY_SIZE)
2738                 goto badkey;
2739
2740         ctx->enckeylen = keys.enckeylen;
2741         ctx->authkeylen = keys.authkeylen;
2742
2743         memcpy(ctx->enckey, keys.enckey, keys.enckeylen);
2744         /* May end up padding auth key. So make sure it's zeroed. */
2745         memset(ctx->authkey, 0, sizeof(ctx->authkey));
2746         memcpy(ctx->authkey, keys.authkey, keys.authkeylen);
2747
2748         switch (ctx->alg->cipher_info.alg) {
2749         case CIPHER_ALG_DES:
2750                 if (verify_aead_des_key(cipher, keys.enckey, keys.enckeylen))
2751                         return -EINVAL;
2752
2753                 ctx->cipher_type = CIPHER_TYPE_DES;
2754                 break;
2755         case CIPHER_ALG_3DES:
2756                 if (verify_aead_des3_key(cipher, keys.enckey, keys.enckeylen))
2757                         return -EINVAL;
2758
2759                 ctx->cipher_type = CIPHER_TYPE_3DES;
2760                 break;
2761         case CIPHER_ALG_AES:
2762                 switch (ctx->enckeylen) {
2763                 case AES_KEYSIZE_128:
2764                         ctx->cipher_type = CIPHER_TYPE_AES128;
2765                         break;
2766                 case AES_KEYSIZE_192:
2767                         ctx->cipher_type = CIPHER_TYPE_AES192;
2768                         break;
2769                 case AES_KEYSIZE_256:
2770                         ctx->cipher_type = CIPHER_TYPE_AES256;
2771                         break;
2772                 default:
2773                         goto badkey;
2774                 }
2775                 break;
2776         default:
2777                 pr_err("%s() Error: Unknown cipher alg\n", __func__);
2778                 return -EINVAL;
2779         }
2780
2781         flow_log("  enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
2782                  ctx->authkeylen);
2783         flow_dump("  enc: ", ctx->enckey, ctx->enckeylen);
2784         flow_dump("  auth: ", ctx->authkey, ctx->authkeylen);
2785
2786         /* setkey the fallback just in case we needto use it */
2787         if (ctx->fallback_cipher) {
2788                 flow_log("  running fallback setkey()\n");
2789
2790                 ctx->fallback_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
2791                 ctx->fallback_cipher->base.crt_flags |=
2792                     tfm->crt_flags & CRYPTO_TFM_REQ_MASK;
2793                 ret = crypto_aead_setkey(ctx->fallback_cipher, key, keylen);
2794                 if (ret)
2795                         flow_log("  fallback setkey() returned:%d\n", ret);
2796         }
2797
2798         ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
2799                                                           ctx->enckeylen,
2800                                                           false);
2801
2802         atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_AEAD]);
2803
2804         return ret;
2805
2806 badkey:
2807         ctx->enckeylen = 0;
2808         ctx->authkeylen = 0;
2809         ctx->digestsize = 0;
2810
2811         return -EINVAL;
2812 }
2813
2814 static int aead_gcm_ccm_setkey(struct crypto_aead *cipher,
2815                                const u8 *key, unsigned int keylen)
2816 {
2817         struct spu_hw *spu = &iproc_priv.spu;
2818         struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2819         struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
2820
2821         int ret = 0;
2822
2823         flow_log("%s() keylen:%u\n", __func__, keylen);
2824         flow_dump("  key: ", key, keylen);
2825
2826         if (!ctx->is_esp)
2827                 ctx->digestsize = keylen;
2828
2829         ctx->enckeylen = keylen;
2830         ctx->authkeylen = 0;
2831
2832         switch (ctx->enckeylen) {
2833         case AES_KEYSIZE_128:
2834                 ctx->cipher_type = CIPHER_TYPE_AES128;
2835                 break;
2836         case AES_KEYSIZE_192:
2837                 ctx->cipher_type = CIPHER_TYPE_AES192;
2838                 break;
2839         case AES_KEYSIZE_256:
2840                 ctx->cipher_type = CIPHER_TYPE_AES256;
2841                 break;
2842         default:
2843                 goto badkey;
2844         }
2845
2846         memcpy(ctx->enckey, key, ctx->enckeylen);
2847
2848         flow_log("  enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
2849                  ctx->authkeylen);
2850         flow_dump("  enc: ", ctx->enckey, ctx->enckeylen);
2851         flow_dump("  auth: ", ctx->authkey, ctx->authkeylen);
2852
2853         /* setkey the fallback just in case we need to use it */
2854         if (ctx->fallback_cipher) {
2855                 flow_log("  running fallback setkey()\n");
2856
2857                 ctx->fallback_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
2858                 ctx->fallback_cipher->base.crt_flags |=
2859                     tfm->crt_flags & CRYPTO_TFM_REQ_MASK;
2860                 ret = crypto_aead_setkey(ctx->fallback_cipher, key,
2861                                          keylen + ctx->salt_len);
2862                 if (ret)
2863                         flow_log("  fallback setkey() returned:%d\n", ret);
2864         }
2865
2866         ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
2867                                                           ctx->enckeylen,
2868                                                           false);
2869
2870         atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_AEAD]);
2871
2872         flow_log("  enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
2873                  ctx->authkeylen);
2874
2875         return ret;
2876
2877 badkey:
2878         ctx->enckeylen = 0;
2879         ctx->authkeylen = 0;
2880         ctx->digestsize = 0;
2881
2882         return -EINVAL;
2883 }
2884
2885 /**
2886  * aead_gcm_esp_setkey() - setkey() operation for ESP variant of GCM AES.
2887  * @cipher: AEAD structure
2888  * @key:    Key followed by 4 bytes of salt
2889  * @keylen: Length of key plus salt, in bytes
2890  *
2891  * Extracts salt from key and stores it to be prepended to IV on each request.
2892  * Digest is always 16 bytes
2893  *
2894  * Return: Value from generic gcm setkey.
2895  */
2896 static int aead_gcm_esp_setkey(struct crypto_aead *cipher,
2897                                const u8 *key, unsigned int keylen)
2898 {
2899         struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2900
2901         flow_log("%s\n", __func__);
2902
2903         if (keylen < GCM_ESP_SALT_SIZE)
2904                 return -EINVAL;
2905
2906         ctx->salt_len = GCM_ESP_SALT_SIZE;
2907         ctx->salt_offset = GCM_ESP_SALT_OFFSET;
2908         memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE);
2909         keylen -= GCM_ESP_SALT_SIZE;
2910         ctx->digestsize = GCM_ESP_DIGESTSIZE;
2911         ctx->is_esp = true;
2912         flow_dump("salt: ", ctx->salt, GCM_ESP_SALT_SIZE);
2913
2914         return aead_gcm_ccm_setkey(cipher, key, keylen);
2915 }
2916
2917 /**
2918  * rfc4543_gcm_esp_setkey() - setkey operation for RFC4543 variant of GCM/GMAC.
2919  * @cipher: AEAD structure
2920  * @key:    Key followed by 4 bytes of salt
2921  * @keylen: Length of key plus salt, in bytes
2922  *
2923  * Extracts salt from key and stores it to be prepended to IV on each request.
2924  * Digest is always 16 bytes
2925  *
2926  * Return: Value from generic gcm setkey.
2927  */
2928 static int rfc4543_gcm_esp_setkey(struct crypto_aead *cipher,
2929                                   const u8 *key, unsigned int keylen)
2930 {
2931         struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2932
2933         flow_log("%s\n", __func__);
2934
2935         if (keylen < GCM_ESP_SALT_SIZE)
2936                 return -EINVAL;
2937
2938         ctx->salt_len = GCM_ESP_SALT_SIZE;
2939         ctx->salt_offset = GCM_ESP_SALT_OFFSET;
2940         memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE);
2941         keylen -= GCM_ESP_SALT_SIZE;
2942         ctx->digestsize = GCM_ESP_DIGESTSIZE;
2943         ctx->is_esp = true;
2944         ctx->is_rfc4543 = true;
2945         flow_dump("salt: ", ctx->salt, GCM_ESP_SALT_SIZE);
2946
2947         return aead_gcm_ccm_setkey(cipher, key, keylen);
2948 }
2949
2950 /**
2951  * aead_ccm_esp_setkey() - setkey() operation for ESP variant of CCM AES.
2952  * @cipher: AEAD structure
2953  * @key:    Key followed by 4 bytes of salt
2954  * @keylen: Length of key plus salt, in bytes
2955  *
2956  * Extracts salt from key and stores it to be prepended to IV on each request.
2957  * Digest is always 16 bytes
2958  *
2959  * Return: Value from generic ccm setkey.
2960  */
2961 static int aead_ccm_esp_setkey(struct crypto_aead *cipher,
2962                                const u8 *key, unsigned int keylen)
2963 {
2964         struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2965
2966         flow_log("%s\n", __func__);
2967
2968         if (keylen < CCM_ESP_SALT_SIZE)
2969                 return -EINVAL;
2970
2971         ctx->salt_len = CCM_ESP_SALT_SIZE;
2972         ctx->salt_offset = CCM_ESP_SALT_OFFSET;
2973         memcpy(ctx->salt, key + keylen - CCM_ESP_SALT_SIZE, CCM_ESP_SALT_SIZE);
2974         keylen -= CCM_ESP_SALT_SIZE;
2975         ctx->is_esp = true;
2976         flow_dump("salt: ", ctx->salt, CCM_ESP_SALT_SIZE);
2977
2978         return aead_gcm_ccm_setkey(cipher, key, keylen);
2979 }
2980
2981 static int aead_setauthsize(struct crypto_aead *cipher, unsigned int authsize)
2982 {
2983         struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2984         int ret = 0;
2985
2986         flow_log("%s() authkeylen:%u authsize:%u\n",
2987                  __func__, ctx->authkeylen, authsize);
2988
2989         ctx->digestsize = authsize;
2990
2991         /* setkey the fallback just in case we needto use it */
2992         if (ctx->fallback_cipher) {
2993                 flow_log("  running fallback setauth()\n");
2994
2995                 ret = crypto_aead_setauthsize(ctx->fallback_cipher, authsize);
2996                 if (ret)
2997                         flow_log("  fallback setauth() returned:%d\n", ret);
2998         }
2999
3000         return ret;
3001 }
3002
3003 static int aead_encrypt(struct aead_request *req)
3004 {
3005         flow_log("%s() cryptlen:%u %08x\n", __func__, req->cryptlen,
3006                  req->cryptlen);
3007         dump_sg(req->src, 0, req->cryptlen + req->assoclen);
3008         flow_log("  assoc_len:%u\n", req->assoclen);
3009
3010         return aead_enqueue(req, true);
3011 }
3012
3013 static int aead_decrypt(struct aead_request *req)
3014 {
3015         flow_log("%s() cryptlen:%u\n", __func__, req->cryptlen);
3016         dump_sg(req->src, 0, req->cryptlen + req->assoclen);
3017         flow_log("  assoc_len:%u\n", req->assoclen);
3018
3019         return aead_enqueue(req, false);
3020 }
3021
3022 /* ==================== Supported Cipher Algorithms ==================== */
3023
3024 static struct iproc_alg_s driver_algs[] = {
3025         {
3026          .type = CRYPTO_ALG_TYPE_AEAD,
3027          .alg.aead = {
3028                  .base = {
3029                         .cra_name = "gcm(aes)",
3030                         .cra_driver_name = "gcm-aes-iproc",
3031                         .cra_blocksize = AES_BLOCK_SIZE,
3032                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3033                  },
3034                  .setkey = aead_gcm_ccm_setkey,
3035                  .ivsize = GCM_AES_IV_SIZE,
3036                 .maxauthsize = AES_BLOCK_SIZE,
3037          },
3038          .cipher_info = {
3039                          .alg = CIPHER_ALG_AES,
3040                          .mode = CIPHER_MODE_GCM,
3041                          },
3042          .auth_info = {
3043                        .alg = HASH_ALG_AES,
3044                        .mode = HASH_MODE_GCM,
3045                        },
3046          .auth_first = 0,
3047          },
3048         {
3049          .type = CRYPTO_ALG_TYPE_AEAD,
3050          .alg.aead = {
3051                  .base = {
3052                         .cra_name = "ccm(aes)",
3053                         .cra_driver_name = "ccm-aes-iproc",
3054                         .cra_blocksize = AES_BLOCK_SIZE,
3055                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3056                  },
3057                  .setkey = aead_gcm_ccm_setkey,
3058                  .ivsize = CCM_AES_IV_SIZE,
3059                 .maxauthsize = AES_BLOCK_SIZE,
3060          },
3061          .cipher_info = {
3062                          .alg = CIPHER_ALG_AES,
3063                          .mode = CIPHER_MODE_CCM,
3064                          },
3065          .auth_info = {
3066                        .alg = HASH_ALG_AES,
3067                        .mode = HASH_MODE_CCM,
3068                        },
3069          .auth_first = 0,
3070          },
3071         {
3072          .type = CRYPTO_ALG_TYPE_AEAD,
3073          .alg.aead = {
3074                  .base = {
3075                         .cra_name = "rfc4106(gcm(aes))",
3076                         .cra_driver_name = "gcm-aes-esp-iproc",
3077                         .cra_blocksize = AES_BLOCK_SIZE,
3078                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3079                  },
3080                  .setkey = aead_gcm_esp_setkey,
3081                  .ivsize = GCM_RFC4106_IV_SIZE,
3082                  .maxauthsize = AES_BLOCK_SIZE,
3083          },
3084          .cipher_info = {
3085                          .alg = CIPHER_ALG_AES,
3086                          .mode = CIPHER_MODE_GCM,
3087                          },
3088          .auth_info = {
3089                        .alg = HASH_ALG_AES,
3090                        .mode = HASH_MODE_GCM,
3091                        },
3092          .auth_first = 0,
3093          },
3094         {
3095          .type = CRYPTO_ALG_TYPE_AEAD,
3096          .alg.aead = {
3097                  .base = {
3098                         .cra_name = "rfc4309(ccm(aes))",
3099                         .cra_driver_name = "ccm-aes-esp-iproc",
3100                         .cra_blocksize = AES_BLOCK_SIZE,
3101                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3102                  },
3103                  .setkey = aead_ccm_esp_setkey,
3104                  .ivsize = CCM_AES_IV_SIZE,
3105                  .maxauthsize = AES_BLOCK_SIZE,
3106          },
3107          .cipher_info = {
3108                          .alg = CIPHER_ALG_AES,
3109                          .mode = CIPHER_MODE_CCM,
3110                          },
3111          .auth_info = {
3112                        .alg = HASH_ALG_AES,
3113                        .mode = HASH_MODE_CCM,
3114                        },
3115          .auth_first = 0,
3116          },
3117         {
3118          .type = CRYPTO_ALG_TYPE_AEAD,
3119          .alg.aead = {
3120                  .base = {
3121                         .cra_name = "rfc4543(gcm(aes))",
3122                         .cra_driver_name = "gmac-aes-esp-iproc",
3123                         .cra_blocksize = AES_BLOCK_SIZE,
3124                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3125                  },
3126                  .setkey = rfc4543_gcm_esp_setkey,
3127                  .ivsize = GCM_RFC4106_IV_SIZE,
3128                  .maxauthsize = AES_BLOCK_SIZE,
3129          },
3130          .cipher_info = {
3131                          .alg = CIPHER_ALG_AES,
3132                          .mode = CIPHER_MODE_GCM,
3133                          },
3134          .auth_info = {
3135                        .alg = HASH_ALG_AES,
3136                        .mode = HASH_MODE_GCM,
3137                        },
3138          .auth_first = 0,
3139          },
3140         {
3141          .type = CRYPTO_ALG_TYPE_AEAD,
3142          .alg.aead = {
3143                  .base = {
3144                         .cra_name = "authenc(hmac(md5),cbc(aes))",
3145                         .cra_driver_name = "authenc-hmac-md5-cbc-aes-iproc",
3146                         .cra_blocksize = AES_BLOCK_SIZE,
3147                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3148                                      CRYPTO_ALG_ASYNC |
3149                                      CRYPTO_ALG_ALLOCATES_MEMORY
3150                  },
3151                  .setkey = aead_authenc_setkey,
3152                 .ivsize = AES_BLOCK_SIZE,
3153                 .maxauthsize = MD5_DIGEST_SIZE,
3154          },
3155          .cipher_info = {
3156                          .alg = CIPHER_ALG_AES,
3157                          .mode = CIPHER_MODE_CBC,
3158                          },
3159          .auth_info = {
3160                        .alg = HASH_ALG_MD5,
3161                        .mode = HASH_MODE_HMAC,
3162                        },
3163          .auth_first = 0,
3164          },
3165         {
3166          .type = CRYPTO_ALG_TYPE_AEAD,
3167          .alg.aead = {
3168                  .base = {
3169                         .cra_name = "authenc(hmac(sha1),cbc(aes))",
3170                         .cra_driver_name = "authenc-hmac-sha1-cbc-aes-iproc",
3171                         .cra_blocksize = AES_BLOCK_SIZE,
3172                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3173                                      CRYPTO_ALG_ASYNC |
3174                                      CRYPTO_ALG_ALLOCATES_MEMORY
3175                  },
3176                  .setkey = aead_authenc_setkey,
3177                  .ivsize = AES_BLOCK_SIZE,
3178                  .maxauthsize = SHA1_DIGEST_SIZE,
3179          },
3180          .cipher_info = {
3181                          .alg = CIPHER_ALG_AES,
3182                          .mode = CIPHER_MODE_CBC,
3183                          },
3184          .auth_info = {
3185                        .alg = HASH_ALG_SHA1,
3186                        .mode = HASH_MODE_HMAC,
3187                        },
3188          .auth_first = 0,
3189          },
3190         {
3191          .type = CRYPTO_ALG_TYPE_AEAD,
3192          .alg.aead = {
3193                  .base = {
3194                         .cra_name = "authenc(hmac(sha256),cbc(aes))",
3195                         .cra_driver_name = "authenc-hmac-sha256-cbc-aes-iproc",
3196                         .cra_blocksize = AES_BLOCK_SIZE,
3197                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3198                                      CRYPTO_ALG_ASYNC |
3199                                      CRYPTO_ALG_ALLOCATES_MEMORY
3200                  },
3201                  .setkey = aead_authenc_setkey,
3202                  .ivsize = AES_BLOCK_SIZE,
3203                  .maxauthsize = SHA256_DIGEST_SIZE,
3204          },
3205          .cipher_info = {
3206                          .alg = CIPHER_ALG_AES,
3207                          .mode = CIPHER_MODE_CBC,
3208                          },
3209          .auth_info = {
3210                        .alg = HASH_ALG_SHA256,
3211                        .mode = HASH_MODE_HMAC,
3212                        },
3213          .auth_first = 0,
3214          },
3215         {
3216          .type = CRYPTO_ALG_TYPE_AEAD,
3217          .alg.aead = {
3218                  .base = {
3219                         .cra_name = "authenc(hmac(md5),cbc(des))",
3220                         .cra_driver_name = "authenc-hmac-md5-cbc-des-iproc",
3221                         .cra_blocksize = DES_BLOCK_SIZE,
3222                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3223                                      CRYPTO_ALG_ASYNC |
3224                                      CRYPTO_ALG_ALLOCATES_MEMORY
3225                  },
3226                  .setkey = aead_authenc_setkey,
3227                  .ivsize = DES_BLOCK_SIZE,
3228                  .maxauthsize = MD5_DIGEST_SIZE,
3229          },
3230          .cipher_info = {
3231                          .alg = CIPHER_ALG_DES,
3232                          .mode = CIPHER_MODE_CBC,
3233                          },
3234          .auth_info = {
3235                        .alg = HASH_ALG_MD5,
3236                        .mode = HASH_MODE_HMAC,
3237                        },
3238          .auth_first = 0,
3239          },
3240         {
3241          .type = CRYPTO_ALG_TYPE_AEAD,
3242          .alg.aead = {
3243                  .base = {
3244                         .cra_name = "authenc(hmac(sha1),cbc(des))",
3245                         .cra_driver_name = "authenc-hmac-sha1-cbc-des-iproc",
3246                         .cra_blocksize = DES_BLOCK_SIZE,
3247                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3248                                      CRYPTO_ALG_ASYNC |
3249                                      CRYPTO_ALG_ALLOCATES_MEMORY
3250                  },
3251                  .setkey = aead_authenc_setkey,
3252                  .ivsize = DES_BLOCK_SIZE,
3253                  .maxauthsize = SHA1_DIGEST_SIZE,
3254          },
3255          .cipher_info = {
3256                          .alg = CIPHER_ALG_DES,
3257                          .mode = CIPHER_MODE_CBC,
3258                          },
3259          .auth_info = {
3260                        .alg = HASH_ALG_SHA1,
3261                        .mode = HASH_MODE_HMAC,
3262                        },
3263          .auth_first = 0,
3264          },
3265         {
3266          .type = CRYPTO_ALG_TYPE_AEAD,
3267          .alg.aead = {
3268                  .base = {
3269                         .cra_name = "authenc(hmac(sha224),cbc(des))",
3270                         .cra_driver_name = "authenc-hmac-sha224-cbc-des-iproc",
3271                         .cra_blocksize = DES_BLOCK_SIZE,
3272                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3273                                      CRYPTO_ALG_ASYNC |
3274                                      CRYPTO_ALG_ALLOCATES_MEMORY
3275                  },
3276                  .setkey = aead_authenc_setkey,
3277                  .ivsize = DES_BLOCK_SIZE,
3278                  .maxauthsize = SHA224_DIGEST_SIZE,
3279          },
3280          .cipher_info = {
3281                          .alg = CIPHER_ALG_DES,
3282                          .mode = CIPHER_MODE_CBC,
3283                          },
3284          .auth_info = {
3285                        .alg = HASH_ALG_SHA224,
3286                        .mode = HASH_MODE_HMAC,
3287                        },
3288          .auth_first = 0,
3289          },
3290         {
3291          .type = CRYPTO_ALG_TYPE_AEAD,
3292          .alg.aead = {
3293                  .base = {
3294                         .cra_name = "authenc(hmac(sha256),cbc(des))",
3295                         .cra_driver_name = "authenc-hmac-sha256-cbc-des-iproc",
3296                         .cra_blocksize = DES_BLOCK_SIZE,
3297                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3298                                      CRYPTO_ALG_ASYNC |
3299                                      CRYPTO_ALG_ALLOCATES_MEMORY
3300                  },
3301                  .setkey = aead_authenc_setkey,
3302                  .ivsize = DES_BLOCK_SIZE,
3303                  .maxauthsize = SHA256_DIGEST_SIZE,
3304          },
3305          .cipher_info = {
3306                          .alg = CIPHER_ALG_DES,
3307                          .mode = CIPHER_MODE_CBC,
3308                          },
3309          .auth_info = {
3310                        .alg = HASH_ALG_SHA256,
3311                        .mode = HASH_MODE_HMAC,
3312                        },
3313          .auth_first = 0,
3314          },
3315         {
3316          .type = CRYPTO_ALG_TYPE_AEAD,
3317          .alg.aead = {
3318                  .base = {
3319                         .cra_name = "authenc(hmac(sha384),cbc(des))",
3320                         .cra_driver_name = "authenc-hmac-sha384-cbc-des-iproc",
3321                         .cra_blocksize = DES_BLOCK_SIZE,
3322                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3323                                      CRYPTO_ALG_ASYNC |
3324                                      CRYPTO_ALG_ALLOCATES_MEMORY
3325                  },
3326                  .setkey = aead_authenc_setkey,
3327                  .ivsize = DES_BLOCK_SIZE,
3328                  .maxauthsize = SHA384_DIGEST_SIZE,
3329          },
3330          .cipher_info = {
3331                          .alg = CIPHER_ALG_DES,
3332                          .mode = CIPHER_MODE_CBC,
3333                          },
3334          .auth_info = {
3335                        .alg = HASH_ALG_SHA384,
3336                        .mode = HASH_MODE_HMAC,
3337                        },
3338          .auth_first = 0,
3339          },
3340         {
3341          .type = CRYPTO_ALG_TYPE_AEAD,
3342          .alg.aead = {
3343                  .base = {
3344                         .cra_name = "authenc(hmac(sha512),cbc(des))",
3345                         .cra_driver_name = "authenc-hmac-sha512-cbc-des-iproc",
3346                         .cra_blocksize = DES_BLOCK_SIZE,
3347                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3348                                      CRYPTO_ALG_ASYNC |
3349                                      CRYPTO_ALG_ALLOCATES_MEMORY
3350                  },
3351                  .setkey = aead_authenc_setkey,
3352                  .ivsize = DES_BLOCK_SIZE,
3353                  .maxauthsize = SHA512_DIGEST_SIZE,
3354          },
3355          .cipher_info = {
3356                          .alg = CIPHER_ALG_DES,
3357                          .mode = CIPHER_MODE_CBC,
3358                          },
3359          .auth_info = {
3360                        .alg = HASH_ALG_SHA512,
3361                        .mode = HASH_MODE_HMAC,
3362                        },
3363          .auth_first = 0,
3364          },
3365         {
3366          .type = CRYPTO_ALG_TYPE_AEAD,
3367          .alg.aead = {
3368                  .base = {
3369                         .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
3370                         .cra_driver_name = "authenc-hmac-md5-cbc-des3-iproc",
3371                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3372                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3373                                      CRYPTO_ALG_ASYNC |
3374                                      CRYPTO_ALG_ALLOCATES_MEMORY
3375                  },
3376                  .setkey = aead_authenc_setkey,
3377                  .ivsize = DES3_EDE_BLOCK_SIZE,
3378                  .maxauthsize = MD5_DIGEST_SIZE,
3379          },
3380          .cipher_info = {
3381                          .alg = CIPHER_ALG_3DES,
3382                          .mode = CIPHER_MODE_CBC,
3383                          },
3384          .auth_info = {
3385                        .alg = HASH_ALG_MD5,
3386                        .mode = HASH_MODE_HMAC,
3387                        },
3388          .auth_first = 0,
3389          },
3390         {
3391          .type = CRYPTO_ALG_TYPE_AEAD,
3392          .alg.aead = {
3393                  .base = {
3394                         .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
3395                         .cra_driver_name = "authenc-hmac-sha1-cbc-des3-iproc",
3396                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3397                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3398                                      CRYPTO_ALG_ASYNC |
3399                                      CRYPTO_ALG_ALLOCATES_MEMORY
3400                  },
3401                  .setkey = aead_authenc_setkey,
3402                  .ivsize = DES3_EDE_BLOCK_SIZE,
3403                  .maxauthsize = SHA1_DIGEST_SIZE,
3404          },
3405          .cipher_info = {
3406                          .alg = CIPHER_ALG_3DES,
3407                          .mode = CIPHER_MODE_CBC,
3408                          },
3409          .auth_info = {
3410                        .alg = HASH_ALG_SHA1,
3411                        .mode = HASH_MODE_HMAC,
3412                        },
3413          .auth_first = 0,
3414          },
3415         {
3416          .type = CRYPTO_ALG_TYPE_AEAD,
3417          .alg.aead = {
3418                  .base = {
3419                         .cra_name = "authenc(hmac(sha224),cbc(des3_ede))",
3420                         .cra_driver_name = "authenc-hmac-sha224-cbc-des3-iproc",
3421                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3422                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3423                                      CRYPTO_ALG_ASYNC |
3424                                      CRYPTO_ALG_ALLOCATES_MEMORY
3425                  },
3426                  .setkey = aead_authenc_setkey,
3427                  .ivsize = DES3_EDE_BLOCK_SIZE,
3428                  .maxauthsize = SHA224_DIGEST_SIZE,
3429          },
3430          .cipher_info = {
3431                          .alg = CIPHER_ALG_3DES,
3432                          .mode = CIPHER_MODE_CBC,
3433                          },
3434          .auth_info = {
3435                        .alg = HASH_ALG_SHA224,
3436                        .mode = HASH_MODE_HMAC,
3437                        },
3438          .auth_first = 0,
3439          },
3440         {
3441          .type = CRYPTO_ALG_TYPE_AEAD,
3442          .alg.aead = {
3443                  .base = {
3444                         .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
3445                         .cra_driver_name = "authenc-hmac-sha256-cbc-des3-iproc",
3446                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3447                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3448                                      CRYPTO_ALG_ASYNC |
3449                                      CRYPTO_ALG_ALLOCATES_MEMORY
3450                  },
3451                  .setkey = aead_authenc_setkey,
3452                  .ivsize = DES3_EDE_BLOCK_SIZE,
3453                  .maxauthsize = SHA256_DIGEST_SIZE,
3454          },
3455          .cipher_info = {
3456                          .alg = CIPHER_ALG_3DES,
3457                          .mode = CIPHER_MODE_CBC,
3458                          },
3459          .auth_info = {
3460                        .alg = HASH_ALG_SHA256,
3461                        .mode = HASH_MODE_HMAC,
3462                        },
3463          .auth_first = 0,
3464          },
3465         {
3466          .type = CRYPTO_ALG_TYPE_AEAD,
3467          .alg.aead = {
3468                  .base = {
3469                         .cra_name = "authenc(hmac(sha384),cbc(des3_ede))",
3470                         .cra_driver_name = "authenc-hmac-sha384-cbc-des3-iproc",
3471                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3472                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3473                                      CRYPTO_ALG_ASYNC |
3474                                      CRYPTO_ALG_ALLOCATES_MEMORY
3475                  },
3476                  .setkey = aead_authenc_setkey,
3477                  .ivsize = DES3_EDE_BLOCK_SIZE,
3478                  .maxauthsize = SHA384_DIGEST_SIZE,
3479          },
3480          .cipher_info = {
3481                          .alg = CIPHER_ALG_3DES,
3482                          .mode = CIPHER_MODE_CBC,
3483                          },
3484          .auth_info = {
3485                        .alg = HASH_ALG_SHA384,
3486                        .mode = HASH_MODE_HMAC,
3487                        },
3488          .auth_first = 0,
3489          },
3490         {
3491          .type = CRYPTO_ALG_TYPE_AEAD,
3492          .alg.aead = {
3493                  .base = {
3494                         .cra_name = "authenc(hmac(sha512),cbc(des3_ede))",
3495                         .cra_driver_name = "authenc-hmac-sha512-cbc-des3-iproc",
3496                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3497                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3498                                      CRYPTO_ALG_ASYNC |
3499                                      CRYPTO_ALG_ALLOCATES_MEMORY
3500                  },
3501                  .setkey = aead_authenc_setkey,
3502                  .ivsize = DES3_EDE_BLOCK_SIZE,
3503                  .maxauthsize = SHA512_DIGEST_SIZE,
3504          },
3505          .cipher_info = {
3506                          .alg = CIPHER_ALG_3DES,
3507                          .mode = CIPHER_MODE_CBC,
3508                          },
3509          .auth_info = {
3510                        .alg = HASH_ALG_SHA512,
3511                        .mode = HASH_MODE_HMAC,
3512                        },
3513          .auth_first = 0,
3514          },
3515
3516 /* SKCIPHER algorithms. */
3517         {
3518          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3519          .alg.skcipher = {
3520                         .base.cra_name = "ofb(des)",
3521                         .base.cra_driver_name = "ofb-des-iproc",
3522                         .base.cra_blocksize = DES_BLOCK_SIZE,
3523                         .min_keysize = DES_KEY_SIZE,
3524                         .max_keysize = DES_KEY_SIZE,
3525                         .ivsize = DES_BLOCK_SIZE,
3526                         },
3527          .cipher_info = {
3528                          .alg = CIPHER_ALG_DES,
3529                          .mode = CIPHER_MODE_OFB,
3530                          },
3531          .auth_info = {
3532                        .alg = HASH_ALG_NONE,
3533                        .mode = HASH_MODE_NONE,
3534                        },
3535          },
3536         {
3537          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3538          .alg.skcipher = {
3539                         .base.cra_name = "cbc(des)",
3540                         .base.cra_driver_name = "cbc-des-iproc",
3541                         .base.cra_blocksize = DES_BLOCK_SIZE,
3542                         .min_keysize = DES_KEY_SIZE,
3543                         .max_keysize = DES_KEY_SIZE,
3544                         .ivsize = DES_BLOCK_SIZE,
3545                         },
3546          .cipher_info = {
3547                          .alg = CIPHER_ALG_DES,
3548                          .mode = CIPHER_MODE_CBC,
3549                          },
3550          .auth_info = {
3551                        .alg = HASH_ALG_NONE,
3552                        .mode = HASH_MODE_NONE,
3553                        },
3554          },
3555         {
3556          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3557          .alg.skcipher = {
3558                         .base.cra_name = "ecb(des)",
3559                         .base.cra_driver_name = "ecb-des-iproc",
3560                         .base.cra_blocksize = DES_BLOCK_SIZE,
3561                         .min_keysize = DES_KEY_SIZE,
3562                         .max_keysize = DES_KEY_SIZE,
3563                         .ivsize = 0,
3564                         },
3565          .cipher_info = {
3566                          .alg = CIPHER_ALG_DES,
3567                          .mode = CIPHER_MODE_ECB,
3568                          },
3569          .auth_info = {
3570                        .alg = HASH_ALG_NONE,
3571                        .mode = HASH_MODE_NONE,
3572                        },
3573          },
3574         {
3575          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3576          .alg.skcipher = {
3577                         .base.cra_name = "ofb(des3_ede)",
3578                         .base.cra_driver_name = "ofb-des3-iproc",
3579                         .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3580                         .min_keysize = DES3_EDE_KEY_SIZE,
3581                         .max_keysize = DES3_EDE_KEY_SIZE,
3582                         .ivsize = DES3_EDE_BLOCK_SIZE,
3583                         },
3584          .cipher_info = {
3585                          .alg = CIPHER_ALG_3DES,
3586                          .mode = CIPHER_MODE_OFB,
3587                          },
3588          .auth_info = {
3589                        .alg = HASH_ALG_NONE,
3590                        .mode = HASH_MODE_NONE,
3591                        },
3592          },
3593         {
3594          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3595          .alg.skcipher = {
3596                         .base.cra_name = "cbc(des3_ede)",
3597                         .base.cra_driver_name = "cbc-des3-iproc",
3598                         .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3599                         .min_keysize = DES3_EDE_KEY_SIZE,
3600                         .max_keysize = DES3_EDE_KEY_SIZE,
3601                         .ivsize = DES3_EDE_BLOCK_SIZE,
3602                         },
3603          .cipher_info = {
3604                          .alg = CIPHER_ALG_3DES,
3605                          .mode = CIPHER_MODE_CBC,
3606                          },
3607          .auth_info = {
3608                        .alg = HASH_ALG_NONE,
3609                        .mode = HASH_MODE_NONE,
3610                        },
3611          },
3612         {
3613          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3614          .alg.skcipher = {
3615                         .base.cra_name = "ecb(des3_ede)",
3616                         .base.cra_driver_name = "ecb-des3-iproc",
3617                         .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3618                         .min_keysize = DES3_EDE_KEY_SIZE,
3619                         .max_keysize = DES3_EDE_KEY_SIZE,
3620                         .ivsize = 0,
3621                         },
3622          .cipher_info = {
3623                          .alg = CIPHER_ALG_3DES,
3624                          .mode = CIPHER_MODE_ECB,
3625                          },
3626          .auth_info = {
3627                        .alg = HASH_ALG_NONE,
3628                        .mode = HASH_MODE_NONE,
3629                        },
3630          },
3631         {
3632          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3633          .alg.skcipher = {
3634                         .base.cra_name = "ofb(aes)",
3635                         .base.cra_driver_name = "ofb-aes-iproc",
3636                         .base.cra_blocksize = AES_BLOCK_SIZE,
3637                         .min_keysize = AES_MIN_KEY_SIZE,
3638                         .max_keysize = AES_MAX_KEY_SIZE,
3639                         .ivsize = AES_BLOCK_SIZE,
3640                         },
3641          .cipher_info = {
3642                          .alg = CIPHER_ALG_AES,
3643                          .mode = CIPHER_MODE_OFB,
3644                          },
3645          .auth_info = {
3646                        .alg = HASH_ALG_NONE,
3647                        .mode = HASH_MODE_NONE,
3648                        },
3649          },
3650         {
3651          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3652          .alg.skcipher = {
3653                         .base.cra_name = "cbc(aes)",
3654                         .base.cra_driver_name = "cbc-aes-iproc",
3655                         .base.cra_blocksize = AES_BLOCK_SIZE,
3656                         .min_keysize = AES_MIN_KEY_SIZE,
3657                         .max_keysize = AES_MAX_KEY_SIZE,
3658                         .ivsize = AES_BLOCK_SIZE,
3659                         },
3660          .cipher_info = {
3661                          .alg = CIPHER_ALG_AES,
3662                          .mode = CIPHER_MODE_CBC,
3663                          },
3664          .auth_info = {
3665                        .alg = HASH_ALG_NONE,
3666                        .mode = HASH_MODE_NONE,
3667                        },
3668          },
3669         {
3670          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3671          .alg.skcipher = {
3672                         .base.cra_name = "ecb(aes)",
3673                         .base.cra_driver_name = "ecb-aes-iproc",
3674                         .base.cra_blocksize = AES_BLOCK_SIZE,
3675                         .min_keysize = AES_MIN_KEY_SIZE,
3676                         .max_keysize = AES_MAX_KEY_SIZE,
3677                         .ivsize = 0,
3678                         },
3679          .cipher_info = {
3680                          .alg = CIPHER_ALG_AES,
3681                          .mode = CIPHER_MODE_ECB,
3682                          },
3683          .auth_info = {
3684                        .alg = HASH_ALG_NONE,
3685                        .mode = HASH_MODE_NONE,
3686                        },
3687          },
3688         {
3689          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3690          .alg.skcipher = {
3691                         .base.cra_name = "ctr(aes)",
3692                         .base.cra_driver_name = "ctr-aes-iproc",
3693                         .base.cra_blocksize = AES_BLOCK_SIZE,
3694                         .min_keysize = AES_MIN_KEY_SIZE,
3695                         .max_keysize = AES_MAX_KEY_SIZE,
3696                         .ivsize = AES_BLOCK_SIZE,
3697                         },
3698          .cipher_info = {
3699                          .alg = CIPHER_ALG_AES,
3700                          .mode = CIPHER_MODE_CTR,
3701                          },
3702          .auth_info = {
3703                        .alg = HASH_ALG_NONE,
3704                        .mode = HASH_MODE_NONE,
3705                        },
3706          },
3707 {
3708          .type = CRYPTO_ALG_TYPE_SKCIPHER,
3709          .alg.skcipher = {
3710                         .base.cra_name = "xts(aes)",
3711                         .base.cra_driver_name = "xts-aes-iproc",
3712                         .base.cra_blocksize = AES_BLOCK_SIZE,
3713                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
3714                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
3715                         .ivsize = AES_BLOCK_SIZE,
3716                         },
3717          .cipher_info = {
3718                          .alg = CIPHER_ALG_AES,
3719                          .mode = CIPHER_MODE_XTS,
3720                          },
3721          .auth_info = {
3722                        .alg = HASH_ALG_NONE,
3723                        .mode = HASH_MODE_NONE,
3724                        },
3725          },
3726
3727 /* AHASH algorithms. */
3728         {
3729          .type = CRYPTO_ALG_TYPE_AHASH,
3730          .alg.hash = {
3731                       .halg.digestsize = MD5_DIGEST_SIZE,
3732                       .halg.base = {
3733                                     .cra_name = "md5",
3734                                     .cra_driver_name = "md5-iproc",
3735                                     .cra_blocksize = MD5_BLOCK_WORDS * 4,
3736                                     .cra_flags = CRYPTO_ALG_ASYNC |
3737                                                  CRYPTO_ALG_ALLOCATES_MEMORY,
3738                                 }
3739                       },
3740          .cipher_info = {
3741                          .alg = CIPHER_ALG_NONE,
3742                          .mode = CIPHER_MODE_NONE,
3743                          },
3744          .auth_info = {
3745                        .alg = HASH_ALG_MD5,
3746                        .mode = HASH_MODE_HASH,
3747                        },
3748          },
3749         {
3750          .type = CRYPTO_ALG_TYPE_AHASH,
3751          .alg.hash = {
3752                       .halg.digestsize = MD5_DIGEST_SIZE,
3753                       .halg.base = {
3754                                     .cra_name = "hmac(md5)",
3755                                     .cra_driver_name = "hmac-md5-iproc",
3756                                     .cra_blocksize = MD5_BLOCK_WORDS * 4,
3757                                 }
3758                       },
3759          .cipher_info = {
3760                          .alg = CIPHER_ALG_NONE,
3761                          .mode = CIPHER_MODE_NONE,
3762                          },
3763          .auth_info = {
3764                        .alg = HASH_ALG_MD5,
3765                        .mode = HASH_MODE_HMAC,
3766                        },
3767          },
3768         {.type = CRYPTO_ALG_TYPE_AHASH,
3769          .alg.hash = {
3770                       .halg.digestsize = SHA1_DIGEST_SIZE,
3771                       .halg.base = {
3772                                     .cra_name = "sha1",
3773                                     .cra_driver_name = "sha1-iproc",
3774                                     .cra_blocksize = SHA1_BLOCK_SIZE,
3775                                 }
3776                       },
3777          .cipher_info = {
3778                          .alg = CIPHER_ALG_NONE,
3779                          .mode = CIPHER_MODE_NONE,
3780                          },
3781          .auth_info = {
3782                        .alg = HASH_ALG_SHA1,
3783                        .mode = HASH_MODE_HASH,
3784                        },
3785          },
3786         {.type = CRYPTO_ALG_TYPE_AHASH,
3787          .alg.hash = {
3788                       .halg.digestsize = SHA1_DIGEST_SIZE,
3789                       .halg.base = {
3790                                     .cra_name = "hmac(sha1)",
3791                                     .cra_driver_name = "hmac-sha1-iproc",
3792                                     .cra_blocksize = SHA1_BLOCK_SIZE,
3793                                 }
3794                       },
3795          .cipher_info = {
3796                          .alg = CIPHER_ALG_NONE,
3797                          .mode = CIPHER_MODE_NONE,
3798                          },
3799          .auth_info = {
3800                        .alg = HASH_ALG_SHA1,
3801                        .mode = HASH_MODE_HMAC,
3802                        },
3803          },
3804         {.type = CRYPTO_ALG_TYPE_AHASH,
3805          .alg.hash = {
3806                         .halg.digestsize = SHA224_DIGEST_SIZE,
3807                         .halg.base = {
3808                                     .cra_name = "sha224",
3809                                     .cra_driver_name = "sha224-iproc",
3810                                     .cra_blocksize = SHA224_BLOCK_SIZE,
3811                         }
3812                       },
3813          .cipher_info = {
3814                          .alg = CIPHER_ALG_NONE,
3815                          .mode = CIPHER_MODE_NONE,
3816                          },
3817          .auth_info = {
3818                        .alg = HASH_ALG_SHA224,
3819                        .mode = HASH_MODE_HASH,
3820                        },
3821          },
3822         {.type = CRYPTO_ALG_TYPE_AHASH,
3823          .alg.hash = {
3824                       .halg.digestsize = SHA224_DIGEST_SIZE,
3825                       .halg.base = {
3826                                     .cra_name = "hmac(sha224)",
3827                                     .cra_driver_name = "hmac-sha224-iproc",
3828                                     .cra_blocksize = SHA224_BLOCK_SIZE,
3829                                 }
3830                       },
3831          .cipher_info = {
3832                          .alg = CIPHER_ALG_NONE,
3833                          .mode = CIPHER_MODE_NONE,
3834                          },
3835          .auth_info = {
3836                        .alg = HASH_ALG_SHA224,
3837                        .mode = HASH_MODE_HMAC,
3838                        },
3839          },
3840         {.type = CRYPTO_ALG_TYPE_AHASH,
3841          .alg.hash = {
3842                       .halg.digestsize = SHA256_DIGEST_SIZE,
3843                       .halg.base = {
3844                                     .cra_name = "sha256",
3845                                     .cra_driver_name = "sha256-iproc",
3846                                     .cra_blocksize = SHA256_BLOCK_SIZE,
3847                                 }
3848                       },
3849          .cipher_info = {
3850                          .alg = CIPHER_ALG_NONE,
3851                          .mode = CIPHER_MODE_NONE,
3852                          },
3853          .auth_info = {
3854                        .alg = HASH_ALG_SHA256,
3855                        .mode = HASH_MODE_HASH,
3856                        },
3857          },
3858         {.type = CRYPTO_ALG_TYPE_AHASH,
3859          .alg.hash = {
3860                       .halg.digestsize = SHA256_DIGEST_SIZE,
3861                       .halg.base = {
3862                                     .cra_name = "hmac(sha256)",
3863                                     .cra_driver_name = "hmac-sha256-iproc",
3864                                     .cra_blocksize = SHA256_BLOCK_SIZE,
3865                                 }
3866                       },
3867          .cipher_info = {
3868                          .alg = CIPHER_ALG_NONE,
3869                          .mode = CIPHER_MODE_NONE,
3870                          },
3871          .auth_info = {
3872                        .alg = HASH_ALG_SHA256,
3873                        .mode = HASH_MODE_HMAC,
3874                        },
3875          },
3876         {
3877         .type = CRYPTO_ALG_TYPE_AHASH,
3878          .alg.hash = {
3879                       .halg.digestsize = SHA384_DIGEST_SIZE,
3880                       .halg.base = {
3881                                     .cra_name = "sha384",
3882                                     .cra_driver_name = "sha384-iproc",
3883                                     .cra_blocksize = SHA384_BLOCK_SIZE,
3884                                 }
3885                       },
3886          .cipher_info = {
3887                          .alg = CIPHER_ALG_NONE,
3888                          .mode = CIPHER_MODE_NONE,
3889                          },
3890          .auth_info = {
3891                        .alg = HASH_ALG_SHA384,
3892                        .mode = HASH_MODE_HASH,
3893                        },
3894          },
3895         {
3896          .type = CRYPTO_ALG_TYPE_AHASH,
3897          .alg.hash = {
3898                       .halg.digestsize = SHA384_DIGEST_SIZE,
3899                       .halg.base = {
3900                                     .cra_name = "hmac(sha384)",
3901                                     .cra_driver_name = "hmac-sha384-iproc",
3902                                     .cra_blocksize = SHA384_BLOCK_SIZE,
3903                                 }
3904                       },
3905          .cipher_info = {
3906                          .alg = CIPHER_ALG_NONE,
3907                          .mode = CIPHER_MODE_NONE,
3908                          },
3909          .auth_info = {
3910                        .alg = HASH_ALG_SHA384,
3911                        .mode = HASH_MODE_HMAC,
3912                        },
3913          },
3914         {
3915          .type = CRYPTO_ALG_TYPE_AHASH,
3916          .alg.hash = {
3917                       .halg.digestsize = SHA512_DIGEST_SIZE,
3918                       .halg.base = {
3919                                     .cra_name = "sha512",
3920                                     .cra_driver_name = "sha512-iproc",
3921                                     .cra_blocksize = SHA512_BLOCK_SIZE,
3922                                 }
3923                       },
3924          .cipher_info = {
3925                          .alg = CIPHER_ALG_NONE,
3926                          .mode = CIPHER_MODE_NONE,
3927                          },
3928          .auth_info = {
3929                        .alg = HASH_ALG_SHA512,
3930                        .mode = HASH_MODE_HASH,
3931                        },
3932          },
3933         {
3934          .type = CRYPTO_ALG_TYPE_AHASH,
3935          .alg.hash = {
3936                       .halg.digestsize = SHA512_DIGEST_SIZE,
3937                       .halg.base = {
3938                                     .cra_name = "hmac(sha512)",
3939                                     .cra_driver_name = "hmac-sha512-iproc",
3940                                     .cra_blocksize = SHA512_BLOCK_SIZE,
3941                                 }
3942                       },
3943          .cipher_info = {
3944                          .alg = CIPHER_ALG_NONE,
3945                          .mode = CIPHER_MODE_NONE,
3946                          },
3947          .auth_info = {
3948                        .alg = HASH_ALG_SHA512,
3949                        .mode = HASH_MODE_HMAC,
3950                        },
3951          },
3952         {
3953          .type = CRYPTO_ALG_TYPE_AHASH,
3954          .alg.hash = {
3955                       .halg.digestsize = SHA3_224_DIGEST_SIZE,
3956                       .halg.base = {
3957                                     .cra_name = "sha3-224",
3958                                     .cra_driver_name = "sha3-224-iproc",
3959                                     .cra_blocksize = SHA3_224_BLOCK_SIZE,
3960                                 }
3961                       },
3962          .cipher_info = {
3963                          .alg = CIPHER_ALG_NONE,
3964                          .mode = CIPHER_MODE_NONE,
3965                          },
3966          .auth_info = {
3967                        .alg = HASH_ALG_SHA3_224,
3968                        .mode = HASH_MODE_HASH,
3969                        },
3970          },
3971         {
3972          .type = CRYPTO_ALG_TYPE_AHASH,
3973          .alg.hash = {
3974                       .halg.digestsize = SHA3_224_DIGEST_SIZE,
3975                       .halg.base = {
3976                                     .cra_name = "hmac(sha3-224)",
3977                                     .cra_driver_name = "hmac-sha3-224-iproc",
3978                                     .cra_blocksize = SHA3_224_BLOCK_SIZE,
3979                                 }
3980                       },
3981          .cipher_info = {
3982                          .alg = CIPHER_ALG_NONE,
3983                          .mode = CIPHER_MODE_NONE,
3984                          },
3985          .auth_info = {
3986                        .alg = HASH_ALG_SHA3_224,
3987                        .mode = HASH_MODE_HMAC
3988                        },
3989          },
3990         {
3991          .type = CRYPTO_ALG_TYPE_AHASH,
3992          .alg.hash = {
3993                       .halg.digestsize = SHA3_256_DIGEST_SIZE,
3994                       .halg.base = {
3995                                     .cra_name = "sha3-256",
3996                                     .cra_driver_name = "sha3-256-iproc",
3997                                     .cra_blocksize = SHA3_256_BLOCK_SIZE,
3998                                 }
3999                       },
4000          .cipher_info = {
4001                          .alg = CIPHER_ALG_NONE,
4002                          .mode = CIPHER_MODE_NONE,
4003                          },
4004          .auth_info = {
4005                        .alg = HASH_ALG_SHA3_256,
4006                        .mode = HASH_MODE_HASH,
4007                        },
4008          },
4009         {
4010          .type = CRYPTO_ALG_TYPE_AHASH,
4011          .alg.hash = {
4012                       .halg.digestsize = SHA3_256_DIGEST_SIZE,
4013                       .halg.base = {
4014                                     .cra_name = "hmac(sha3-256)",
4015                                     .cra_driver_name = "hmac-sha3-256-iproc",
4016                                     .cra_blocksize = SHA3_256_BLOCK_SIZE,
4017                                 }
4018                       },
4019          .cipher_info = {
4020                          .alg = CIPHER_ALG_NONE,
4021                          .mode = CIPHER_MODE_NONE,
4022                          },
4023          .auth_info = {
4024                        .alg = HASH_ALG_SHA3_256,
4025                        .mode = HASH_MODE_HMAC,
4026                        },
4027          },
4028         {
4029          .type = CRYPTO_ALG_TYPE_AHASH,
4030          .alg.hash = {
4031                       .halg.digestsize = SHA3_384_DIGEST_SIZE,
4032                       .halg.base = {
4033                                     .cra_name = "sha3-384",
4034                                     .cra_driver_name = "sha3-384-iproc",
4035                                     .cra_blocksize = SHA3_224_BLOCK_SIZE,
4036                                 }
4037                       },
4038          .cipher_info = {
4039                          .alg = CIPHER_ALG_NONE,
4040                          .mode = CIPHER_MODE_NONE,
4041                          },
4042          .auth_info = {
4043                        .alg = HASH_ALG_SHA3_384,
4044                        .mode = HASH_MODE_HASH,
4045                        },
4046          },
4047         {
4048          .type = CRYPTO_ALG_TYPE_AHASH,
4049          .alg.hash = {
4050                       .halg.digestsize = SHA3_384_DIGEST_SIZE,
4051                       .halg.base = {
4052                                     .cra_name = "hmac(sha3-384)",
4053                                     .cra_driver_name = "hmac-sha3-384-iproc",
4054                                     .cra_blocksize = SHA3_384_BLOCK_SIZE,
4055                                 }
4056                       },
4057          .cipher_info = {
4058                          .alg = CIPHER_ALG_NONE,
4059                          .mode = CIPHER_MODE_NONE,
4060                          },
4061          .auth_info = {
4062                        .alg = HASH_ALG_SHA3_384,
4063                        .mode = HASH_MODE_HMAC,
4064                        },
4065          },
4066         {
4067          .type = CRYPTO_ALG_TYPE_AHASH,
4068          .alg.hash = {
4069                       .halg.digestsize = SHA3_512_DIGEST_SIZE,
4070                       .halg.base = {
4071                                     .cra_name = "sha3-512",
4072                                     .cra_driver_name = "sha3-512-iproc",
4073                                     .cra_blocksize = SHA3_512_BLOCK_SIZE,
4074                                 }
4075                       },
4076          .cipher_info = {
4077                          .alg = CIPHER_ALG_NONE,
4078                          .mode = CIPHER_MODE_NONE,
4079                          },
4080          .auth_info = {
4081                        .alg = HASH_ALG_SHA3_512,
4082                        .mode = HASH_MODE_HASH,
4083                        },
4084          },
4085         {
4086          .type = CRYPTO_ALG_TYPE_AHASH,
4087          .alg.hash = {
4088                       .halg.digestsize = SHA3_512_DIGEST_SIZE,
4089                       .halg.base = {
4090                                     .cra_name = "hmac(sha3-512)",
4091                                     .cra_driver_name = "hmac-sha3-512-iproc",
4092                                     .cra_blocksize = SHA3_512_BLOCK_SIZE,
4093                                 }
4094                       },
4095          .cipher_info = {
4096                          .alg = CIPHER_ALG_NONE,
4097                          .mode = CIPHER_MODE_NONE,
4098                          },
4099          .auth_info = {
4100                        .alg = HASH_ALG_SHA3_512,
4101                        .mode = HASH_MODE_HMAC,
4102                        },
4103          },
4104         {
4105          .type = CRYPTO_ALG_TYPE_AHASH,
4106          .alg.hash = {
4107                       .halg.digestsize = AES_BLOCK_SIZE,
4108                       .halg.base = {
4109                                     .cra_name = "xcbc(aes)",
4110                                     .cra_driver_name = "xcbc-aes-iproc",
4111                                     .cra_blocksize = AES_BLOCK_SIZE,
4112                                 }
4113                       },
4114          .cipher_info = {
4115                          .alg = CIPHER_ALG_NONE,
4116                          .mode = CIPHER_MODE_NONE,
4117                          },
4118          .auth_info = {
4119                        .alg = HASH_ALG_AES,
4120                        .mode = HASH_MODE_XCBC,
4121                        },
4122          },
4123         {
4124          .type = CRYPTO_ALG_TYPE_AHASH,
4125          .alg.hash = {
4126                       .halg.digestsize = AES_BLOCK_SIZE,
4127                       .halg.base = {
4128                                     .cra_name = "cmac(aes)",
4129                                     .cra_driver_name = "cmac-aes-iproc",
4130                                     .cra_blocksize = AES_BLOCK_SIZE,
4131                                 }
4132                       },
4133          .cipher_info = {
4134                          .alg = CIPHER_ALG_NONE,
4135                          .mode = CIPHER_MODE_NONE,
4136                          },
4137          .auth_info = {
4138                        .alg = HASH_ALG_AES,
4139                        .mode = HASH_MODE_CMAC,
4140                        },
4141          },
4142 };
4143
4144 static int generic_cra_init(struct crypto_tfm *tfm,
4145                             struct iproc_alg_s *cipher_alg)
4146 {
4147         struct spu_hw *spu = &iproc_priv.spu;
4148         struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
4149         unsigned int blocksize = crypto_tfm_alg_blocksize(tfm);
4150
4151         flow_log("%s()\n", __func__);
4152
4153         ctx->alg = cipher_alg;
4154         ctx->cipher = cipher_alg->cipher_info;
4155         ctx->auth = cipher_alg->auth_info;
4156         ctx->auth_first = cipher_alg->auth_first;
4157         ctx->max_payload = spu->spu_ctx_max_payload(ctx->cipher.alg,
4158                                                     ctx->cipher.mode,
4159                                                     blocksize);
4160         ctx->fallback_cipher = NULL;
4161
4162         ctx->enckeylen = 0;
4163         ctx->authkeylen = 0;
4164
4165         atomic_inc(&iproc_priv.stream_count);
4166         atomic_inc(&iproc_priv.session_count);
4167
4168         return 0;
4169 }
4170
4171 static int skcipher_init_tfm(struct crypto_skcipher *skcipher)
4172 {
4173         struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
4174         struct skcipher_alg *alg = crypto_skcipher_alg(skcipher);
4175         struct iproc_alg_s *cipher_alg;
4176
4177         flow_log("%s()\n", __func__);
4178
4179         crypto_skcipher_set_reqsize(skcipher, sizeof(struct iproc_reqctx_s));
4180
4181         cipher_alg = container_of(alg, struct iproc_alg_s, alg.skcipher);
4182         return generic_cra_init(tfm, cipher_alg);
4183 }
4184
4185 static int ahash_cra_init(struct crypto_tfm *tfm)
4186 {
4187         int err;
4188         struct crypto_alg *alg = tfm->__crt_alg;
4189         struct iproc_alg_s *cipher_alg;
4190
4191         cipher_alg = container_of(__crypto_ahash_alg(alg), struct iproc_alg_s,
4192                                   alg.hash);
4193
4194         err = generic_cra_init(tfm, cipher_alg);
4195         flow_log("%s()\n", __func__);
4196
4197         /*
4198          * export state size has to be < 512 bytes. So don't include msg bufs
4199          * in state size.
4200          */
4201         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
4202                                  sizeof(struct iproc_reqctx_s));
4203
4204         return err;
4205 }
4206
4207 static int aead_cra_init(struct crypto_aead *aead)
4208 {
4209         unsigned int reqsize = sizeof(struct iproc_reqctx_s);
4210         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
4211         struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
4212         struct crypto_alg *alg = tfm->__crt_alg;
4213         struct aead_alg *aalg = container_of(alg, struct aead_alg, base);
4214         struct iproc_alg_s *cipher_alg = container_of(aalg, struct iproc_alg_s,
4215                                                       alg.aead);
4216
4217         int err = generic_cra_init(tfm, cipher_alg);
4218
4219         flow_log("%s()\n", __func__);
4220
4221         ctx->is_esp = false;
4222         ctx->salt_len = 0;
4223         ctx->salt_offset = 0;
4224
4225         /* random first IV */
4226         get_random_bytes(ctx->iv, MAX_IV_SIZE);
4227         flow_dump("  iv: ", ctx->iv, MAX_IV_SIZE);
4228
4229         if (err)
4230                 goto out;
4231
4232         if (!(alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK))
4233                 goto reqsize;
4234
4235         flow_log("%s() creating fallback cipher\n", __func__);
4236
4237         ctx->fallback_cipher = crypto_alloc_aead(alg->cra_name, 0,
4238                                                  CRYPTO_ALG_ASYNC |
4239                                                  CRYPTO_ALG_NEED_FALLBACK);
4240         if (IS_ERR(ctx->fallback_cipher)) {
4241                 pr_err("%s() Error: failed to allocate fallback for %s\n",
4242                        __func__, alg->cra_name);
4243                 return PTR_ERR(ctx->fallback_cipher);
4244         }
4245
4246         reqsize += crypto_aead_reqsize(ctx->fallback_cipher);
4247
4248 reqsize:
4249         crypto_aead_set_reqsize(aead, reqsize);
4250
4251 out:
4252         return err;
4253 }
4254
4255 static void generic_cra_exit(struct crypto_tfm *tfm)
4256 {
4257         atomic_dec(&iproc_priv.session_count);
4258 }
4259
4260 static void skcipher_exit_tfm(struct crypto_skcipher *tfm)
4261 {
4262         generic_cra_exit(crypto_skcipher_tfm(tfm));
4263 }
4264
4265 static void aead_cra_exit(struct crypto_aead *aead)
4266 {
4267         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
4268         struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
4269
4270         generic_cra_exit(tfm);
4271
4272         if (ctx->fallback_cipher) {
4273                 crypto_free_aead(ctx->fallback_cipher);
4274                 ctx->fallback_cipher = NULL;
4275         }
4276 }
4277
4278 /**
4279  * spu_functions_register() - Specify hardware-specific SPU functions based on
4280  * SPU type read from device tree.
4281  * @dev:        device structure
4282  * @spu_type:   SPU hardware generation
4283  * @spu_subtype: SPU hardware version
4284  */
4285 static void spu_functions_register(struct device *dev,
4286                                    enum spu_spu_type spu_type,
4287                                    enum spu_spu_subtype spu_subtype)
4288 {
4289         struct spu_hw *spu = &iproc_priv.spu;
4290
4291         if (spu_type == SPU_TYPE_SPUM) {
4292                 dev_dbg(dev, "Registering SPUM functions");
4293                 spu->spu_dump_msg_hdr = spum_dump_msg_hdr;
4294                 spu->spu_payload_length = spum_payload_length;
4295                 spu->spu_response_hdr_len = spum_response_hdr_len;
4296                 spu->spu_hash_pad_len = spum_hash_pad_len;
4297                 spu->spu_gcm_ccm_pad_len = spum_gcm_ccm_pad_len;
4298                 spu->spu_assoc_resp_len = spum_assoc_resp_len;
4299                 spu->spu_aead_ivlen = spum_aead_ivlen;
4300                 spu->spu_hash_type = spum_hash_type;
4301                 spu->spu_digest_size = spum_digest_size;
4302                 spu->spu_create_request = spum_create_request;
4303                 spu->spu_cipher_req_init = spum_cipher_req_init;
4304                 spu->spu_cipher_req_finish = spum_cipher_req_finish;
4305                 spu->spu_request_pad = spum_request_pad;
4306                 spu->spu_tx_status_len = spum_tx_status_len;
4307                 spu->spu_rx_status_len = spum_rx_status_len;
4308                 spu->spu_status_process = spum_status_process;
4309                 spu->spu_xts_tweak_in_payload = spum_xts_tweak_in_payload;
4310                 spu->spu_ccm_update_iv = spum_ccm_update_iv;
4311                 spu->spu_wordalign_padlen = spum_wordalign_padlen;
4312                 if (spu_subtype == SPU_SUBTYPE_SPUM_NS2)
4313                         spu->spu_ctx_max_payload = spum_ns2_ctx_max_payload;
4314                 else
4315                         spu->spu_ctx_max_payload = spum_nsp_ctx_max_payload;
4316         } else {
4317                 dev_dbg(dev, "Registering SPU2 functions");
4318                 spu->spu_dump_msg_hdr = spu2_dump_msg_hdr;
4319                 spu->spu_ctx_max_payload = spu2_ctx_max_payload;
4320                 spu->spu_payload_length = spu2_payload_length;
4321                 spu->spu_response_hdr_len = spu2_response_hdr_len;
4322                 spu->spu_hash_pad_len = spu2_hash_pad_len;
4323                 spu->spu_gcm_ccm_pad_len = spu2_gcm_ccm_pad_len;
4324                 spu->spu_assoc_resp_len = spu2_assoc_resp_len;
4325                 spu->spu_aead_ivlen = spu2_aead_ivlen;
4326                 spu->spu_hash_type = spu2_hash_type;
4327                 spu->spu_digest_size = spu2_digest_size;
4328                 spu->spu_create_request = spu2_create_request;
4329                 spu->spu_cipher_req_init = spu2_cipher_req_init;
4330                 spu->spu_cipher_req_finish = spu2_cipher_req_finish;
4331                 spu->spu_request_pad = spu2_request_pad;
4332                 spu->spu_tx_status_len = spu2_tx_status_len;
4333                 spu->spu_rx_status_len = spu2_rx_status_len;
4334                 spu->spu_status_process = spu2_status_process;
4335                 spu->spu_xts_tweak_in_payload = spu2_xts_tweak_in_payload;
4336                 spu->spu_ccm_update_iv = spu2_ccm_update_iv;
4337                 spu->spu_wordalign_padlen = spu2_wordalign_padlen;
4338         }
4339 }
4340
4341 /**
4342  * spu_mb_init() - Initialize mailbox client. Request ownership of a mailbox
4343  * channel for the SPU being probed.
4344  * @dev:  SPU driver device structure
4345  *
4346  * Return: 0 if successful
4347  *         < 0 otherwise
4348  */
4349 static int spu_mb_init(struct device *dev)
4350 {
4351         struct mbox_client *mcl = &iproc_priv.mcl;
4352         int err, i;
4353
4354         iproc_priv.mbox = devm_kcalloc(dev, iproc_priv.spu.num_chan,
4355                                   sizeof(struct mbox_chan *), GFP_KERNEL);
4356         if (!iproc_priv.mbox)
4357                 return -ENOMEM;
4358
4359         mcl->dev = dev;
4360         mcl->tx_block = false;
4361         mcl->tx_tout = 0;
4362         mcl->knows_txdone = true;
4363         mcl->rx_callback = spu_rx_callback;
4364         mcl->tx_done = NULL;
4365
4366         for (i = 0; i < iproc_priv.spu.num_chan; i++) {
4367                 iproc_priv.mbox[i] = mbox_request_channel(mcl, i);
4368                 if (IS_ERR(iproc_priv.mbox[i])) {
4369                         err = PTR_ERR(iproc_priv.mbox[i]);
4370                         dev_err(dev,
4371                                 "Mbox channel %d request failed with err %d",
4372                                 i, err);
4373                         iproc_priv.mbox[i] = NULL;
4374                         goto free_channels;
4375                 }
4376         }
4377
4378         return 0;
4379 free_channels:
4380         for (i = 0; i < iproc_priv.spu.num_chan; i++) {
4381                 if (iproc_priv.mbox[i])
4382                         mbox_free_channel(iproc_priv.mbox[i]);
4383         }
4384
4385         return err;
4386 }
4387
4388 static void spu_mb_release(struct platform_device *pdev)
4389 {
4390         int i;
4391
4392         for (i = 0; i < iproc_priv.spu.num_chan; i++)
4393                 mbox_free_channel(iproc_priv.mbox[i]);
4394 }
4395
4396 static void spu_counters_init(void)
4397 {
4398         int i;
4399         int j;
4400
4401         atomic_set(&iproc_priv.session_count, 0);
4402         atomic_set(&iproc_priv.stream_count, 0);
4403         atomic_set(&iproc_priv.next_chan, (int)iproc_priv.spu.num_chan);
4404         atomic64_set(&iproc_priv.bytes_in, 0);
4405         atomic64_set(&iproc_priv.bytes_out, 0);
4406         for (i = 0; i < SPU_OP_NUM; i++) {
4407                 atomic_set(&iproc_priv.op_counts[i], 0);
4408                 atomic_set(&iproc_priv.setkey_cnt[i], 0);
4409         }
4410         for (i = 0; i < CIPHER_ALG_LAST; i++)
4411                 for (j = 0; j < CIPHER_MODE_LAST; j++)
4412                         atomic_set(&iproc_priv.cipher_cnt[i][j], 0);
4413
4414         for (i = 0; i < HASH_ALG_LAST; i++) {
4415                 atomic_set(&iproc_priv.hash_cnt[i], 0);
4416                 atomic_set(&iproc_priv.hmac_cnt[i], 0);
4417         }
4418         for (i = 0; i < AEAD_TYPE_LAST; i++)
4419                 atomic_set(&iproc_priv.aead_cnt[i], 0);
4420
4421         atomic_set(&iproc_priv.mb_no_spc, 0);
4422         atomic_set(&iproc_priv.mb_send_fail, 0);
4423         atomic_set(&iproc_priv.bad_icv, 0);
4424 }
4425
4426 static int spu_register_skcipher(struct iproc_alg_s *driver_alg)
4427 {
4428         struct skcipher_alg *crypto = &driver_alg->alg.skcipher;
4429         int err;
4430
4431         crypto->base.cra_module = THIS_MODULE;
4432         crypto->base.cra_priority = cipher_pri;
4433         crypto->base.cra_alignmask = 0;
4434         crypto->base.cra_ctxsize = sizeof(struct iproc_ctx_s);
4435         crypto->base.cra_flags = CRYPTO_ALG_ASYNC |
4436                                  CRYPTO_ALG_ALLOCATES_MEMORY |
4437                                  CRYPTO_ALG_KERN_DRIVER_ONLY;
4438
4439         crypto->init = skcipher_init_tfm;
4440         crypto->exit = skcipher_exit_tfm;
4441         crypto->setkey = skcipher_setkey;
4442         crypto->encrypt = skcipher_encrypt;
4443         crypto->decrypt = skcipher_decrypt;
4444
4445         err = crypto_register_skcipher(crypto);
4446         /* Mark alg as having been registered, if successful */
4447         if (err == 0)
4448                 driver_alg->registered = true;
4449         pr_debug("  registered skcipher %s\n", crypto->base.cra_driver_name);
4450         return err;
4451 }
4452
4453 static int spu_register_ahash(struct iproc_alg_s *driver_alg)
4454 {
4455         struct spu_hw *spu = &iproc_priv.spu;
4456         struct ahash_alg *hash = &driver_alg->alg.hash;
4457         int err;
4458
4459         /* AES-XCBC is the only AES hash type currently supported on SPU-M */
4460         if ((driver_alg->auth_info.alg == HASH_ALG_AES) &&
4461             (driver_alg->auth_info.mode != HASH_MODE_XCBC) &&
4462             (spu->spu_type == SPU_TYPE_SPUM))
4463                 return 0;
4464
4465         /* SHA3 algorithm variants are not registered for SPU-M or SPU2. */
4466         if ((driver_alg->auth_info.alg >= HASH_ALG_SHA3_224) &&
4467             (spu->spu_subtype != SPU_SUBTYPE_SPU2_V2))
4468                 return 0;
4469
4470         hash->halg.base.cra_module = THIS_MODULE;
4471         hash->halg.base.cra_priority = hash_pri;
4472         hash->halg.base.cra_alignmask = 0;
4473         hash->halg.base.cra_ctxsize = sizeof(struct iproc_ctx_s);
4474         hash->halg.base.cra_init = ahash_cra_init;
4475         hash->halg.base.cra_exit = generic_cra_exit;
4476         hash->halg.base.cra_flags = CRYPTO_ALG_ASYNC |
4477                                     CRYPTO_ALG_ALLOCATES_MEMORY;
4478         hash->halg.statesize = sizeof(struct spu_hash_export_s);
4479
4480         if (driver_alg->auth_info.mode != HASH_MODE_HMAC) {
4481                 hash->init = ahash_init;
4482                 hash->update = ahash_update;
4483                 hash->final = ahash_final;
4484                 hash->finup = ahash_finup;
4485                 hash->digest = ahash_digest;
4486                 if ((driver_alg->auth_info.alg == HASH_ALG_AES) &&
4487                     ((driver_alg->auth_info.mode == HASH_MODE_XCBC) ||
4488                     (driver_alg->auth_info.mode == HASH_MODE_CMAC))) {
4489                         hash->setkey = ahash_setkey;
4490                 }
4491         } else {
4492                 hash->setkey = ahash_hmac_setkey;
4493                 hash->init = ahash_hmac_init;
4494                 hash->update = ahash_hmac_update;
4495                 hash->final = ahash_hmac_final;
4496                 hash->finup = ahash_hmac_finup;
4497                 hash->digest = ahash_hmac_digest;
4498         }
4499         hash->export = ahash_export;
4500         hash->import = ahash_import;
4501
4502         err = crypto_register_ahash(hash);
4503         /* Mark alg as having been registered, if successful */
4504         if (err == 0)
4505                 driver_alg->registered = true;
4506         pr_debug("  registered ahash %s\n",
4507                  hash->halg.base.cra_driver_name);
4508         return err;
4509 }
4510
4511 static int spu_register_aead(struct iproc_alg_s *driver_alg)
4512 {
4513         struct aead_alg *aead = &driver_alg->alg.aead;
4514         int err;
4515
4516         aead->base.cra_module = THIS_MODULE;
4517         aead->base.cra_priority = aead_pri;
4518         aead->base.cra_alignmask = 0;
4519         aead->base.cra_ctxsize = sizeof(struct iproc_ctx_s);
4520
4521         aead->base.cra_flags |= CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY;
4522         /* setkey set in alg initialization */
4523         aead->setauthsize = aead_setauthsize;
4524         aead->encrypt = aead_encrypt;
4525         aead->decrypt = aead_decrypt;
4526         aead->init = aead_cra_init;
4527         aead->exit = aead_cra_exit;
4528
4529         err = crypto_register_aead(aead);
4530         /* Mark alg as having been registered, if successful */
4531         if (err == 0)
4532                 driver_alg->registered = true;
4533         pr_debug("  registered aead %s\n", aead->base.cra_driver_name);
4534         return err;
4535 }
4536
4537 /* register crypto algorithms the device supports */
4538 static int spu_algs_register(struct device *dev)
4539 {
4540         int i, j;
4541         int err;
4542
4543         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4544                 switch (driver_algs[i].type) {
4545                 case CRYPTO_ALG_TYPE_SKCIPHER:
4546                         err = spu_register_skcipher(&driver_algs[i]);
4547                         break;
4548                 case CRYPTO_ALG_TYPE_AHASH:
4549                         err = spu_register_ahash(&driver_algs[i]);
4550                         break;
4551                 case CRYPTO_ALG_TYPE_AEAD:
4552                         err = spu_register_aead(&driver_algs[i]);
4553                         break;
4554                 default:
4555                         dev_err(dev,
4556                                 "iproc-crypto: unknown alg type: %d",
4557                                 driver_algs[i].type);
4558                         err = -EINVAL;
4559                 }
4560
4561                 if (err) {
4562                         dev_err(dev, "alg registration failed with error %d\n",
4563                                 err);
4564                         goto err_algs;
4565                 }
4566         }
4567
4568         return 0;
4569
4570 err_algs:
4571         for (j = 0; j < i; j++) {
4572                 /* Skip any algorithm not registered */
4573                 if (!driver_algs[j].registered)
4574                         continue;
4575                 switch (driver_algs[j].type) {
4576                 case CRYPTO_ALG_TYPE_SKCIPHER:
4577                         crypto_unregister_skcipher(&driver_algs[j].alg.skcipher);
4578                         driver_algs[j].registered = false;
4579                         break;
4580                 case CRYPTO_ALG_TYPE_AHASH:
4581                         crypto_unregister_ahash(&driver_algs[j].alg.hash);
4582                         driver_algs[j].registered = false;
4583                         break;
4584                 case CRYPTO_ALG_TYPE_AEAD:
4585                         crypto_unregister_aead(&driver_algs[j].alg.aead);
4586                         driver_algs[j].registered = false;
4587                         break;
4588                 }
4589         }
4590         return err;
4591 }
4592
4593 /* ==================== Kernel Platform API ==================== */
4594
4595 static struct spu_type_subtype spum_ns2_types = {
4596         SPU_TYPE_SPUM, SPU_SUBTYPE_SPUM_NS2
4597 };
4598
4599 static struct spu_type_subtype spum_nsp_types = {
4600         SPU_TYPE_SPUM, SPU_SUBTYPE_SPUM_NSP
4601 };
4602
4603 static struct spu_type_subtype spu2_types = {
4604         SPU_TYPE_SPU2, SPU_SUBTYPE_SPU2_V1
4605 };
4606
4607 static struct spu_type_subtype spu2_v2_types = {
4608         SPU_TYPE_SPU2, SPU_SUBTYPE_SPU2_V2
4609 };
4610
4611 static const struct of_device_id bcm_spu_dt_ids[] = {
4612         {
4613                 .compatible = "brcm,spum-crypto",
4614                 .data = &spum_ns2_types,
4615         },
4616         {
4617                 .compatible = "brcm,spum-nsp-crypto",
4618                 .data = &spum_nsp_types,
4619         },
4620         {
4621                 .compatible = "brcm,spu2-crypto",
4622                 .data = &spu2_types,
4623         },
4624         {
4625                 .compatible = "brcm,spu2-v2-crypto",
4626                 .data = &spu2_v2_types,
4627         },
4628         { /* sentinel */ }
4629 };
4630
4631 MODULE_DEVICE_TABLE(of, bcm_spu_dt_ids);
4632
4633 static int spu_dt_read(struct platform_device *pdev)
4634 {
4635         struct device *dev = &pdev->dev;
4636         struct spu_hw *spu = &iproc_priv.spu;
4637         struct resource *spu_ctrl_regs;
4638         const struct spu_type_subtype *matched_spu_type;
4639         struct device_node *dn = pdev->dev.of_node;
4640         int err, i;
4641
4642         /* Count number of mailbox channels */
4643         spu->num_chan = of_count_phandle_with_args(dn, "mboxes", "#mbox-cells");
4644
4645         matched_spu_type = of_device_get_match_data(dev);
4646         if (!matched_spu_type) {
4647                 dev_err(dev, "Failed to match device\n");
4648                 return -ENODEV;
4649         }
4650
4651         spu->spu_type = matched_spu_type->type;
4652         spu->spu_subtype = matched_spu_type->subtype;
4653
4654         for (i = 0; (i < MAX_SPUS) && ((spu_ctrl_regs =
4655                 platform_get_resource(pdev, IORESOURCE_MEM, i)) != NULL); i++) {
4656
4657                 spu->reg_vbase[i] = devm_ioremap_resource(dev, spu_ctrl_regs);
4658                 if (IS_ERR(spu->reg_vbase[i])) {
4659                         err = PTR_ERR(spu->reg_vbase[i]);
4660                         dev_err(dev, "Failed to map registers: %d\n",
4661                                 err);
4662                         spu->reg_vbase[i] = NULL;
4663                         return err;
4664                 }
4665         }
4666         spu->num_spu = i;
4667         dev_dbg(dev, "Device has %d SPUs", spu->num_spu);
4668
4669         return 0;
4670 }
4671
4672 static int bcm_spu_probe(struct platform_device *pdev)
4673 {
4674         struct device *dev = &pdev->dev;
4675         struct spu_hw *spu = &iproc_priv.spu;
4676         int err;
4677
4678         iproc_priv.pdev  = pdev;
4679         platform_set_drvdata(iproc_priv.pdev,
4680                              &iproc_priv);
4681
4682         err = spu_dt_read(pdev);
4683         if (err < 0)
4684                 goto failure;
4685
4686         err = spu_mb_init(dev);
4687         if (err < 0)
4688                 goto failure;
4689
4690         if (spu->spu_type == SPU_TYPE_SPUM)
4691                 iproc_priv.bcm_hdr_len = 8;
4692         else if (spu->spu_type == SPU_TYPE_SPU2)
4693                 iproc_priv.bcm_hdr_len = 0;
4694
4695         spu_functions_register(dev, spu->spu_type, spu->spu_subtype);
4696
4697         spu_counters_init();
4698
4699         spu_setup_debugfs();
4700
4701         err = spu_algs_register(dev);
4702         if (err < 0)
4703                 goto fail_reg;
4704
4705         return 0;
4706
4707 fail_reg:
4708         spu_free_debugfs();
4709 failure:
4710         spu_mb_release(pdev);
4711         dev_err(dev, "%s failed with error %d.\n", __func__, err);
4712
4713         return err;
4714 }
4715
4716 static int bcm_spu_remove(struct platform_device *pdev)
4717 {
4718         int i;
4719         struct device *dev = &pdev->dev;
4720         char *cdn;
4721
4722         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4723                 /*
4724                  * Not all algorithms were registered, depending on whether
4725                  * hardware is SPU or SPU2.  So here we make sure to skip
4726                  * those algorithms that were not previously registered.
4727                  */
4728                 if (!driver_algs[i].registered)
4729                         continue;
4730
4731                 switch (driver_algs[i].type) {
4732                 case CRYPTO_ALG_TYPE_SKCIPHER:
4733                         crypto_unregister_skcipher(&driver_algs[i].alg.skcipher);
4734                         dev_dbg(dev, "  unregistered cipher %s\n",
4735                                 driver_algs[i].alg.skcipher.base.cra_driver_name);
4736                         driver_algs[i].registered = false;
4737                         break;
4738                 case CRYPTO_ALG_TYPE_AHASH:
4739                         crypto_unregister_ahash(&driver_algs[i].alg.hash);
4740                         cdn = driver_algs[i].alg.hash.halg.base.cra_driver_name;
4741                         dev_dbg(dev, "  unregistered hash %s\n", cdn);
4742                         driver_algs[i].registered = false;
4743                         break;
4744                 case CRYPTO_ALG_TYPE_AEAD:
4745                         crypto_unregister_aead(&driver_algs[i].alg.aead);
4746                         dev_dbg(dev, "  unregistered aead %s\n",
4747                                 driver_algs[i].alg.aead.base.cra_driver_name);
4748                         driver_algs[i].registered = false;
4749                         break;
4750                 }
4751         }
4752         spu_free_debugfs();
4753         spu_mb_release(pdev);
4754         return 0;
4755 }
4756
4757 /* ===== Kernel Module API ===== */
4758
4759 static struct platform_driver bcm_spu_pdriver = {
4760         .driver = {
4761                    .name = "brcm-spu-crypto",
4762                    .of_match_table = of_match_ptr(bcm_spu_dt_ids),
4763                    },
4764         .probe = bcm_spu_probe,
4765         .remove = bcm_spu_remove,
4766 };
4767 module_platform_driver(bcm_spu_pdriver);
4768
4769 MODULE_AUTHOR("Rob Rice <[email protected]>");
4770 MODULE_DESCRIPTION("Broadcom symmetric crypto offload driver");
4771 MODULE_LICENSE("GPL v2");