drivers/crypto/ccp/ccp-ops.c

   1 /*
   2  * AMD Cryptographic Coprocessor (CCP) driver
   3  *
   4  * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
   5  *
   6  * Author: Tom Lendacky <[email protected]>
   7  * Author: Gary R Hook <[email protected]>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  */
  13
  14 #include <linux/module.h>
  15 #include <linux/kernel.h>
  16 #include <linux/pci.h>
  17 #include <linux/interrupt.h>
  18 #include <crypto/scatterwalk.h>
  19 #include <crypto/des.h>
  20 #include <linux/ccp.h>
  21
  22 #include "ccp-dev.h"
  23
  24 /* SHA initial context values */
  25 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
  26         cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
  27         cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
  28         cpu_to_be32(SHA1_H4),
  29 };
  30
  31 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  32         cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
  33         cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
  34         cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
  35         cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
  36 };
  37
  38 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  39         cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
  40         cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
  41         cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
  42         cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
  43 };
  44
  45 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  46         cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
  47         cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
  48         cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
  49         cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
  50 };
  51
  52 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  53         cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
  54         cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
  55         cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
  56         cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
  57 };
  58
  59 #define CCP_NEW_JOBID(ccp)      ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
  60                                         ccp_gen_jobid(ccp) : 0)
  61
  62 static u32 ccp_gen_jobid(struct ccp_device *ccp)
  63 {
  64         return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
  65 }
  66
  67 static void ccp_sg_free(struct ccp_sg_workarea *wa)
  68 {
  69         if (wa->dma_count)
  70                 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
  71
  72         wa->dma_count = 0;
  73 }
  74
  75 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
  76                                 struct scatterlist *sg, u64 len,
  77                                 enum dma_data_direction dma_dir)
  78 {
  79         memset(wa, 0, sizeof(*wa));
  80
  81         wa->sg = sg;
  82         if (!sg)
  83                 return 0;
  84
  85         wa->nents = sg_nents_for_len(sg, len);
  86         if (wa->nents < 0)
  87                 return wa->nents;
  88
  89         wa->bytes_left = len;
  90         wa->sg_used = 0;
  91
  92         if (len == 0)
  93                 return 0;
  94
  95         if (dma_dir == DMA_NONE)
  96                 return 0;
  97
  98         wa->dma_sg = sg;
  99         wa->dma_dev = dev;
 100         wa->dma_dir = dma_dir;
 101         wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
 102         if (!wa->dma_count)
 103                 return -ENOMEM;
 104
 105         return 0;
 106 }
 107
 108 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
 109 {
 110         unsigned int nbytes = min_t(u64, len, wa->bytes_left);
 111
 112         if (!wa->sg)
 113                 return;
 114
 115         wa->sg_used += nbytes;
 116         wa->bytes_left -= nbytes;
 117         if (wa->sg_used == wa->sg->length) {
 118                 wa->sg = sg_next(wa->sg);
 119                 wa->sg_used = 0;
 120         }
 121 }
 122
 123 static void ccp_dm_free(struct ccp_dm_workarea *wa)
 124 {
 125         if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
 126                 if (wa->address)
 127                         dma_pool_free(wa->dma_pool, wa->address,
 128                                       wa->dma.address);
 129         } else {
 130                 if (wa->dma.address)
 131                         dma_unmap_single(wa->dev, wa->dma.address, wa->length,
 132                                          wa->dma.dir);
 133                 kfree(wa->address);
 134         }
 135
 136         wa->address = NULL;
 137         wa->dma.address = 0;
 138 }
 139
 140 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
 141                                 struct ccp_cmd_queue *cmd_q,
 142                                 unsigned int len,
 143                                 enum dma_data_direction dir)
 144 {
 145         memset(wa, 0, sizeof(*wa));
 146
 147         if (!len)
 148                 return 0;
 149
 150         wa->dev = cmd_q->ccp->dev;
 151         wa->length = len;
 152
 153         if (len <= CCP_DMAPOOL_MAX_SIZE) {
 154                 wa->dma_pool = cmd_q->dma_pool;
 155
 156                 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
 157                                              &wa->dma.address);
 158                 if (!wa->address)
 159                         return -ENOMEM;
 160
 161                 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
 162
 163                 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
 164         } else {
 165                 wa->address = kzalloc(len, GFP_KERNEL);
 166                 if (!wa->address)
 167                         return -ENOMEM;
 168
 169                 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
 170                                                  dir);
 171                 if (dma_mapping_error(wa->dev, wa->dma.address))
 172                         return -ENOMEM;
 173
 174                 wa->dma.length = len;
 175         }
 176         wa->dma.dir = dir;
 177
 178         return 0;
 179 }
 180
 181 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 182                             struct scatterlist *sg, unsigned int sg_offset,
 183                             unsigned int len)
 184 {
 185         WARN_ON(!wa->address);
 186
 187         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 188                                  0);
 189 }
 190
 191 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 192                             struct scatterlist *sg, unsigned int sg_offset,
 193                             unsigned int len)
 194 {
 195         WARN_ON(!wa->address);
 196
 197         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 198                                  1);
 199 }
 200
 201 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 202                                    unsigned int wa_offset,
 203                                    struct scatterlist *sg,
 204                                    unsigned int sg_offset,
 205                                    unsigned int len)
 206 {
 207         u8 *p, *q;
 208
 209         ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
 210
 211         p = wa->address + wa_offset;
 212         q = p + len - 1;
 213         while (p < q) {
 214                 *p = *p ^ *q;
 215                 *q = *p ^ *q;
 216                 *p = *p ^ *q;
 217                 p++;
 218                 q--;
 219         }
 220         return 0;
 221 }
 222
 223 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
 224                                     unsigned int wa_offset,
 225                                     struct scatterlist *sg,
 226                                     unsigned int sg_offset,
 227                                     unsigned int len)
 228 {
 229         u8 *p, *q;
 230
 231         p = wa->address + wa_offset;
 232         q = p + len - 1;
 233         while (p < q) {
 234                 *p = *p ^ *q;
 235                 *q = *p ^ *q;
 236                 *p = *p ^ *q;
 237                 p++;
 238                 q--;
 239         }
 240
 241         ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
 242 }
 243
 244 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
 245 {
 246         ccp_dm_free(&data->dm_wa);
 247         ccp_sg_free(&data->sg_wa);
 248 }
 249
 250 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
 251                          struct scatterlist *sg, u64 sg_len,
 252                          unsigned int dm_len,
 253                          enum dma_data_direction dir)
 254 {
 255         int ret;
 256
 257         memset(data, 0, sizeof(*data));
 258
 259         ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
 260                                    dir);
 261         if (ret)
 262                 goto e_err;
 263
 264         ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
 265         if (ret)
 266                 goto e_err;
 267
 268         return 0;
 269
 270 e_err:
 271         ccp_free_data(data, cmd_q);
 272
 273         return ret;
 274 }
 275
 276 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
 277 {
 278         struct ccp_sg_workarea *sg_wa = &data->sg_wa;
 279         struct ccp_dm_workarea *dm_wa = &data->dm_wa;
 280         unsigned int buf_count, nbytes;
 281
 282         /* Clear the buffer if setting it */
 283         if (!from)
 284                 memset(dm_wa->address, 0, dm_wa->length);
 285
 286         if (!sg_wa->sg)
 287                 return 0;
 288
 289         /* Perform the copy operation
 290          *   nbytes will always be <= UINT_MAX because dm_wa->length is
 291          *   an unsigned int
 292          */
 293         nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
 294         scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
 295                                  nbytes, from);
 296
 297         /* Update the structures and generate the count */
 298         buf_count = 0;
 299         while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
 300                 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
 301                              dm_wa->length - buf_count);
 302                 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
 303
 304                 buf_count += nbytes;
 305                 ccp_update_sg_workarea(sg_wa, nbytes);
 306         }
 307
 308         return buf_count;
 309 }
 310
 311 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
 312 {
 313         return ccp_queue_buf(data, 0);
 314 }
 315
 316 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
 317 {
 318         return ccp_queue_buf(data, 1);
 319 }
 320
 321 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
 322                              struct ccp_op *op, unsigned int block_size,
 323                              bool blocksize_op)
 324 {
 325         unsigned int sg_src_len, sg_dst_len, op_len;
 326
 327         /* The CCP can only DMA from/to one address each per operation. This
 328          * requires that we find the smallest DMA area between the source
 329          * and destination. The resulting len values will always be <= UINT_MAX
 330          * because the dma length is an unsigned int.
 331          */
 332         sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
 333         sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
 334
 335         if (dst) {
 336                 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
 337                 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
 338                 op_len = min(sg_src_len, sg_dst_len);
 339         } else {
 340                 op_len = sg_src_len;
 341         }
 342
 343         /* The data operation length will be at least block_size in length
 344          * or the smaller of available sg room remaining for the source or
 345          * the destination
 346          */
 347         op_len = max(op_len, block_size);
 348
 349         /* Unless we have to buffer data, there's no reason to wait */
 350         op->soc = 0;
 351
 352         if (sg_src_len < block_size) {
 353                 /* Not enough data in the sg element, so it
 354                  * needs to be buffered into a blocksize chunk
 355                  */
 356                 int cp_len = ccp_fill_queue_buf(src);
 357
 358                 op->soc = 1;
 359                 op->src.u.dma.address = src->dm_wa.dma.address;
 360                 op->src.u.dma.offset = 0;
 361                 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
 362         } else {
 363                 /* Enough data in the sg element, but we need to
 364                  * adjust for any previously copied data
 365                  */
 366                 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
 367                 op->src.u.dma.offset = src->sg_wa.sg_used;
 368                 op->src.u.dma.length = op_len & ~(block_size - 1);
 369
 370                 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
 371         }
 372
 373         if (dst) {
 374                 if (sg_dst_len < block_size) {
 375                         /* Not enough room in the sg element or we're on the
 376                          * last piece of data (when using padding), so the
 377                          * output needs to be buffered into a blocksize chunk
 378                          */
 379                         op->soc = 1;
 380                         op->dst.u.dma.address = dst->dm_wa.dma.address;
 381                         op->dst.u.dma.offset = 0;
 382                         op->dst.u.dma.length = op->src.u.dma.length;
 383                 } else {
 384                         /* Enough room in the sg element, but we need to
 385                          * adjust for any previously used area
 386                          */
 387                         op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
 388                         op->dst.u.dma.offset = dst->sg_wa.sg_used;
 389                         op->dst.u.dma.length = op->src.u.dma.length;
 390                 }
 391         }
 392 }
 393
 394 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
 395                              struct ccp_op *op)
 396 {
 397         op->init = 0;
 398
 399         if (dst) {
 400                 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
 401                         ccp_empty_queue_buf(dst);
 402                 else
 403                         ccp_update_sg_workarea(&dst->sg_wa,
 404                                                op->dst.u.dma.length);
 405         }
 406 }
 407
 408 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
 409                                struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 410                                u32 byte_swap, bool from)
 411 {
 412         struct ccp_op op;
 413
 414         memset(&op, 0, sizeof(op));
 415
 416         op.cmd_q = cmd_q;
 417         op.jobid = jobid;
 418         op.eom = 1;
 419
 420         if (from) {
 421                 op.soc = 1;
 422                 op.src.type = CCP_MEMTYPE_SB;
 423                 op.src.u.sb = sb;
 424                 op.dst.type = CCP_MEMTYPE_SYSTEM;
 425                 op.dst.u.dma.address = wa->dma.address;
 426                 op.dst.u.dma.length = wa->length;
 427         } else {
 428                 op.src.type = CCP_MEMTYPE_SYSTEM;
 429                 op.src.u.dma.address = wa->dma.address;
 430                 op.src.u.dma.length = wa->length;
 431                 op.dst.type = CCP_MEMTYPE_SB;
 432                 op.dst.u.sb = sb;
 433         }
 434
 435         op.u.passthru.byte_swap = byte_swap;
 436
 437         return cmd_q->ccp->vdata->perform->passthru(&op);
 438 }
 439
 440 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
 441                           struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 442                           u32 byte_swap)
 443 {
 444         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
 445 }
 446
 447 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
 448                             struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 449                             u32 byte_swap)
 450 {
 451         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
 452 }
 453
 454 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
 455                                 struct ccp_cmd *cmd)
 456 {
 457         struct ccp_aes_engine *aes = &cmd->u.aes;
 458         struct ccp_dm_workarea key, ctx;
 459         struct ccp_data src;
 460         struct ccp_op op;
 461         unsigned int dm_offset;
 462         int ret;
 463
 464         if (!((aes->key_len == AES_KEYSIZE_128) ||
 465               (aes->key_len == AES_KEYSIZE_192) ||
 466               (aes->key_len == AES_KEYSIZE_256)))
 467                 return -EINVAL;
 468
 469         if (aes->src_len & (AES_BLOCK_SIZE - 1))
 470                 return -EINVAL;
 471
 472         if (aes->iv_len != AES_BLOCK_SIZE)
 473                 return -EINVAL;
 474
 475         if (!aes->key || !aes->iv || !aes->src)
 476                 return -EINVAL;
 477
 478         if (aes->cmac_final) {
 479                 if (aes->cmac_key_len != AES_BLOCK_SIZE)
 480                         return -EINVAL;
 481
 482                 if (!aes->cmac_key)
 483                         return -EINVAL;
 484         }
 485
 486         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 487         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 488
 489         ret = -EIO;
 490         memset(&op, 0, sizeof(op));
 491         op.cmd_q = cmd_q;
 492         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 493         op.sb_key = cmd_q->sb_key;
 494         op.sb_ctx = cmd_q->sb_ctx;
 495         op.init = 1;
 496         op.u.aes.type = aes->type;
 497         op.u.aes.mode = aes->mode;
 498         op.u.aes.action = aes->action;
 499
 500         /* All supported key sizes fit in a single (32-byte) SB entry
 501          * and must be in little endian format. Use the 256-bit byte
 502          * swap passthru option to convert from big endian to little
 503          * endian.
 504          */
 505         ret = ccp_init_dm_workarea(&key, cmd_q,
 506                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 507                                    DMA_TO_DEVICE);
 508         if (ret)
 509                 return ret;
 510
 511         dm_offset = CCP_SB_BYTES - aes->key_len;
 512         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 513         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 514                              CCP_PASSTHRU_BYTESWAP_256BIT);
 515         if (ret) {
 516                 cmd->engine_error = cmd_q->cmd_error;
 517                 goto e_key;
 518         }
 519
 520         /* The AES context fits in a single (32-byte) SB entry and
 521          * must be in little endian format. Use the 256-bit byte swap
 522          * passthru option to convert from big endian to little endian.
 523          */
 524         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 525                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 526                                    DMA_BIDIRECTIONAL);
 527         if (ret)
 528                 goto e_key;
 529
 530         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 531         ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 532         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 533                              CCP_PASSTHRU_BYTESWAP_256BIT);
 534         if (ret) {
 535                 cmd->engine_error = cmd_q->cmd_error;
 536                 goto e_ctx;
 537         }
 538
 539         /* Send data to the CCP AES engine */
 540         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 541                             AES_BLOCK_SIZE, DMA_TO_DEVICE);
 542         if (ret)
 543                 goto e_ctx;
 544
 545         while (src.sg_wa.bytes_left) {
 546                 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
 547                 if (aes->cmac_final && !src.sg_wa.bytes_left) {
 548                         op.eom = 1;
 549
 550                         /* Push the K1/K2 key to the CCP now */
 551                         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
 552                                                op.sb_ctx,
 553                                                CCP_PASSTHRU_BYTESWAP_256BIT);
 554                         if (ret) {
 555                                 cmd->engine_error = cmd_q->cmd_error;
 556                                 goto e_src;
 557                         }
 558
 559                         ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
 560                                         aes->cmac_key_len);
 561                         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 562                                              CCP_PASSTHRU_BYTESWAP_256BIT);
 563                         if (ret) {
 564                                 cmd->engine_error = cmd_q->cmd_error;
 565                                 goto e_src;
 566                         }
 567                 }
 568
 569                 ret = cmd_q->ccp->vdata->perform->aes(&op);
 570                 if (ret) {
 571                         cmd->engine_error = cmd_q->cmd_error;
 572                         goto e_src;
 573                 }
 574
 575                 ccp_process_data(&src, NULL, &op);
 576         }
 577
 578         /* Retrieve the AES context - convert from LE to BE using
 579          * 32-byte (256-bit) byteswapping
 580          */
 581         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 582                                CCP_PASSTHRU_BYTESWAP_256BIT);
 583         if (ret) {
 584                 cmd->engine_error = cmd_q->cmd_error;
 585                 goto e_src;
 586         }
 587
 588         /* ...but we only need AES_BLOCK_SIZE bytes */
 589         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 590         ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 591
 592 e_src:
 593         ccp_free_data(&src, cmd_q);
 594
 595 e_ctx:
 596         ccp_dm_free(&ctx);
 597
 598 e_key:
 599         ccp_dm_free(&key);
 600
 601         return ret;
 602 }
 603
 604 static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q,
 605                                struct ccp_cmd *cmd)
 606 {
 607         struct ccp_aes_engine *aes = &cmd->u.aes;
 608         struct ccp_dm_workarea key, ctx, final_wa, tag;
 609         struct ccp_data src, dst;
 610         struct ccp_data aad;
 611         struct ccp_op op;
 612
 613         unsigned long long *final;
 614         unsigned int dm_offset;
 615         unsigned int ilen;
 616         bool in_place = true; /* Default value */
 617         int ret;
 618
 619         struct scatterlist *p_inp, sg_inp[2];
 620         struct scatterlist *p_tag, sg_tag[2];
 621         struct scatterlist *p_outp, sg_outp[2];
 622         struct scatterlist *p_aad;
 623
 624         if (!aes->iv)
 625                 return -EINVAL;
 626
 627         if (!((aes->key_len == AES_KEYSIZE_128) ||
 628                 (aes->key_len == AES_KEYSIZE_192) ||
 629                 (aes->key_len == AES_KEYSIZE_256)))
 630                 return -EINVAL;
 631
 632         if (!aes->key) /* Gotta have a key SGL */
 633                 return -EINVAL;
 634
 635         /* First, decompose the source buffer into AAD & PT,
 636          * and the destination buffer into AAD, CT & tag, or
 637          * the input into CT & tag.
 638          * It is expected that the input and output SGs will
 639          * be valid, even if the AAD and input lengths are 0.
 640          */
 641         p_aad = aes->src;
 642         p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
 643         p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
 644         if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 645                 ilen = aes->src_len;
 646                 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
 647         } else {
 648                 /* Input length for decryption includes tag */
 649                 ilen = aes->src_len - AES_BLOCK_SIZE;
 650                 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
 651         }
 652
 653         memset(&op, 0, sizeof(op));
 654         op.cmd_q = cmd_q;
 655         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 656         op.sb_key = cmd_q->sb_key; /* Pre-allocated */
 657         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
 658         op.init = 1;
 659         op.u.aes.type = aes->type;
 660
 661         /* Copy the key to the LSB */
 662         ret = ccp_init_dm_workarea(&key, cmd_q,
 663                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 664                                    DMA_TO_DEVICE);
 665         if (ret)
 666                 return ret;
 667
 668         dm_offset = CCP_SB_BYTES - aes->key_len;
 669         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 670         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 671                              CCP_PASSTHRU_BYTESWAP_256BIT);
 672         if (ret) {
 673                 cmd->engine_error = cmd_q->cmd_error;
 674                 goto e_key;
 675         }
 676
 677         /* Copy the context (IV) to the LSB.
 678          * There is an assumption here that the IV is 96 bits in length, plus
 679          * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
 680          */
 681         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 682                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 683                                    DMA_BIDIRECTIONAL);
 684         if (ret)
 685                 goto e_key;
 686
 687         dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
 688         ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 689
 690         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 691                              CCP_PASSTHRU_BYTESWAP_256BIT);
 692         if (ret) {
 693                 cmd->engine_error = cmd_q->cmd_error;
 694                 goto e_ctx;
 695         }
 696
 697         op.init = 1;
 698         if (aes->aad_len > 0) {
 699                 /* Step 1: Run a GHASH over the Additional Authenticated Data */
 700                 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
 701                                     AES_BLOCK_SIZE,
 702                                     DMA_TO_DEVICE);
 703                 if (ret)
 704                         goto e_ctx;
 705
 706                 op.u.aes.mode = CCP_AES_MODE_GHASH;
 707                 op.u.aes.action = CCP_AES_GHASHAAD;
 708
 709                 while (aad.sg_wa.bytes_left) {
 710                         ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
 711
 712                         ret = cmd_q->ccp->vdata->perform->aes(&op);
 713                         if (ret) {
 714                                 cmd->engine_error = cmd_q->cmd_error;
 715                                 goto e_aad;
 716                         }
 717
 718                         ccp_process_data(&aad, NULL, &op);
 719                         op.init = 0;
 720                 }
 721         }
 722
 723         op.u.aes.mode = CCP_AES_MODE_GCTR;
 724         op.u.aes.action = aes->action;
 725
 726         if (ilen > 0) {
 727                 /* Step 2: Run a GCTR over the plaintext */
 728                 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
 729
 730                 ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
 731                                     AES_BLOCK_SIZE,
 732                                     in_place ? DMA_BIDIRECTIONAL
 733                                              : DMA_TO_DEVICE);
 734                 if (ret)
 735                         goto e_ctx;
 736
 737                 if (in_place) {
 738                         dst = src;
 739                 } else {
 740                         ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
 741                                             AES_BLOCK_SIZE, DMA_FROM_DEVICE);
 742                         if (ret)
 743                                 goto e_src;
 744                 }
 745
 746                 op.soc = 0;
 747                 op.eom = 0;
 748                 op.init = 1;
 749                 while (src.sg_wa.bytes_left) {
 750                         ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
 751                         if (!src.sg_wa.bytes_left) {
 752                                 unsigned int nbytes = aes->src_len
 753                                                       % AES_BLOCK_SIZE;
 754
 755                                 if (nbytes) {
 756                                         op.eom = 1;
 757                                         op.u.aes.size = (nbytes * 8) - 1;
 758                                 }
 759                         }
 760
 761                         ret = cmd_q->ccp->vdata->perform->aes(&op);
 762                         if (ret) {
 763                                 cmd->engine_error = cmd_q->cmd_error;
 764                                 goto e_dst;
 765                         }
 766
 767                         ccp_process_data(&src, &dst, &op);
 768                         op.init = 0;
 769                 }
 770         }
 771
 772         /* Step 3: Update the IV portion of the context with the original IV */
 773         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 774                                CCP_PASSTHRU_BYTESWAP_256BIT);
 775         if (ret) {
 776                 cmd->engine_error = cmd_q->cmd_error;
 777                 goto e_dst;
 778         }
 779
 780         ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 781
 782         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 783                              CCP_PASSTHRU_BYTESWAP_256BIT);
 784         if (ret) {
 785                 cmd->engine_error = cmd_q->cmd_error;
 786                 goto e_dst;
 787         }
 788
 789         /* Step 4: Concatenate the lengths of the AAD and source, and
 790          * hash that 16 byte buffer.
 791          */
 792         ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
 793                                    DMA_BIDIRECTIONAL);
 794         if (ret)
 795                 goto e_dst;
 796         final = (unsigned long long *) final_wa.address;
 797         final[0] = cpu_to_be64(aes->aad_len * 8);
 798         final[1] = cpu_to_be64(ilen * 8);
 799
 800         op.u.aes.mode = CCP_AES_MODE_GHASH;
 801         op.u.aes.action = CCP_AES_GHASHFINAL;
 802         op.src.type = CCP_MEMTYPE_SYSTEM;
 803         op.src.u.dma.address = final_wa.dma.address;
 804         op.src.u.dma.length = AES_BLOCK_SIZE;
 805         op.dst.type = CCP_MEMTYPE_SYSTEM;
 806         op.dst.u.dma.address = final_wa.dma.address;
 807         op.dst.u.dma.length = AES_BLOCK_SIZE;
 808         op.eom = 1;
 809         op.u.aes.size = 0;
 810         ret = cmd_q->ccp->vdata->perform->aes(&op);
 811         if (ret)
 812                 goto e_dst;
 813
 814         if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 815                 /* Put the ciphered tag after the ciphertext. */
 816                 ccp_get_dm_area(&final_wa, 0, p_tag, 0, AES_BLOCK_SIZE);
 817         } else {
 818                 /* Does this ciphered tag match the input? */
 819                 ret = ccp_init_dm_workarea(&tag, cmd_q, AES_BLOCK_SIZE,
 820                                            DMA_BIDIRECTIONAL);
 821                 if (ret)
 822                         goto e_tag;
 823                 ccp_set_dm_area(&tag, 0, p_tag, 0, AES_BLOCK_SIZE);
 824
 825                 ret = memcmp(tag.address, final_wa.address, AES_BLOCK_SIZE);
 826                 ccp_dm_free(&tag);
 827         }
 828
 829 e_tag:
 830         ccp_dm_free(&final_wa);
 831
 832 e_dst:
 833         if (aes->src_len && !in_place)
 834                 ccp_free_data(&dst, cmd_q);
 835
 836 e_src:
 837         if (aes->src_len)
 838                 ccp_free_data(&src, cmd_q);
 839
 840 e_aad:
 841         if (aes->aad_len)
 842                 ccp_free_data(&aad, cmd_q);
 843
 844 e_ctx:
 845         ccp_dm_free(&ctx);
 846
 847 e_key:
 848         ccp_dm_free(&key);
 849
 850         return ret;
 851 }
 852
 853 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 854 {
 855         struct ccp_aes_engine *aes = &cmd->u.aes;
 856         struct ccp_dm_workarea key, ctx;
 857         struct ccp_data src, dst;
 858         struct ccp_op op;
 859         unsigned int dm_offset;
 860         bool in_place = false;
 861         int ret;
 862
 863         if (aes->mode == CCP_AES_MODE_CMAC)
 864                 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
 865
 866         if (aes->mode == CCP_AES_MODE_GCM)
 867                 return ccp_run_aes_gcm_cmd(cmd_q, cmd);
 868
 869         if (!((aes->key_len == AES_KEYSIZE_128) ||
 870               (aes->key_len == AES_KEYSIZE_192) ||
 871               (aes->key_len == AES_KEYSIZE_256)))
 872                 return -EINVAL;
 873
 874         if (((aes->mode == CCP_AES_MODE_ECB) ||
 875              (aes->mode == CCP_AES_MODE_CBC) ||
 876              (aes->mode == CCP_AES_MODE_CFB)) &&
 877             (aes->src_len & (AES_BLOCK_SIZE - 1)))
 878                 return -EINVAL;
 879
 880         if (!aes->key || !aes->src || !aes->dst)
 881                 return -EINVAL;
 882
 883         if (aes->mode != CCP_AES_MODE_ECB) {
 884                 if (aes->iv_len != AES_BLOCK_SIZE)
 885                         return -EINVAL;
 886
 887                 if (!aes->iv)
 888                         return -EINVAL;
 889         }
 890
 891         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 892         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 893
 894         ret = -EIO;
 895         memset(&op, 0, sizeof(op));
 896         op.cmd_q = cmd_q;
 897         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 898         op.sb_key = cmd_q->sb_key;
 899         op.sb_ctx = cmd_q->sb_ctx;
 900         op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
 901         op.u.aes.type = aes->type;
 902         op.u.aes.mode = aes->mode;
 903         op.u.aes.action = aes->action;
 904
 905         /* All supported key sizes fit in a single (32-byte) SB entry
 906          * and must be in little endian format. Use the 256-bit byte
 907          * swap passthru option to convert from big endian to little
 908          * endian.
 909          */
 910         ret = ccp_init_dm_workarea(&key, cmd_q,
 911                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 912                                    DMA_TO_DEVICE);
 913         if (ret)
 914                 return ret;
 915
 916         dm_offset = CCP_SB_BYTES - aes->key_len;
 917         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 918         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 919                              CCP_PASSTHRU_BYTESWAP_256BIT);
 920         if (ret) {
 921                 cmd->engine_error = cmd_q->cmd_error;
 922                 goto e_key;
 923         }
 924
 925         /* The AES context fits in a single (32-byte) SB entry and
 926          * must be in little endian format. Use the 256-bit byte swap
 927          * passthru option to convert from big endian to little endian.
 928          */
 929         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 930                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 931                                    DMA_BIDIRECTIONAL);
 932         if (ret)
 933                 goto e_key;
 934
 935         if (aes->mode != CCP_AES_MODE_ECB) {
 936                 /* Load the AES context - convert to LE */
 937                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 938                 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 939                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 940                                      CCP_PASSTHRU_BYTESWAP_256BIT);
 941                 if (ret) {
 942                         cmd->engine_error = cmd_q->cmd_error;
 943                         goto e_ctx;
 944                 }
 945         }
 946         switch (aes->mode) {
 947         case CCP_AES_MODE_CFB: /* CFB128 only */
 948         case CCP_AES_MODE_CTR:
 949                 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
 950                 break;
 951         default:
 952                 op.u.aes.size = 0;
 953         }
 954
 955         /* Prepare the input and output data workareas. For in-place
 956          * operations we need to set the dma direction to BIDIRECTIONAL
 957          * and copy the src workarea to the dst workarea.
 958          */
 959         if (sg_virt(aes->src) == sg_virt(aes->dst))
 960                 in_place = true;
 961
 962         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 963                             AES_BLOCK_SIZE,
 964                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
 965         if (ret)
 966                 goto e_ctx;
 967
 968         if (in_place) {
 969                 dst = src;
 970         } else {
 971                 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
 972                                     AES_BLOCK_SIZE, DMA_FROM_DEVICE);
 973                 if (ret)
 974                         goto e_src;
 975         }
 976
 977         /* Send data to the CCP AES engine */
 978         while (src.sg_wa.bytes_left) {
 979                 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
 980                 if (!src.sg_wa.bytes_left) {
 981                         op.eom = 1;
 982
 983                         /* Since we don't retrieve the AES context in ECB
 984                          * mode we have to wait for the operation to complete
 985                          * on the last piece of data
 986                          */
 987                         if (aes->mode == CCP_AES_MODE_ECB)
 988                                 op.soc = 1;
 989                 }
 990
 991                 ret = cmd_q->ccp->vdata->perform->aes(&op);
 992                 if (ret) {
 993                         cmd->engine_error = cmd_q->cmd_error;
 994                         goto e_dst;
 995                 }
 996
 997                 ccp_process_data(&src, &dst, &op);
 998         }
 999
1000         if (aes->mode != CCP_AES_MODE_ECB) {
1001                 /* Retrieve the AES context - convert from LE to BE using
1002                  * 32-byte (256-bit) byteswapping
1003                  */
1004                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1005                                        CCP_PASSTHRU_BYTESWAP_256BIT);
1006                 if (ret) {
1007                         cmd->engine_error = cmd_q->cmd_error;
1008                         goto e_dst;
1009                 }
1010
1011                 /* ...but we only need AES_BLOCK_SIZE bytes */
1012                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1013                 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1014         }
1015
1016 e_dst:
1017         if (!in_place)
1018                 ccp_free_data(&dst, cmd_q);
1019
1020 e_src:
1021         ccp_free_data(&src, cmd_q);
1022
1023 e_ctx:
1024         ccp_dm_free(&ctx);
1025
1026 e_key:
1027         ccp_dm_free(&key);
1028
1029         return ret;
1030 }
1031
1032 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1033                                struct ccp_cmd *cmd)
1034 {
1035         struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1036         struct ccp_dm_workarea key, ctx;
1037         struct ccp_data src, dst;
1038         struct ccp_op op;
1039         unsigned int unit_size, dm_offset;
1040         bool in_place = false;
1041         unsigned int sb_count;
1042         enum ccp_aes_type aestype;
1043         int ret;
1044
1045         switch (xts->unit_size) {
1046         case CCP_XTS_AES_UNIT_SIZE_16:
1047                 unit_size = 16;
1048                 break;
1049         case CCP_XTS_AES_UNIT_SIZE_512:
1050                 unit_size = 512;
1051                 break;
1052         case CCP_XTS_AES_UNIT_SIZE_1024:
1053                 unit_size = 1024;
1054                 break;
1055         case CCP_XTS_AES_UNIT_SIZE_2048:
1056                 unit_size = 2048;
1057                 break;
1058         case CCP_XTS_AES_UNIT_SIZE_4096:
1059                 unit_size = 4096;
1060                 break;
1061
1062         default:
1063                 return -EINVAL;
1064         }
1065
1066         if (xts->key_len == AES_KEYSIZE_128)
1067                 aestype = CCP_AES_TYPE_128;
1068         else if (xts->key_len == AES_KEYSIZE_256)
1069                 aestype = CCP_AES_TYPE_256;
1070         else
1071                 return -EINVAL;
1072
1073         if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1074                 return -EINVAL;
1075
1076         if (xts->iv_len != AES_BLOCK_SIZE)
1077                 return -EINVAL;
1078
1079         if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1080                 return -EINVAL;
1081
1082         BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1083         BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1084
1085         ret = -EIO;
1086         memset(&op, 0, sizeof(op));
1087         op.cmd_q = cmd_q;
1088         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1089         op.sb_key = cmd_q->sb_key;
1090         op.sb_ctx = cmd_q->sb_ctx;
1091         op.init = 1;
1092         op.u.xts.type = aestype;
1093         op.u.xts.action = xts->action;
1094         op.u.xts.unit_size = xts->unit_size;
1095
1096         /* A version 3 device only supports 128-bit keys, which fits into a
1097          * single SB entry. A version 5 device uses a 512-bit vector, so two
1098          * SB entries.
1099          */
1100         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1101                 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1102         else
1103                 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1104         ret = ccp_init_dm_workarea(&key, cmd_q,
1105                                    sb_count * CCP_SB_BYTES,
1106                                    DMA_TO_DEVICE);
1107         if (ret)
1108                 return ret;
1109
1110         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1111                 /* All supported key sizes must be in little endian format.
1112                  * Use the 256-bit byte swap passthru option to convert from
1113                  * big endian to little endian.
1114                  */
1115                 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1116                 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1117                 ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1118         } else {
1119                 /* Version 5 CCPs use a 512-bit space for the key: each portion
1120                  * occupies 256 bits, or one entire slot, and is zero-padded.
1121                  */
1122                 unsigned int pad;
1123
1124                 dm_offset = CCP_SB_BYTES;
1125                 pad = dm_offset - xts->key_len;
1126                 ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1127                 ccp_set_dm_area(&key, dm_offset + pad, xts->key, xts->key_len,
1128                                 xts->key_len);
1129         }
1130         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1131                              CCP_PASSTHRU_BYTESWAP_256BIT);
1132         if (ret) {
1133                 cmd->engine_error = cmd_q->cmd_error;
1134                 goto e_key;
1135         }
1136
1137         /* The AES context fits in a single (32-byte) SB entry and
1138          * for XTS is already in little endian format so no byte swapping
1139          * is needed.
1140          */
1141         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1142                                    CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1143                                    DMA_BIDIRECTIONAL);
1144         if (ret)
1145                 goto e_key;
1146
1147         ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1148         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1149                              CCP_PASSTHRU_BYTESWAP_NOOP);
1150         if (ret) {
1151                 cmd->engine_error = cmd_q->cmd_error;
1152                 goto e_ctx;
1153         }
1154
1155         /* Prepare the input and output data workareas. For in-place
1156          * operations we need to set the dma direction to BIDIRECTIONAL
1157          * and copy the src workarea to the dst workarea.
1158          */
1159         if (sg_virt(xts->src) == sg_virt(xts->dst))
1160                 in_place = true;
1161
1162         ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1163                             unit_size,
1164                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1165         if (ret)
1166                 goto e_ctx;
1167
1168         if (in_place) {
1169                 dst = src;
1170         } else {
1171                 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1172                                     unit_size, DMA_FROM_DEVICE);
1173                 if (ret)
1174                         goto e_src;
1175         }
1176
1177         /* Send data to the CCP AES engine */
1178         while (src.sg_wa.bytes_left) {
1179                 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1180                 if (!src.sg_wa.bytes_left)
1181                         op.eom = 1;
1182
1183                 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1184                 if (ret) {
1185                         cmd->engine_error = cmd_q->cmd_error;
1186                         goto e_dst;
1187                 }
1188
1189                 ccp_process_data(&src, &dst, &op);
1190         }
1191
1192         /* Retrieve the AES context - convert from LE to BE using
1193          * 32-byte (256-bit) byteswapping
1194          */
1195         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1196                                CCP_PASSTHRU_BYTESWAP_256BIT);
1197         if (ret) {
1198                 cmd->engine_error = cmd_q->cmd_error;
1199                 goto e_dst;
1200         }
1201
1202         /* ...but we only need AES_BLOCK_SIZE bytes */
1203         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1204         ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1205
1206 e_dst:
1207         if (!in_place)
1208                 ccp_free_data(&dst, cmd_q);
1209
1210 e_src:
1211         ccp_free_data(&src, cmd_q);
1212
1213 e_ctx:
1214         ccp_dm_free(&ctx);
1215
1216 e_key:
1217         ccp_dm_free(&key);
1218
1219         return ret;
1220 }
1221
1222 static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1223 {
1224         struct ccp_des3_engine *des3 = &cmd->u.des3;
1225
1226         struct ccp_dm_workarea key, ctx;
1227         struct ccp_data src, dst;
1228         struct ccp_op op;
1229         unsigned int dm_offset;
1230         unsigned int len_singlekey;
1231         bool in_place = false;
1232         int ret;
1233
1234         /* Error checks */
1235         if (!cmd_q->ccp->vdata->perform->des3)
1236                 return -EINVAL;
1237
1238         if (des3->key_len != DES3_EDE_KEY_SIZE)
1239                 return -EINVAL;
1240
1241         if (((des3->mode == CCP_DES3_MODE_ECB) ||
1242                 (des3->mode == CCP_DES3_MODE_CBC)) &&
1243                 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1244                 return -EINVAL;
1245
1246         if (!des3->key || !des3->src || !des3->dst)
1247                 return -EINVAL;
1248
1249         if (des3->mode != CCP_DES3_MODE_ECB) {
1250                 if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1251                         return -EINVAL;
1252
1253                 if (!des3->iv)
1254                         return -EINVAL;
1255         }
1256
1257         ret = -EIO;
1258         /* Zero out all the fields of the command desc */
1259         memset(&op, 0, sizeof(op));
1260
1261         /* Set up the Function field */
1262         op.cmd_q = cmd_q;
1263         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1264         op.sb_key = cmd_q->sb_key;
1265
1266         op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1267         op.u.des3.type = des3->type;
1268         op.u.des3.mode = des3->mode;
1269         op.u.des3.action = des3->action;
1270
1271         /*
1272          * All supported key sizes fit in a single (32-byte) KSB entry and
1273          * (like AES) must be in little endian format. Use the 256-bit byte
1274          * swap passthru option to convert from big endian to little endian.
1275          */
1276         ret = ccp_init_dm_workarea(&key, cmd_q,
1277                                    CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1278                                    DMA_TO_DEVICE);
1279         if (ret)
1280                 return ret;
1281
1282         /*
1283          * The contents of the key triplet are in the reverse order of what
1284          * is required by the engine. Copy the 3 pieces individually to put
1285          * them where they belong.
1286          */
1287         dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1288
1289         len_singlekey = des3->key_len / 3;
1290         ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1291                         des3->key, 0, len_singlekey);
1292         ccp_set_dm_area(&key, dm_offset + len_singlekey,
1293                         des3->key, len_singlekey, len_singlekey);
1294         ccp_set_dm_area(&key, dm_offset,
1295                         des3->key, 2 * len_singlekey, len_singlekey);
1296
1297         /* Copy the key to the SB */
1298         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1299                              CCP_PASSTHRU_BYTESWAP_256BIT);
1300         if (ret) {
1301                 cmd->engine_error = cmd_q->cmd_error;
1302                 goto e_key;
1303         }
1304
1305         /*
1306          * The DES3 context fits in a single (32-byte) KSB entry and
1307          * must be in little endian format. Use the 256-bit byte swap
1308          * passthru option to convert from big endian to little endian.
1309          */
1310         if (des3->mode != CCP_DES3_MODE_ECB) {
1311                 u32 load_mode;
1312
1313                 op.sb_ctx = cmd_q->sb_ctx;
1314
1315                 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1316                                            CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1317                                            DMA_BIDIRECTIONAL);
1318                 if (ret)
1319                         goto e_key;
1320
1321                 /* Load the context into the LSB */
1322                 dm_offset = CCP_SB_BYTES - des3->iv_len;
1323                 ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0, des3->iv_len);
1324
1325                 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1326                         load_mode = CCP_PASSTHRU_BYTESWAP_NOOP;
1327                 else
1328                         load_mode = CCP_PASSTHRU_BYTESWAP_256BIT;
1329                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1330                                      load_mode);
1331                 if (ret) {
1332                         cmd->engine_error = cmd_q->cmd_error;
1333                         goto e_ctx;
1334                 }
1335         }
1336
1337         /*
1338          * Prepare the input and output data workareas. For in-place
1339          * operations we need to set the dma direction to BIDIRECTIONAL
1340          * and copy the src workarea to the dst workarea.
1341          */
1342         if (sg_virt(des3->src) == sg_virt(des3->dst))
1343                 in_place = true;
1344
1345         ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1346                         DES3_EDE_BLOCK_SIZE,
1347                         in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1348         if (ret)
1349                 goto e_ctx;
1350
1351         if (in_place)
1352                 dst = src;
1353         else {
1354                 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1355                                 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1356                 if (ret)
1357                         goto e_src;
1358         }
1359
1360         /* Send data to the CCP DES3 engine */
1361         while (src.sg_wa.bytes_left) {
1362                 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1363                 if (!src.sg_wa.bytes_left) {
1364                         op.eom = 1;
1365
1366                         /* Since we don't retrieve the context in ECB mode
1367                          * we have to wait for the operation to complete
1368                          * on the last piece of data
1369                          */
1370                         op.soc = 0;
1371                 }
1372
1373                 ret = cmd_q->ccp->vdata->perform->des3(&op);
1374                 if (ret) {
1375                         cmd->engine_error = cmd_q->cmd_error;
1376                         goto e_dst;
1377                 }
1378
1379                 ccp_process_data(&src, &dst, &op);
1380         }
1381
1382         if (des3->mode != CCP_DES3_MODE_ECB) {
1383                 /* Retrieve the context and make BE */
1384                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1385                                        CCP_PASSTHRU_BYTESWAP_256BIT);
1386                 if (ret) {
1387                         cmd->engine_error = cmd_q->cmd_error;
1388                         goto e_dst;
1389                 }
1390
1391                 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1392                 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1393                         dm_offset = CCP_SB_BYTES - des3->iv_len;
1394                 else
1395                         dm_offset = 0;
1396                 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1397                                 DES3_EDE_BLOCK_SIZE);
1398         }
1399 e_dst:
1400         if (!in_place)
1401                 ccp_free_data(&dst, cmd_q);
1402
1403 e_src:
1404         ccp_free_data(&src, cmd_q);
1405
1406 e_ctx:
1407         if (des3->mode != CCP_DES3_MODE_ECB)
1408                 ccp_dm_free(&ctx);
1409
1410 e_key:
1411         ccp_dm_free(&key);
1412
1413         return ret;
1414 }
1415
1416 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1417 {
1418         struct ccp_sha_engine *sha = &cmd->u.sha;
1419         struct ccp_dm_workarea ctx;
1420         struct ccp_data src;
1421         struct ccp_op op;
1422         unsigned int ioffset, ooffset;
1423         unsigned int digest_size;
1424         int sb_count;
1425         const void *init;
1426         u64 block_size;
1427         int ctx_size;
1428         int ret;
1429
1430         switch (sha->type) {
1431         case CCP_SHA_TYPE_1:
1432                 if (sha->ctx_len < SHA1_DIGEST_SIZE)
1433                         return -EINVAL;
1434                 block_size = SHA1_BLOCK_SIZE;
1435                 break;
1436         case CCP_SHA_TYPE_224:
1437                 if (sha->ctx_len < SHA224_DIGEST_SIZE)
1438                         return -EINVAL;
1439                 block_size = SHA224_BLOCK_SIZE;
1440                 break;
1441         case CCP_SHA_TYPE_256:
1442                 if (sha->ctx_len < SHA256_DIGEST_SIZE)
1443                         return -EINVAL;
1444                 block_size = SHA256_BLOCK_SIZE;
1445                 break;
1446         case CCP_SHA_TYPE_384:
1447                 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1448                     || sha->ctx_len < SHA384_DIGEST_SIZE)
1449                         return -EINVAL;
1450                 block_size = SHA384_BLOCK_SIZE;
1451                 break;
1452         case CCP_SHA_TYPE_512:
1453                 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1454                     || sha->ctx_len < SHA512_DIGEST_SIZE)
1455                         return -EINVAL;
1456                 block_size = SHA512_BLOCK_SIZE;
1457                 break;
1458         default:
1459                 return -EINVAL;
1460         }
1461
1462         if (!sha->ctx)
1463                 return -EINVAL;
1464
1465         if (!sha->final && (sha->src_len & (block_size - 1)))
1466                 return -EINVAL;
1467
1468         /* The version 3 device can't handle zero-length input */
1469         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1470
1471                 if (!sha->src_len) {
1472                         unsigned int digest_len;
1473                         const u8 *sha_zero;
1474
1475                         /* Not final, just return */
1476                         if (!sha->final)
1477                                 return 0;
1478
1479                         /* CCP can't do a zero length sha operation so the
1480                          * caller must buffer the data.
1481                          */
1482                         if (sha->msg_bits)
1483                                 return -EINVAL;
1484
1485                         /* The CCP cannot perform zero-length sha operations
1486                          * so the caller is required to buffer data for the
1487                          * final operation. However, a sha operation for a
1488                          * message with a total length of zero is valid so
1489                          * known values are required to supply the result.
1490                          */
1491                         switch (sha->type) {
1492                         case CCP_SHA_TYPE_1:
1493                                 sha_zero = sha1_zero_message_hash;
1494                                 digest_len = SHA1_DIGEST_SIZE;
1495                                 break;
1496                         case CCP_SHA_TYPE_224:
1497                                 sha_zero = sha224_zero_message_hash;
1498                                 digest_len = SHA224_DIGEST_SIZE;
1499                                 break;
1500                         case CCP_SHA_TYPE_256:
1501                                 sha_zero = sha256_zero_message_hash;
1502                                 digest_len = SHA256_DIGEST_SIZE;
1503                                 break;
1504                         default:
1505                                 return -EINVAL;
1506                         }
1507
1508                         scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1509                                                  digest_len, 1);
1510
1511                         return 0;
1512                 }
1513         }
1514
1515         /* Set variables used throughout */
1516         switch (sha->type) {
1517         case CCP_SHA_TYPE_1:
1518                 digest_size = SHA1_DIGEST_SIZE;
1519                 init = (void *) ccp_sha1_init;
1520                 ctx_size = SHA1_DIGEST_SIZE;
1521                 sb_count = 1;
1522                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1523                         ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1524                 else
1525                         ooffset = ioffset = 0;
1526                 break;
1527         case CCP_SHA_TYPE_224:
1528                 digest_size = SHA224_DIGEST_SIZE;
1529                 init = (void *) ccp_sha224_init;
1530                 ctx_size = SHA256_DIGEST_SIZE;
1531                 sb_count = 1;
1532                 ioffset = 0;
1533                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1534                         ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1535                 else
1536                         ooffset = 0;
1537                 break;
1538         case CCP_SHA_TYPE_256:
1539                 digest_size = SHA256_DIGEST_SIZE;
1540                 init = (void *) ccp_sha256_init;
1541                 ctx_size = SHA256_DIGEST_SIZE;
1542                 sb_count = 1;
1543                 ooffset = ioffset = 0;
1544                 break;
1545         case CCP_SHA_TYPE_384:
1546                 digest_size = SHA384_DIGEST_SIZE;
1547                 init = (void *) ccp_sha384_init;
1548                 ctx_size = SHA512_DIGEST_SIZE;
1549                 sb_count = 2;
1550                 ioffset = 0;
1551                 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1552                 break;
1553         case CCP_SHA_TYPE_512:
1554                 digest_size = SHA512_DIGEST_SIZE;
1555                 init = (void *) ccp_sha512_init;
1556                 ctx_size = SHA512_DIGEST_SIZE;
1557                 sb_count = 2;
1558                 ooffset = ioffset = 0;
1559                 break;
1560         default:
1561                 ret = -EINVAL;
1562                 goto e_data;
1563         }
1564
1565         /* For zero-length plaintext the src pointer is ignored;
1566          * otherwise both parts must be valid
1567          */
1568         if (sha->src_len && !sha->src)
1569                 return -EINVAL;
1570
1571         memset(&op, 0, sizeof(op));
1572         op.cmd_q = cmd_q;
1573         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1574         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1575         op.u.sha.type = sha->type;
1576         op.u.sha.msg_bits = sha->msg_bits;
1577
1578         /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1579          * SHA384/512 require 2 adjacent SB slots, with the right half in the
1580          * first slot, and the left half in the second. Each portion must then
1581          * be in little endian format: use the 256-bit byte swap option.
1582          */
1583         ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1584                                    DMA_BIDIRECTIONAL);
1585         if (ret)
1586                 return ret;
1587         if (sha->first) {
1588                 switch (sha->type) {
1589                 case CCP_SHA_TYPE_1:
1590                 case CCP_SHA_TYPE_224:
1591                 case CCP_SHA_TYPE_256:
1592                         memcpy(ctx.address + ioffset, init, ctx_size);
1593                         break;
1594                 case CCP_SHA_TYPE_384:
1595                 case CCP_SHA_TYPE_512:
1596                         memcpy(ctx.address + ctx_size / 2, init,
1597                                ctx_size / 2);
1598                         memcpy(ctx.address, init + ctx_size / 2,
1599                                ctx_size / 2);
1600                         break;
1601                 default:
1602                         ret = -EINVAL;
1603                         goto e_ctx;
1604                 }
1605         } else {
1606                 /* Restore the context */
1607                 ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1608                                 sb_count * CCP_SB_BYTES);
1609         }
1610
1611         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1612                              CCP_PASSTHRU_BYTESWAP_256BIT);
1613         if (ret) {
1614                 cmd->engine_error = cmd_q->cmd_error;
1615                 goto e_ctx;
1616         }
1617
1618         if (sha->src) {
1619                 /* Send data to the CCP SHA engine; block_size is set above */
1620                 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1621                                     block_size, DMA_TO_DEVICE);
1622                 if (ret)
1623                         goto e_ctx;
1624
1625                 while (src.sg_wa.bytes_left) {
1626                         ccp_prepare_data(&src, NULL, &op, block_size, false);
1627                         if (sha->final && !src.sg_wa.bytes_left)
1628                                 op.eom = 1;
1629
1630                         ret = cmd_q->ccp->vdata->perform->sha(&op);
1631                         if (ret) {
1632                                 cmd->engine_error = cmd_q->cmd_error;
1633                                 goto e_data;
1634                         }
1635
1636                         ccp_process_data(&src, NULL, &op);
1637                 }
1638         } else {
1639                 op.eom = 1;
1640                 ret = cmd_q->ccp->vdata->perform->sha(&op);
1641                 if (ret) {
1642                         cmd->engine_error = cmd_q->cmd_error;
1643                         goto e_data;
1644                 }
1645         }
1646
1647         /* Retrieve the SHA context - convert from LE to BE using
1648          * 32-byte (256-bit) byteswapping to BE
1649          */
1650         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1651                                CCP_PASSTHRU_BYTESWAP_256BIT);
1652         if (ret) {
1653                 cmd->engine_error = cmd_q->cmd_error;
1654                 goto e_data;
1655         }
1656
1657         if (sha->final) {
1658                 /* Finishing up, so get the digest */
1659                 switch (sha->type) {
1660                 case CCP_SHA_TYPE_1:
1661                 case CCP_SHA_TYPE_224:
1662                 case CCP_SHA_TYPE_256:
1663                         ccp_get_dm_area(&ctx, ooffset,
1664                                         sha->ctx, 0,
1665                                         digest_size);
1666                         break;
1667                 case CCP_SHA_TYPE_384:
1668                 case CCP_SHA_TYPE_512:
1669                         ccp_get_dm_area(&ctx, 0,
1670                                         sha->ctx, LSB_ITEM_SIZE - ooffset,
1671                                         LSB_ITEM_SIZE);
1672                         ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1673                                         sha->ctx, 0,
1674                                         LSB_ITEM_SIZE - ooffset);
1675                         break;
1676                 default:
1677                         ret = -EINVAL;
1678                         goto e_ctx;
1679                 }
1680         } else {
1681                 /* Stash the context */
1682                 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1683                                 sb_count * CCP_SB_BYTES);
1684         }
1685
1686         if (sha->final && sha->opad) {
1687                 /* HMAC operation, recursively perform final SHA */
1688                 struct ccp_cmd hmac_cmd;
1689                 struct scatterlist sg;
1690                 u8 *hmac_buf;
1691
1692                 if (sha->opad_len != block_size) {
1693                         ret = -EINVAL;
1694                         goto e_data;
1695                 }
1696
1697                 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1698                 if (!hmac_buf) {
1699                         ret = -ENOMEM;
1700                         goto e_data;
1701                 }
1702                 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1703
1704                 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1705                 switch (sha->type) {
1706                 case CCP_SHA_TYPE_1:
1707                 case CCP_SHA_TYPE_224:
1708                 case CCP_SHA_TYPE_256:
1709                         memcpy(hmac_buf + block_size,
1710                                ctx.address + ooffset,
1711                                digest_size);
1712                         break;
1713                 case CCP_SHA_TYPE_384:
1714                 case CCP_SHA_TYPE_512:
1715                         memcpy(hmac_buf + block_size,
1716                                ctx.address + LSB_ITEM_SIZE + ooffset,
1717                                LSB_ITEM_SIZE);
1718                         memcpy(hmac_buf + block_size +
1719                                (LSB_ITEM_SIZE - ooffset),
1720                                ctx.address,
1721                                LSB_ITEM_SIZE);
1722                         break;
1723                 default:
1724                         ret = -EINVAL;
1725                         goto e_ctx;
1726                 }
1727
1728                 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1729                 hmac_cmd.engine = CCP_ENGINE_SHA;
1730                 hmac_cmd.u.sha.type = sha->type;
1731                 hmac_cmd.u.sha.ctx = sha->ctx;
1732                 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1733                 hmac_cmd.u.sha.src = &sg;
1734                 hmac_cmd.u.sha.src_len = block_size + digest_size;
1735                 hmac_cmd.u.sha.opad = NULL;
1736                 hmac_cmd.u.sha.opad_len = 0;
1737                 hmac_cmd.u.sha.first = 1;
1738                 hmac_cmd.u.sha.final = 1;
1739                 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1740
1741                 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1742                 if (ret)
1743                         cmd->engine_error = hmac_cmd.engine_error;
1744
1745                 kfree(hmac_buf);
1746         }
1747
1748 e_data:
1749         if (sha->src)
1750                 ccp_free_data(&src, cmd_q);
1751
1752 e_ctx:
1753         ccp_dm_free(&ctx);
1754
1755         return ret;
1756 }
1757
1758 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1759 {
1760         struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1761         struct ccp_dm_workarea exp, src, dst;
1762         struct ccp_op op;
1763         unsigned int sb_count, i_len, o_len;
1764         int ret;
1765
1766         /* Check against the maximum allowable size, in bits */
1767         if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1768                 return -EINVAL;
1769
1770         if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1771                 return -EINVAL;
1772
1773         memset(&op, 0, sizeof(op));
1774         op.cmd_q = cmd_q;
1775         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1776
1777         /* The RSA modulus must precede the message being acted upon, so
1778          * it must be copied to a DMA area where the message and the
1779          * modulus can be concatenated.  Therefore the input buffer
1780          * length required is twice the output buffer length (which
1781          * must be a multiple of 256-bits).  Compute o_len, i_len in bytes.
1782          * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1783          * required.
1784          */
1785         o_len = 32 * ((rsa->key_size + 255) / 256);
1786         i_len = o_len * 2;
1787
1788         sb_count = 0;
1789         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1790                 /* sb_count is the number of storage block slots required
1791                  * for the modulus.
1792                  */
1793                 sb_count = o_len / CCP_SB_BYTES;
1794                 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1795                                                                 sb_count);
1796                 if (!op.sb_key)
1797                         return -EIO;
1798         } else {
1799                 /* A version 5 device allows a modulus size that will not fit
1800                  * in the LSB, so the command will transfer it from memory.
1801                  * Set the sb key to the default, even though it's not used.
1802                  */
1803                 op.sb_key = cmd_q->sb_key;
1804         }
1805
1806         /* The RSA exponent must be in little endian format. Reverse its
1807          * byte order.
1808          */
1809         ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1810         if (ret)
1811                 goto e_sb;
1812
1813         ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1814         if (ret)
1815                 goto e_exp;
1816
1817         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1818                 /* Copy the exponent to the local storage block, using
1819                  * as many 32-byte blocks as were allocated above. It's
1820                  * already little endian, so no further change is required.
1821                  */
1822                 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1823                                      CCP_PASSTHRU_BYTESWAP_NOOP);
1824                 if (ret) {
1825                         cmd->engine_error = cmd_q->cmd_error;
1826                         goto e_exp;
1827                 }
1828         } else {
1829                 /* The exponent can be retrieved from memory via DMA. */
1830                 op.exp.u.dma.address = exp.dma.address;
1831                 op.exp.u.dma.offset = 0;
1832         }
1833
1834         /* Concatenate the modulus and the message. Both the modulus and
1835          * the operands must be in little endian format.  Since the input
1836          * is in big endian format it must be converted.
1837          */
1838         ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1839         if (ret)
1840                 goto e_exp;
1841
1842         ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1843         if (ret)
1844                 goto e_src;
1845         ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1846         if (ret)
1847                 goto e_src;
1848
1849         /* Prepare the output area for the operation */
1850         ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1851         if (ret)
1852                 goto e_src;
1853
1854         op.soc = 1;
1855         op.src.u.dma.address = src.dma.address;
1856         op.src.u.dma.offset = 0;
1857         op.src.u.dma.length = i_len;
1858         op.dst.u.dma.address = dst.dma.address;
1859         op.dst.u.dma.offset = 0;
1860         op.dst.u.dma.length = o_len;
1861
1862         op.u.rsa.mod_size = rsa->key_size;
1863         op.u.rsa.input_len = i_len;
1864
1865         ret = cmd_q->ccp->vdata->perform->rsa(&op);
1866         if (ret) {
1867                 cmd->engine_error = cmd_q->cmd_error;
1868                 goto e_dst;
1869         }
1870
1871         ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1872
1873 e_dst:
1874         ccp_dm_free(&dst);
1875
1876 e_src:
1877         ccp_dm_free(&src);
1878
1879 e_exp:
1880         ccp_dm_free(&exp);
1881
1882 e_sb:
1883         if (sb_count)
1884                 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1885
1886         return ret;
1887 }
1888
1889 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1890                                 struct ccp_cmd *cmd)
1891 {
1892         struct ccp_passthru_engine *pt = &cmd->u.passthru;
1893         struct ccp_dm_workarea mask;
1894         struct ccp_data src, dst;
1895         struct ccp_op op;
1896         bool in_place = false;
1897         unsigned int i;
1898         int ret = 0;
1899
1900         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1901                 return -EINVAL;
1902
1903         if (!pt->src || !pt->dst)
1904                 return -EINVAL;
1905
1906         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1907                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1908                         return -EINVAL;
1909                 if (!pt->mask)
1910                         return -EINVAL;
1911         }
1912
1913         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1914
1915         memset(&op, 0, sizeof(op));
1916         op.cmd_q = cmd_q;
1917         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1918
1919         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1920                 /* Load the mask */
1921                 op.sb_key = cmd_q->sb_key;
1922
1923                 ret = ccp_init_dm_workarea(&mask, cmd_q,
1924                                            CCP_PASSTHRU_SB_COUNT *
1925                                            CCP_SB_BYTES,
1926                                            DMA_TO_DEVICE);
1927                 if (ret)
1928                         return ret;
1929
1930                 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1931                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
1932                                      CCP_PASSTHRU_BYTESWAP_NOOP);
1933                 if (ret) {
1934                         cmd->engine_error = cmd_q->cmd_error;
1935                         goto e_mask;
1936                 }
1937         }
1938
1939         /* Prepare the input and output data workareas. For in-place
1940          * operations we need to set the dma direction to BIDIRECTIONAL
1941          * and copy the src workarea to the dst workarea.
1942          */
1943         if (sg_virt(pt->src) == sg_virt(pt->dst))
1944                 in_place = true;
1945
1946         ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1947                             CCP_PASSTHRU_MASKSIZE,
1948                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1949         if (ret)
1950                 goto e_mask;
1951
1952         if (in_place) {
1953                 dst = src;
1954         } else {
1955                 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1956                                     CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1957                 if (ret)
1958                         goto e_src;
1959         }
1960
1961         /* Send data to the CCP Passthru engine
1962          *   Because the CCP engine works on a single source and destination
1963          *   dma address at a time, each entry in the source scatterlist
1964          *   (after the dma_map_sg call) must be less than or equal to the
1965          *   (remaining) length in the destination scatterlist entry and the
1966          *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
1967          */
1968         dst.sg_wa.sg_used = 0;
1969         for (i = 1; i <= src.sg_wa.dma_count; i++) {
1970                 if (!dst.sg_wa.sg ||
1971                     (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
1972                         ret = -EINVAL;
1973                         goto e_dst;
1974                 }
1975
1976                 if (i == src.sg_wa.dma_count) {
1977                         op.eom = 1;
1978                         op.soc = 1;
1979                 }
1980
1981                 op.src.type = CCP_MEMTYPE_SYSTEM;
1982                 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
1983                 op.src.u.dma.offset = 0;
1984                 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
1985
1986                 op.dst.type = CCP_MEMTYPE_SYSTEM;
1987                 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
1988                 op.dst.u.dma.offset = dst.sg_wa.sg_used;
1989                 op.dst.u.dma.length = op.src.u.dma.length;
1990
1991                 ret = cmd_q->ccp->vdata->perform->passthru(&op);
1992                 if (ret) {
1993                         cmd->engine_error = cmd_q->cmd_error;
1994                         goto e_dst;
1995                 }
1996
1997                 dst.sg_wa.sg_used += src.sg_wa.sg->length;
1998                 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
1999                         dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2000                         dst.sg_wa.sg_used = 0;
2001                 }
2002                 src.sg_wa.sg = sg_next(src.sg_wa.sg);
2003         }
2004
2005 e_dst:
2006         if (!in_place)
2007                 ccp_free_data(&dst, cmd_q);
2008
2009 e_src:
2010         ccp_free_data(&src, cmd_q);
2011
2012 e_mask:
2013         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2014                 ccp_dm_free(&mask);
2015
2016         return ret;
2017 }
2018
2019 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2020                                       struct ccp_cmd *cmd)
2021 {
2022         struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2023         struct ccp_dm_workarea mask;
2024         struct ccp_op op;
2025         int ret;
2026
2027         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2028                 return -EINVAL;
2029
2030         if (!pt->src_dma || !pt->dst_dma)
2031                 return -EINVAL;
2032
2033         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2034                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2035                         return -EINVAL;
2036                 if (!pt->mask)
2037                         return -EINVAL;
2038         }
2039
2040         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2041
2042         memset(&op, 0, sizeof(op));
2043         op.cmd_q = cmd_q;
2044         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2045
2046         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2047                 /* Load the mask */
2048                 op.sb_key = cmd_q->sb_key;
2049
2050                 mask.length = pt->mask_len;
2051                 mask.dma.address = pt->mask;
2052                 mask.dma.length = pt->mask_len;
2053
2054                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2055                                      CCP_PASSTHRU_BYTESWAP_NOOP);
2056                 if (ret) {
2057                         cmd->engine_error = cmd_q->cmd_error;
2058                         return ret;
2059                 }
2060         }
2061
2062         /* Send data to the CCP Passthru engine */
2063         op.eom = 1;
2064         op.soc = 1;
2065
2066         op.src.type = CCP_MEMTYPE_SYSTEM;
2067         op.src.u.dma.address = pt->src_dma;
2068         op.src.u.dma.offset = 0;
2069         op.src.u.dma.length = pt->src_len;
2070
2071         op.dst.type = CCP_MEMTYPE_SYSTEM;
2072         op.dst.u.dma.address = pt->dst_dma;
2073         op.dst.u.dma.offset = 0;
2074         op.dst.u.dma.length = pt->src_len;
2075
2076         ret = cmd_q->ccp->vdata->perform->passthru(&op);
2077         if (ret)
2078                 cmd->engine_error = cmd_q->cmd_error;
2079
2080         return ret;
2081 }
2082
2083 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2084 {
2085         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2086         struct ccp_dm_workarea src, dst;
2087         struct ccp_op op;
2088         int ret;
2089         u8 *save;
2090
2091         if (!ecc->u.mm.operand_1 ||
2092             (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2093                 return -EINVAL;
2094
2095         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2096                 if (!ecc->u.mm.operand_2 ||
2097                     (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2098                         return -EINVAL;
2099
2100         if (!ecc->u.mm.result ||
2101             (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2102                 return -EINVAL;
2103
2104         memset(&op, 0, sizeof(op));
2105         op.cmd_q = cmd_q;
2106         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2107
2108         /* Concatenate the modulus and the operands. Both the modulus and
2109          * the operands must be in little endian format.  Since the input
2110          * is in big endian format it must be converted and placed in a
2111          * fixed length buffer.
2112          */
2113         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2114                                    DMA_TO_DEVICE);
2115         if (ret)
2116                 return ret;
2117
2118         /* Save the workarea address since it is updated in order to perform
2119          * the concatenation
2120          */
2121         save = src.address;
2122
2123         /* Copy the ECC modulus */
2124         ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2125         if (ret)
2126                 goto e_src;
2127         src.address += CCP_ECC_OPERAND_SIZE;
2128
2129         /* Copy the first operand */
2130         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2131                                       ecc->u.mm.operand_1_len);
2132         if (ret)
2133                 goto e_src;
2134         src.address += CCP_ECC_OPERAND_SIZE;
2135
2136         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2137                 /* Copy the second operand */
2138                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2139                                               ecc->u.mm.operand_2_len);
2140                 if (ret)
2141                         goto e_src;
2142                 src.address += CCP_ECC_OPERAND_SIZE;
2143         }
2144
2145         /* Restore the workarea address */
2146         src.address = save;
2147
2148         /* Prepare the output area for the operation */
2149         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2150                                    DMA_FROM_DEVICE);
2151         if (ret)
2152                 goto e_src;
2153
2154         op.soc = 1;
2155         op.src.u.dma.address = src.dma.address;
2156         op.src.u.dma.offset = 0;
2157         op.src.u.dma.length = src.length;
2158         op.dst.u.dma.address = dst.dma.address;
2159         op.dst.u.dma.offset = 0;
2160         op.dst.u.dma.length = dst.length;
2161
2162         op.u.ecc.function = cmd->u.ecc.function;
2163
2164         ret = cmd_q->ccp->vdata->perform->ecc(&op);
2165         if (ret) {
2166                 cmd->engine_error = cmd_q->cmd_error;
2167                 goto e_dst;
2168         }
2169
2170         ecc->ecc_result = le16_to_cpup(
2171                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2172         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2173                 ret = -EIO;
2174                 goto e_dst;
2175         }
2176
2177         /* Save the ECC result */
2178         ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2179                                 CCP_ECC_MODULUS_BYTES);
2180
2181 e_dst:
2182         ccp_dm_free(&dst);
2183
2184 e_src:
2185         ccp_dm_free(&src);
2186
2187         return ret;
2188 }
2189
2190 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2191 {
2192         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2193         struct ccp_dm_workarea src, dst;
2194         struct ccp_op op;
2195         int ret;
2196         u8 *save;
2197
2198         if (!ecc->u.pm.point_1.x ||
2199             (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2200             !ecc->u.pm.point_1.y ||
2201             (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2202                 return -EINVAL;
2203
2204         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2205                 if (!ecc->u.pm.point_2.x ||
2206                     (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2207                     !ecc->u.pm.point_2.y ||
2208                     (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2209                         return -EINVAL;
2210         } else {
2211                 if (!ecc->u.pm.domain_a ||
2212                     (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2213                         return -EINVAL;
2214
2215                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2216                         if (!ecc->u.pm.scalar ||
2217                             (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2218                                 return -EINVAL;
2219         }
2220
2221         if (!ecc->u.pm.result.x ||
2222             (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2223             !ecc->u.pm.result.y ||
2224             (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2225                 return -EINVAL;
2226
2227         memset(&op, 0, sizeof(op));
2228         op.cmd_q = cmd_q;
2229         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2230
2231         /* Concatenate the modulus and the operands. Both the modulus and
2232          * the operands must be in little endian format.  Since the input
2233          * is in big endian format it must be converted and placed in a
2234          * fixed length buffer.
2235          */
2236         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2237                                    DMA_TO_DEVICE);
2238         if (ret)
2239                 return ret;
2240
2241         /* Save the workarea address since it is updated in order to perform
2242          * the concatenation
2243          */
2244         save = src.address;
2245
2246         /* Copy the ECC modulus */
2247         ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2248         if (ret)
2249                 goto e_src;
2250         src.address += CCP_ECC_OPERAND_SIZE;
2251
2252         /* Copy the first point X and Y coordinate */
2253         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2254                                       ecc->u.pm.point_1.x_len);
2255         if (ret)
2256                 goto e_src;
2257         src.address += CCP_ECC_OPERAND_SIZE;
2258         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2259                                       ecc->u.pm.point_1.y_len);
2260         if (ret)
2261                 goto e_src;
2262         src.address += CCP_ECC_OPERAND_SIZE;
2263
2264         /* Set the first point Z coordinate to 1 */
2265         *src.address = 0x01;
2266         src.address += CCP_ECC_OPERAND_SIZE;
2267
2268         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2269                 /* Copy the second point X and Y coordinate */
2270                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2271                                               ecc->u.pm.point_2.x_len);
2272                 if (ret)
2273                         goto e_src;
2274                 src.address += CCP_ECC_OPERAND_SIZE;
2275                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2276                                               ecc->u.pm.point_2.y_len);
2277                 if (ret)
2278                         goto e_src;
2279                 src.address += CCP_ECC_OPERAND_SIZE;
2280
2281                 /* Set the second point Z coordinate to 1 */
2282                 *src.address = 0x01;
2283                 src.address += CCP_ECC_OPERAND_SIZE;
2284         } else {
2285                 /* Copy the Domain "a" parameter */
2286                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2287                                               ecc->u.pm.domain_a_len);
2288                 if (ret)
2289                         goto e_src;
2290                 src.address += CCP_ECC_OPERAND_SIZE;
2291
2292                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2293                         /* Copy the scalar value */
2294                         ret = ccp_reverse_set_dm_area(&src, 0,
2295                                                       ecc->u.pm.scalar, 0,
2296                                                       ecc->u.pm.scalar_len);
2297                         if (ret)
2298                                 goto e_src;
2299                         src.address += CCP_ECC_OPERAND_SIZE;
2300                 }
2301         }
2302
2303         /* Restore the workarea address */
2304         src.address = save;
2305
2306         /* Prepare the output area for the operation */
2307         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2308                                    DMA_FROM_DEVICE);
2309         if (ret)
2310                 goto e_src;
2311
2312         op.soc = 1;
2313         op.src.u.dma.address = src.dma.address;
2314         op.src.u.dma.offset = 0;
2315         op.src.u.dma.length = src.length;
2316         op.dst.u.dma.address = dst.dma.address;
2317         op.dst.u.dma.offset = 0;
2318         op.dst.u.dma.length = dst.length;
2319
2320         op.u.ecc.function = cmd->u.ecc.function;
2321
2322         ret = cmd_q->ccp->vdata->perform->ecc(&op);
2323         if (ret) {
2324                 cmd->engine_error = cmd_q->cmd_error;
2325                 goto e_dst;
2326         }
2327
2328         ecc->ecc_result = le16_to_cpup(
2329                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2330         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2331                 ret = -EIO;
2332                 goto e_dst;
2333         }
2334
2335         /* Save the workarea address since it is updated as we walk through
2336          * to copy the point math result
2337          */
2338         save = dst.address;
2339
2340         /* Save the ECC result X and Y coordinates */
2341         ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2342                                 CCP_ECC_MODULUS_BYTES);
2343         dst.address += CCP_ECC_OUTPUT_SIZE;
2344         ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2345                                 CCP_ECC_MODULUS_BYTES);
2346         dst.address += CCP_ECC_OUTPUT_SIZE;
2347
2348         /* Restore the workarea address */
2349         dst.address = save;
2350
2351 e_dst:
2352         ccp_dm_free(&dst);
2353
2354 e_src:
2355         ccp_dm_free(&src);
2356
2357         return ret;
2358 }
2359
2360 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2361 {
2362         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2363
2364         ecc->ecc_result = 0;
2365
2366         if (!ecc->mod ||
2367             (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2368                 return -EINVAL;
2369
2370         switch (ecc->function) {
2371         case CCP_ECC_FUNCTION_MMUL_384BIT:
2372         case CCP_ECC_FUNCTION_MADD_384BIT:
2373         case CCP_ECC_FUNCTION_MINV_384BIT:
2374                 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2375
2376         case CCP_ECC_FUNCTION_PADD_384BIT:
2377         case CCP_ECC_FUNCTION_PMUL_384BIT:
2378         case CCP_ECC_FUNCTION_PDBL_384BIT:
2379                 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2380
2381         default:
2382                 return -EINVAL;
2383         }
2384 }
2385
2386 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2387 {
2388         int ret;
2389
2390         cmd->engine_error = 0;
2391         cmd_q->cmd_error = 0;
2392         cmd_q->int_rcvd = 0;
2393         cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2394
2395         switch (cmd->engine) {
2396         case CCP_ENGINE_AES:
2397                 ret = ccp_run_aes_cmd(cmd_q, cmd);
2398                 break;
2399         case CCP_ENGINE_XTS_AES_128:
2400                 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2401                 break;
2402         case CCP_ENGINE_DES3:
2403                 ret = ccp_run_des3_cmd(cmd_q, cmd);
2404                 break;
2405         case CCP_ENGINE_SHA:
2406                 ret = ccp_run_sha_cmd(cmd_q, cmd);
2407                 break;
2408         case CCP_ENGINE_RSA:
2409                 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2410                 break;
2411         case CCP_ENGINE_PASSTHRU:
2412                 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2413                         ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2414                 else
2415                         ret = ccp_run_passthru_cmd(cmd_q, cmd);
2416                 break;
2417         case CCP_ENGINE_ECC:
2418                 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2419                 break;
2420         default:
2421                 ret = -EINVAL;
2422         }
2423
2424         return ret;
2425 }