drivers/crypto/ccp/ccp-ops.c

   1 /*
   2  * AMD Cryptographic Coprocessor (CCP) driver
   3  *
   4  * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
   5  *
   6  * Author: Tom Lendacky <[email protected]>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/kernel.h>
  15 #include <linux/pci.h>
  16 #include <linux/interrupt.h>
  17 #include <crypto/scatterwalk.h>
  18 #include <linux/ccp.h>
  19
  20 #include "ccp-dev.h"
  21
  22 /* SHA initial context values */
  23 static const __be32 ccp_sha1_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
  24         cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
  25         cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
  26         cpu_to_be32(SHA1_H4), 0, 0, 0,
  27 };
  28
  29 static const __be32 ccp_sha224_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
  30         cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
  31         cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
  32         cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
  33         cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
  34 };
  35
  36 static const __be32 ccp_sha256_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
  37         cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
  38         cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
  39         cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
  40         cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
  41 };
  42
  43 static u32 ccp_alloc_ksb(struct ccp_device *ccp, unsigned int count)
  44 {
  45         int start;
  46
  47         for (;;) {
  48                 mutex_lock(&ccp->ksb_mutex);
  49
  50                 start = (u32)bitmap_find_next_zero_area(ccp->ksb,
  51                                                         ccp->ksb_count,
  52                                                         ccp->ksb_start,
  53                                                         count, 0);
  54                 if (start <= ccp->ksb_count) {
  55                         bitmap_set(ccp->ksb, start, count);
  56
  57                         mutex_unlock(&ccp->ksb_mutex);
  58                         break;
  59                 }
  60
  61                 ccp->ksb_avail = 0;
  62
  63                 mutex_unlock(&ccp->ksb_mutex);
  64
  65                 /* Wait for KSB entries to become available */
  66                 if (wait_event_interruptible(ccp->ksb_queue, ccp->ksb_avail))
  67                         return 0;
  68         }
  69
  70         return KSB_START + start;
  71 }
  72
  73 static void ccp_free_ksb(struct ccp_device *ccp, unsigned int start,
  74                          unsigned int count)
  75 {
  76         if (!start)
  77                 return;
  78
  79         mutex_lock(&ccp->ksb_mutex);
  80
  81         bitmap_clear(ccp->ksb, start - KSB_START, count);
  82
  83         ccp->ksb_avail = 1;
  84
  85         mutex_unlock(&ccp->ksb_mutex);
  86
  87         wake_up_interruptible_all(&ccp->ksb_queue);
  88 }
  89
  90 static u32 ccp_gen_jobid(struct ccp_device *ccp)
  91 {
  92         return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
  93 }
  94
  95 static void ccp_sg_free(struct ccp_sg_workarea *wa)
  96 {
  97         if (wa->dma_count)
  98                 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
  99
 100         wa->dma_count = 0;
 101 }
 102
 103 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
 104                                 struct scatterlist *sg, u64 len,
 105                                 enum dma_data_direction dma_dir)
 106 {
 107         memset(wa, 0, sizeof(*wa));
 108
 109         wa->sg = sg;
 110         if (!sg)
 111                 return 0;
 112
 113         wa->nents = sg_nents_for_len(sg, len);
 114         if (wa->nents < 0)
 115                 return wa->nents;
 116
 117         wa->bytes_left = len;
 118         wa->sg_used = 0;
 119
 120         if (len == 0)
 121                 return 0;
 122
 123         if (dma_dir == DMA_NONE)
 124                 return 0;
 125
 126         wa->dma_sg = sg;
 127         wa->dma_dev = dev;
 128         wa->dma_dir = dma_dir;
 129         wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
 130         if (!wa->dma_count)
 131                 return -ENOMEM;
 132
 133         return 0;
 134 }
 135
 136 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
 137 {
 138         unsigned int nbytes = min_t(u64, len, wa->bytes_left);
 139
 140         if (!wa->sg)
 141                 return;
 142
 143         wa->sg_used += nbytes;
 144         wa->bytes_left -= nbytes;
 145         if (wa->sg_used == wa->sg->length) {
 146                 wa->sg = sg_next(wa->sg);
 147                 wa->sg_used = 0;
 148         }
 149 }
 150
 151 static void ccp_dm_free(struct ccp_dm_workarea *wa)
 152 {
 153         if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
 154                 if (wa->address)
 155                         dma_pool_free(wa->dma_pool, wa->address,
 156                                       wa->dma.address);
 157         } else {
 158                 if (wa->dma.address)
 159                         dma_unmap_single(wa->dev, wa->dma.address, wa->length,
 160                                          wa->dma.dir);
 161                 kfree(wa->address);
 162         }
 163
 164         wa->address = NULL;
 165         wa->dma.address = 0;
 166 }
 167
 168 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
 169                                 struct ccp_cmd_queue *cmd_q,
 170                                 unsigned int len,
 171                                 enum dma_data_direction dir)
 172 {
 173         memset(wa, 0, sizeof(*wa));
 174
 175         if (!len)
 176                 return 0;
 177
 178         wa->dev = cmd_q->ccp->dev;
 179         wa->length = len;
 180
 181         if (len <= CCP_DMAPOOL_MAX_SIZE) {
 182                 wa->dma_pool = cmd_q->dma_pool;
 183
 184                 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
 185                                              &wa->dma.address);
 186                 if (!wa->address)
 187                         return -ENOMEM;
 188
 189                 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
 190
 191                 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
 192         } else {
 193                 wa->address = kzalloc(len, GFP_KERNEL);
 194                 if (!wa->address)
 195                         return -ENOMEM;
 196
 197                 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
 198                                                  dir);
 199                 if (!wa->dma.address)
 200                         return -ENOMEM;
 201
 202                 wa->dma.length = len;
 203         }
 204         wa->dma.dir = dir;
 205
 206         return 0;
 207 }
 208
 209 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 210                             struct scatterlist *sg, unsigned int sg_offset,
 211                             unsigned int len)
 212 {
 213         WARN_ON(!wa->address);
 214
 215         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 216                                  0);
 217 }
 218
 219 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 220                             struct scatterlist *sg, unsigned int sg_offset,
 221                             unsigned int len)
 222 {
 223         WARN_ON(!wa->address);
 224
 225         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 226                                  1);
 227 }
 228
 229 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 230                                    struct scatterlist *sg,
 231                                    unsigned int len, unsigned int se_len,
 232                                    bool sign_extend)
 233 {
 234         unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
 235         u8 buffer[CCP_REVERSE_BUF_SIZE];
 236
 237         if (WARN_ON(se_len > sizeof(buffer)))
 238                 return -EINVAL;
 239
 240         sg_offset = len;
 241         dm_offset = 0;
 242         nbytes = len;
 243         while (nbytes) {
 244                 ksb_len = min_t(unsigned int, nbytes, se_len);
 245                 sg_offset -= ksb_len;
 246
 247                 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 0);
 248                 for (i = 0; i < ksb_len; i++)
 249                         wa->address[dm_offset + i] = buffer[ksb_len - i - 1];
 250
 251                 dm_offset += ksb_len;
 252                 nbytes -= ksb_len;
 253
 254                 if ((ksb_len != se_len) && sign_extend) {
 255                         /* Must sign-extend to nearest sign-extend length */
 256                         if (wa->address[dm_offset - 1] & 0x80)
 257                                 memset(wa->address + dm_offset, 0xff,
 258                                        se_len - ksb_len);
 259                 }
 260         }
 261
 262         return 0;
 263 }
 264
 265 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
 266                                     struct scatterlist *sg,
 267                                     unsigned int len)
 268 {
 269         unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
 270         u8 buffer[CCP_REVERSE_BUF_SIZE];
 271
 272         sg_offset = 0;
 273         dm_offset = len;
 274         nbytes = len;
 275         while (nbytes) {
 276                 ksb_len = min_t(unsigned int, nbytes, sizeof(buffer));
 277                 dm_offset -= ksb_len;
 278
 279                 for (i = 0; i < ksb_len; i++)
 280                         buffer[ksb_len - i - 1] = wa->address[dm_offset + i];
 281                 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 1);
 282
 283                 sg_offset += ksb_len;
 284                 nbytes -= ksb_len;
 285         }
 286 }
 287
 288 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
 289 {
 290         ccp_dm_free(&data->dm_wa);
 291         ccp_sg_free(&data->sg_wa);
 292 }
 293
 294 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
 295                          struct scatterlist *sg, u64 sg_len,
 296                          unsigned int dm_len,
 297                          enum dma_data_direction dir)
 298 {
 299         int ret;
 300
 301         memset(data, 0, sizeof(*data));
 302
 303         ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
 304                                    dir);
 305         if (ret)
 306                 goto e_err;
 307
 308         ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
 309         if (ret)
 310                 goto e_err;
 311
 312         return 0;
 313
 314 e_err:
 315         ccp_free_data(data, cmd_q);
 316
 317         return ret;
 318 }
 319
 320 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
 321 {
 322         struct ccp_sg_workarea *sg_wa = &data->sg_wa;
 323         struct ccp_dm_workarea *dm_wa = &data->dm_wa;
 324         unsigned int buf_count, nbytes;
 325
 326         /* Clear the buffer if setting it */
 327         if (!from)
 328                 memset(dm_wa->address, 0, dm_wa->length);
 329
 330         if (!sg_wa->sg)
 331                 return 0;
 332
 333         /* Perform the copy operation
 334          *   nbytes will always be <= UINT_MAX because dm_wa->length is
 335          *   an unsigned int
 336          */
 337         nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
 338         scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
 339                                  nbytes, from);
 340
 341         /* Update the structures and generate the count */
 342         buf_count = 0;
 343         while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
 344                 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
 345                              dm_wa->length - buf_count);
 346                 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
 347
 348                 buf_count += nbytes;
 349                 ccp_update_sg_workarea(sg_wa, nbytes);
 350         }
 351
 352         return buf_count;
 353 }
 354
 355 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
 356 {
 357         return ccp_queue_buf(data, 0);
 358 }
 359
 360 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
 361 {
 362         return ccp_queue_buf(data, 1);
 363 }
 364
 365 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
 366                              struct ccp_op *op, unsigned int block_size,
 367                              bool blocksize_op)
 368 {
 369         unsigned int sg_src_len, sg_dst_len, op_len;
 370
 371         /* The CCP can only DMA from/to one address each per operation. This
 372          * requires that we find the smallest DMA area between the source
 373          * and destination. The resulting len values will always be <= UINT_MAX
 374          * because the dma length is an unsigned int.
 375          */
 376         sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
 377         sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
 378
 379         if (dst) {
 380                 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
 381                 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
 382                 op_len = min(sg_src_len, sg_dst_len);
 383         } else {
 384                 op_len = sg_src_len;
 385         }
 386
 387         /* The data operation length will be at least block_size in length
 388          * or the smaller of available sg room remaining for the source or
 389          * the destination
 390          */
 391         op_len = max(op_len, block_size);
 392
 393         /* Unless we have to buffer data, there's no reason to wait */
 394         op->soc = 0;
 395
 396         if (sg_src_len < block_size) {
 397                 /* Not enough data in the sg element, so it
 398                  * needs to be buffered into a blocksize chunk
 399                  */
 400                 int cp_len = ccp_fill_queue_buf(src);
 401
 402                 op->soc = 1;
 403                 op->src.u.dma.address = src->dm_wa.dma.address;
 404                 op->src.u.dma.offset = 0;
 405                 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
 406         } else {
 407                 /* Enough data in the sg element, but we need to
 408                  * adjust for any previously copied data
 409                  */
 410                 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
 411                 op->src.u.dma.offset = src->sg_wa.sg_used;
 412                 op->src.u.dma.length = op_len & ~(block_size - 1);
 413
 414                 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
 415         }
 416
 417         if (dst) {
 418                 if (sg_dst_len < block_size) {
 419                         /* Not enough room in the sg element or we're on the
 420                          * last piece of data (when using padding), so the
 421                          * output needs to be buffered into a blocksize chunk
 422                          */
 423                         op->soc = 1;
 424                         op->dst.u.dma.address = dst->dm_wa.dma.address;
 425                         op->dst.u.dma.offset = 0;
 426                         op->dst.u.dma.length = op->src.u.dma.length;
 427                 } else {
 428                         /* Enough room in the sg element, but we need to
 429                          * adjust for any previously used area
 430                          */
 431                         op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
 432                         op->dst.u.dma.offset = dst->sg_wa.sg_used;
 433                         op->dst.u.dma.length = op->src.u.dma.length;
 434                 }
 435         }
 436 }
 437
 438 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
 439                              struct ccp_op *op)
 440 {
 441         op->init = 0;
 442
 443         if (dst) {
 444                 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
 445                         ccp_empty_queue_buf(dst);
 446                 else
 447                         ccp_update_sg_workarea(&dst->sg_wa,
 448                                                op->dst.u.dma.length);
 449         }
 450 }
 451
 452 static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q,
 453                                 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
 454                                 u32 byte_swap, bool from)
 455 {
 456         struct ccp_op op;
 457
 458         memset(&op, 0, sizeof(op));
 459
 460         op.cmd_q = cmd_q;
 461         op.jobid = jobid;
 462         op.eom = 1;
 463
 464         if (from) {
 465                 op.soc = 1;
 466                 op.src.type = CCP_MEMTYPE_KSB;
 467                 op.src.u.ksb = ksb;
 468                 op.dst.type = CCP_MEMTYPE_SYSTEM;
 469                 op.dst.u.dma.address = wa->dma.address;
 470                 op.dst.u.dma.length = wa->length;
 471         } else {
 472                 op.src.type = CCP_MEMTYPE_SYSTEM;
 473                 op.src.u.dma.address = wa->dma.address;
 474                 op.src.u.dma.length = wa->length;
 475                 op.dst.type = CCP_MEMTYPE_KSB;
 476                 op.dst.u.ksb = ksb;
 477         }
 478
 479         op.u.passthru.byte_swap = byte_swap;
 480
 481         return cmd_q->ccp->vdata->perform->perform_passthru(&op);
 482 }
 483
 484 static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q,
 485                            struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
 486                            u32 byte_swap)
 487 {
 488         return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, false);
 489 }
 490
 491 static int ccp_copy_from_ksb(struct ccp_cmd_queue *cmd_q,
 492                              struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
 493                              u32 byte_swap)
 494 {
 495         return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, true);
 496 }
 497
 498 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
 499                                 struct ccp_cmd *cmd)
 500 {
 501         struct ccp_aes_engine *aes = &cmd->u.aes;
 502         struct ccp_dm_workarea key, ctx;
 503         struct ccp_data src;
 504         struct ccp_op op;
 505         unsigned int dm_offset;
 506         int ret;
 507
 508         if (!((aes->key_len == AES_KEYSIZE_128) ||
 509               (aes->key_len == AES_KEYSIZE_192) ||
 510               (aes->key_len == AES_KEYSIZE_256)))
 511                 return -EINVAL;
 512
 513         if (aes->src_len & (AES_BLOCK_SIZE - 1))
 514                 return -EINVAL;
 515
 516         if (aes->iv_len != AES_BLOCK_SIZE)
 517                 return -EINVAL;
 518
 519         if (!aes->key || !aes->iv || !aes->src)
 520                 return -EINVAL;
 521
 522         if (aes->cmac_final) {
 523                 if (aes->cmac_key_len != AES_BLOCK_SIZE)
 524                         return -EINVAL;
 525
 526                 if (!aes->cmac_key)
 527                         return -EINVAL;
 528         }
 529
 530         BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
 531         BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
 532
 533         ret = -EIO;
 534         memset(&op, 0, sizeof(op));
 535         op.cmd_q = cmd_q;
 536         op.jobid = ccp_gen_jobid(cmd_q->ccp);
 537         op.ksb_key = cmd_q->ksb_key;
 538         op.ksb_ctx = cmd_q->ksb_ctx;
 539         op.init = 1;
 540         op.u.aes.type = aes->type;
 541         op.u.aes.mode = aes->mode;
 542         op.u.aes.action = aes->action;
 543
 544         /* All supported key sizes fit in a single (32-byte) KSB entry
 545          * and must be in little endian format. Use the 256-bit byte
 546          * swap passthru option to convert from big endian to little
 547          * endian.
 548          */
 549         ret = ccp_init_dm_workarea(&key, cmd_q,
 550                                    CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
 551                                    DMA_TO_DEVICE);
 552         if (ret)
 553                 return ret;
 554
 555         dm_offset = CCP_KSB_BYTES - aes->key_len;
 556         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 557         ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
 558                               CCP_PASSTHRU_BYTESWAP_256BIT);
 559         if (ret) {
 560                 cmd->engine_error = cmd_q->cmd_error;
 561                 goto e_key;
 562         }
 563
 564         /* The AES context fits in a single (32-byte) KSB entry and
 565          * must be in little endian format. Use the 256-bit byte swap
 566          * passthru option to convert from big endian to little endian.
 567          */
 568         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 569                                    CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
 570                                    DMA_BIDIRECTIONAL);
 571         if (ret)
 572                 goto e_key;
 573
 574         dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
 575         ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 576         ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
 577                               CCP_PASSTHRU_BYTESWAP_256BIT);
 578         if (ret) {
 579                 cmd->engine_error = cmd_q->cmd_error;
 580                 goto e_ctx;
 581         }
 582
 583         /* Send data to the CCP AES engine */
 584         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 585                             AES_BLOCK_SIZE, DMA_TO_DEVICE);
 586         if (ret)
 587                 goto e_ctx;
 588
 589         while (src.sg_wa.bytes_left) {
 590                 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
 591                 if (aes->cmac_final && !src.sg_wa.bytes_left) {
 592                         op.eom = 1;
 593
 594                         /* Push the K1/K2 key to the CCP now */
 595                         ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid,
 596                                                 op.ksb_ctx,
 597                                                 CCP_PASSTHRU_BYTESWAP_256BIT);
 598                         if (ret) {
 599                                 cmd->engine_error = cmd_q->cmd_error;
 600                                 goto e_src;
 601                         }
 602
 603                         ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
 604                                         aes->cmac_key_len);
 605                         ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
 606                                               CCP_PASSTHRU_BYTESWAP_256BIT);
 607                         if (ret) {
 608                                 cmd->engine_error = cmd_q->cmd_error;
 609                                 goto e_src;
 610                         }
 611                 }
 612
 613                 ret = cmd_q->ccp->vdata->perform->perform_aes(&op);
 614                 if (ret) {
 615                         cmd->engine_error = cmd_q->cmd_error;
 616                         goto e_src;
 617                 }
 618
 619                 ccp_process_data(&src, NULL, &op);
 620         }
 621
 622         /* Retrieve the AES context - convert from LE to BE using
 623          * 32-byte (256-bit) byteswapping
 624          */
 625         ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
 626                                 CCP_PASSTHRU_BYTESWAP_256BIT);
 627         if (ret) {
 628                 cmd->engine_error = cmd_q->cmd_error;
 629                 goto e_src;
 630         }
 631
 632         /* ...but we only need AES_BLOCK_SIZE bytes */
 633         dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
 634         ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 635
 636 e_src:
 637         ccp_free_data(&src, cmd_q);
 638
 639 e_ctx:
 640         ccp_dm_free(&ctx);
 641
 642 e_key:
 643         ccp_dm_free(&key);
 644
 645         return ret;
 646 }
 647
 648 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 649 {
 650         struct ccp_aes_engine *aes = &cmd->u.aes;
 651         struct ccp_dm_workarea key, ctx;
 652         struct ccp_data src, dst;
 653         struct ccp_op op;
 654         unsigned int dm_offset;
 655         bool in_place = false;
 656         int ret;
 657
 658         if (aes->mode == CCP_AES_MODE_CMAC)
 659                 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
 660
 661         if (!((aes->key_len == AES_KEYSIZE_128) ||
 662               (aes->key_len == AES_KEYSIZE_192) ||
 663               (aes->key_len == AES_KEYSIZE_256)))
 664                 return -EINVAL;
 665
 666         if (((aes->mode == CCP_AES_MODE_ECB) ||
 667              (aes->mode == CCP_AES_MODE_CBC) ||
 668              (aes->mode == CCP_AES_MODE_CFB)) &&
 669             (aes->src_len & (AES_BLOCK_SIZE - 1)))
 670                 return -EINVAL;
 671
 672         if (!aes->key || !aes->src || !aes->dst)
 673                 return -EINVAL;
 674
 675         if (aes->mode != CCP_AES_MODE_ECB) {
 676                 if (aes->iv_len != AES_BLOCK_SIZE)
 677                         return -EINVAL;
 678
 679                 if (!aes->iv)
 680                         return -EINVAL;
 681         }
 682
 683         BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
 684         BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
 685
 686         ret = -EIO;
 687         memset(&op, 0, sizeof(op));
 688         op.cmd_q = cmd_q;
 689         op.jobid = ccp_gen_jobid(cmd_q->ccp);
 690         op.ksb_key = cmd_q->ksb_key;
 691         op.ksb_ctx = cmd_q->ksb_ctx;
 692         op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
 693         op.u.aes.type = aes->type;
 694         op.u.aes.mode = aes->mode;
 695         op.u.aes.action = aes->action;
 696
 697         /* All supported key sizes fit in a single (32-byte) KSB entry
 698          * and must be in little endian format. Use the 256-bit byte
 699          * swap passthru option to convert from big endian to little
 700          * endian.
 701          */
 702         ret = ccp_init_dm_workarea(&key, cmd_q,
 703                                    CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
 704                                    DMA_TO_DEVICE);
 705         if (ret)
 706                 return ret;
 707
 708         dm_offset = CCP_KSB_BYTES - aes->key_len;
 709         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 710         ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
 711                               CCP_PASSTHRU_BYTESWAP_256BIT);
 712         if (ret) {
 713                 cmd->engine_error = cmd_q->cmd_error;
 714                 goto e_key;
 715         }
 716
 717         /* The AES context fits in a single (32-byte) KSB entry and
 718          * must be in little endian format. Use the 256-bit byte swap
 719          * passthru option to convert from big endian to little endian.
 720          */
 721         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 722                                    CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
 723                                    DMA_BIDIRECTIONAL);
 724         if (ret)
 725                 goto e_key;
 726
 727         if (aes->mode != CCP_AES_MODE_ECB) {
 728                 /* Load the AES context - conver to LE */
 729                 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
 730                 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 731                 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
 732                                       CCP_PASSTHRU_BYTESWAP_256BIT);
 733                 if (ret) {
 734                         cmd->engine_error = cmd_q->cmd_error;
 735                         goto e_ctx;
 736                 }
 737         }
 738
 739         /* Prepare the input and output data workareas. For in-place
 740          * operations we need to set the dma direction to BIDIRECTIONAL
 741          * and copy the src workarea to the dst workarea.
 742          */
 743         if (sg_virt(aes->src) == sg_virt(aes->dst))
 744                 in_place = true;
 745
 746         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 747                             AES_BLOCK_SIZE,
 748                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
 749         if (ret)
 750                 goto e_ctx;
 751
 752         if (in_place) {
 753                 dst = src;
 754         } else {
 755                 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
 756                                     AES_BLOCK_SIZE, DMA_FROM_DEVICE);
 757                 if (ret)
 758                         goto e_src;
 759         }
 760
 761         /* Send data to the CCP AES engine */
 762         while (src.sg_wa.bytes_left) {
 763                 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
 764                 if (!src.sg_wa.bytes_left) {
 765                         op.eom = 1;
 766
 767                         /* Since we don't retrieve the AES context in ECB
 768                          * mode we have to wait for the operation to complete
 769                          * on the last piece of data
 770                          */
 771                         if (aes->mode == CCP_AES_MODE_ECB)
 772                                 op.soc = 1;
 773                 }
 774
 775                 ret = cmd_q->ccp->vdata->perform->perform_aes(&op);
 776                 if (ret) {
 777                         cmd->engine_error = cmd_q->cmd_error;
 778                         goto e_dst;
 779                 }
 780
 781                 ccp_process_data(&src, &dst, &op);
 782         }
 783
 784         if (aes->mode != CCP_AES_MODE_ECB) {
 785                 /* Retrieve the AES context - convert from LE to BE using
 786                  * 32-byte (256-bit) byteswapping
 787                  */
 788                 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
 789                                         CCP_PASSTHRU_BYTESWAP_256BIT);
 790                 if (ret) {
 791                         cmd->engine_error = cmd_q->cmd_error;
 792                         goto e_dst;
 793                 }
 794
 795                 /* ...but we only need AES_BLOCK_SIZE bytes */
 796                 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
 797                 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 798         }
 799
 800 e_dst:
 801         if (!in_place)
 802                 ccp_free_data(&dst, cmd_q);
 803
 804 e_src:
 805         ccp_free_data(&src, cmd_q);
 806
 807 e_ctx:
 808         ccp_dm_free(&ctx);
 809
 810 e_key:
 811         ccp_dm_free(&key);
 812
 813         return ret;
 814 }
 815
 816 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
 817                                struct ccp_cmd *cmd)
 818 {
 819         struct ccp_xts_aes_engine *xts = &cmd->u.xts;
 820         struct ccp_dm_workarea key, ctx;
 821         struct ccp_data src, dst;
 822         struct ccp_op op;
 823         unsigned int unit_size, dm_offset;
 824         bool in_place = false;
 825         int ret;
 826
 827         switch (xts->unit_size) {
 828         case CCP_XTS_AES_UNIT_SIZE_16:
 829                 unit_size = 16;
 830                 break;
 831         case CCP_XTS_AES_UNIT_SIZE_512:
 832                 unit_size = 512;
 833                 break;
 834         case CCP_XTS_AES_UNIT_SIZE_1024:
 835                 unit_size = 1024;
 836                 break;
 837         case CCP_XTS_AES_UNIT_SIZE_2048:
 838                 unit_size = 2048;
 839                 break;
 840         case CCP_XTS_AES_UNIT_SIZE_4096:
 841                 unit_size = 4096;
 842                 break;
 843
 844         default:
 845                 return -EINVAL;
 846         }
 847
 848         if (xts->key_len != AES_KEYSIZE_128)
 849                 return -EINVAL;
 850
 851         if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
 852                 return -EINVAL;
 853
 854         if (xts->iv_len != AES_BLOCK_SIZE)
 855                 return -EINVAL;
 856
 857         if (!xts->key || !xts->iv || !xts->src || !xts->dst)
 858                 return -EINVAL;
 859
 860         BUILD_BUG_ON(CCP_XTS_AES_KEY_KSB_COUNT != 1);
 861         BUILD_BUG_ON(CCP_XTS_AES_CTX_KSB_COUNT != 1);
 862
 863         ret = -EIO;
 864         memset(&op, 0, sizeof(op));
 865         op.cmd_q = cmd_q;
 866         op.jobid = ccp_gen_jobid(cmd_q->ccp);
 867         op.ksb_key = cmd_q->ksb_key;
 868         op.ksb_ctx = cmd_q->ksb_ctx;
 869         op.init = 1;
 870         op.u.xts.action = xts->action;
 871         op.u.xts.unit_size = xts->unit_size;
 872
 873         /* All supported key sizes fit in a single (32-byte) KSB entry
 874          * and must be in little endian format. Use the 256-bit byte
 875          * swap passthru option to convert from big endian to little
 876          * endian.
 877          */
 878         ret = ccp_init_dm_workarea(&key, cmd_q,
 879                                    CCP_XTS_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
 880                                    DMA_TO_DEVICE);
 881         if (ret)
 882                 return ret;
 883
 884         dm_offset = CCP_KSB_BYTES - AES_KEYSIZE_128;
 885         ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
 886         ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len);
 887         ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
 888                               CCP_PASSTHRU_BYTESWAP_256BIT);
 889         if (ret) {
 890                 cmd->engine_error = cmd_q->cmd_error;
 891                 goto e_key;
 892         }
 893
 894         /* The AES context fits in a single (32-byte) KSB entry and
 895          * for XTS is already in little endian format so no byte swapping
 896          * is needed.
 897          */
 898         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 899                                    CCP_XTS_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
 900                                    DMA_BIDIRECTIONAL);
 901         if (ret)
 902                 goto e_key;
 903
 904         ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
 905         ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
 906                               CCP_PASSTHRU_BYTESWAP_NOOP);
 907         if (ret) {
 908                 cmd->engine_error = cmd_q->cmd_error;
 909                 goto e_ctx;
 910         }
 911
 912         /* Prepare the input and output data workareas. For in-place
 913          * operations we need to set the dma direction to BIDIRECTIONAL
 914          * and copy the src workarea to the dst workarea.
 915          */
 916         if (sg_virt(xts->src) == sg_virt(xts->dst))
 917                 in_place = true;
 918
 919         ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
 920                             unit_size,
 921                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
 922         if (ret)
 923                 goto e_ctx;
 924
 925         if (in_place) {
 926                 dst = src;
 927         } else {
 928                 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
 929                                     unit_size, DMA_FROM_DEVICE);
 930                 if (ret)
 931                         goto e_src;
 932         }
 933
 934         /* Send data to the CCP AES engine */
 935         while (src.sg_wa.bytes_left) {
 936                 ccp_prepare_data(&src, &dst, &op, unit_size, true);
 937                 if (!src.sg_wa.bytes_left)
 938                         op.eom = 1;
 939
 940                 ret = cmd_q->ccp->vdata->perform->perform_xts_aes(&op);
 941                 if (ret) {
 942                         cmd->engine_error = cmd_q->cmd_error;
 943                         goto e_dst;
 944                 }
 945
 946                 ccp_process_data(&src, &dst, &op);
 947         }
 948
 949         /* Retrieve the AES context - convert from LE to BE using
 950          * 32-byte (256-bit) byteswapping
 951          */
 952         ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
 953                                 CCP_PASSTHRU_BYTESWAP_256BIT);
 954         if (ret) {
 955                 cmd->engine_error = cmd_q->cmd_error;
 956                 goto e_dst;
 957         }
 958
 959         /* ...but we only need AES_BLOCK_SIZE bytes */
 960         dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
 961         ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
 962
 963 e_dst:
 964         if (!in_place)
 965                 ccp_free_data(&dst, cmd_q);
 966
 967 e_src:
 968         ccp_free_data(&src, cmd_q);
 969
 970 e_ctx:
 971         ccp_dm_free(&ctx);
 972
 973 e_key:
 974         ccp_dm_free(&key);
 975
 976         return ret;
 977 }
 978
 979 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 980 {
 981         struct ccp_sha_engine *sha = &cmd->u.sha;
 982         struct ccp_dm_workarea ctx;
 983         struct ccp_data src;
 984         struct ccp_op op;
 985         int ret;
 986
 987         if (sha->ctx_len != CCP_SHA_CTXSIZE)
 988                 return -EINVAL;
 989
 990         if (!sha->ctx)
 991                 return -EINVAL;
 992
 993         if (!sha->final && (sha->src_len & (CCP_SHA_BLOCKSIZE - 1)))
 994                 return -EINVAL;
 995
 996         if (!sha->src_len) {
 997                 const u8 *sha_zero;
 998
 999                 /* Not final, just return */
1000                 if (!sha->final)
1001                         return 0;
1002
1003                 /* CCP can't do a zero length sha operation so the caller
1004                  * must buffer the data.
1005                  */
1006                 if (sha->msg_bits)
1007                         return -EINVAL;
1008
1009                 /* The CCP cannot perform zero-length sha operations so the
1010                  * caller is required to buffer data for the final operation.
1011                  * However, a sha operation for a message with a total length
1012                  * of zero is valid so known values are required to supply
1013                  * the result.
1014                  */
1015                 switch (sha->type) {
1016                 case CCP_SHA_TYPE_1:
1017                         sha_zero = sha1_zero_message_hash;
1018                         break;
1019                 case CCP_SHA_TYPE_224:
1020                         sha_zero = sha224_zero_message_hash;
1021                         break;
1022                 case CCP_SHA_TYPE_256:
1023                         sha_zero = sha256_zero_message_hash;
1024                         break;
1025                 default:
1026                         return -EINVAL;
1027                 }
1028
1029                 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1030                                          sha->ctx_len, 1);
1031
1032                 return 0;
1033         }
1034
1035         if (!sha->src)
1036                 return -EINVAL;
1037
1038         BUILD_BUG_ON(CCP_SHA_KSB_COUNT != 1);
1039
1040         memset(&op, 0, sizeof(op));
1041         op.cmd_q = cmd_q;
1042         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1043         op.ksb_ctx = cmd_q->ksb_ctx;
1044         op.u.sha.type = sha->type;
1045         op.u.sha.msg_bits = sha->msg_bits;
1046
1047         /* The SHA context fits in a single (32-byte) KSB entry and
1048          * must be in little endian format. Use the 256-bit byte swap
1049          * passthru option to convert from big endian to little endian.
1050          */
1051         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1052                                    CCP_SHA_KSB_COUNT * CCP_KSB_BYTES,
1053                                    DMA_BIDIRECTIONAL);
1054         if (ret)
1055                 return ret;
1056
1057         if (sha->first) {
1058                 const __be32 *init;
1059
1060                 switch (sha->type) {
1061                 case CCP_SHA_TYPE_1:
1062                         init = ccp_sha1_init;
1063                         break;
1064                 case CCP_SHA_TYPE_224:
1065                         init = ccp_sha224_init;
1066                         break;
1067                 case CCP_SHA_TYPE_256:
1068                         init = ccp_sha256_init;
1069                         break;
1070                 default:
1071                         ret = -EINVAL;
1072                         goto e_ctx;
1073                 }
1074                 memcpy(ctx.address, init, CCP_SHA_CTXSIZE);
1075         } else {
1076                 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1077         }
1078
1079         ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1080                               CCP_PASSTHRU_BYTESWAP_256BIT);
1081         if (ret) {
1082                 cmd->engine_error = cmd_q->cmd_error;
1083                 goto e_ctx;
1084         }
1085
1086         /* Send data to the CCP SHA engine */
1087         ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1088                             CCP_SHA_BLOCKSIZE, DMA_TO_DEVICE);
1089         if (ret)
1090                 goto e_ctx;
1091
1092         while (src.sg_wa.bytes_left) {
1093                 ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false);
1094                 if (sha->final && !src.sg_wa.bytes_left)
1095                         op.eom = 1;
1096
1097                 ret = cmd_q->ccp->vdata->perform->perform_sha(&op);
1098                 if (ret) {
1099                         cmd->engine_error = cmd_q->cmd_error;
1100                         goto e_data;
1101                 }
1102
1103                 ccp_process_data(&src, NULL, &op);
1104         }
1105
1106         /* Retrieve the SHA context - convert from LE to BE using
1107          * 32-byte (256-bit) byteswapping to BE
1108          */
1109         ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1110                                 CCP_PASSTHRU_BYTESWAP_256BIT);
1111         if (ret) {
1112                 cmd->engine_error = cmd_q->cmd_error;
1113                 goto e_data;
1114         }
1115
1116         ccp_get_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1117
1118         if (sha->final && sha->opad) {
1119                 /* HMAC operation, recursively perform final SHA */
1120                 struct ccp_cmd hmac_cmd;
1121                 struct scatterlist sg;
1122                 u64 block_size, digest_size;
1123                 u8 *hmac_buf;
1124
1125                 switch (sha->type) {
1126                 case CCP_SHA_TYPE_1:
1127                         block_size = SHA1_BLOCK_SIZE;
1128                         digest_size = SHA1_DIGEST_SIZE;
1129                         break;
1130                 case CCP_SHA_TYPE_224:
1131                         block_size = SHA224_BLOCK_SIZE;
1132                         digest_size = SHA224_DIGEST_SIZE;
1133                         break;
1134                 case CCP_SHA_TYPE_256:
1135                         block_size = SHA256_BLOCK_SIZE;
1136                         digest_size = SHA256_DIGEST_SIZE;
1137                         break;
1138                 default:
1139                         ret = -EINVAL;
1140                         goto e_data;
1141                 }
1142
1143                 if (sha->opad_len != block_size) {
1144                         ret = -EINVAL;
1145                         goto e_data;
1146                 }
1147
1148                 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1149                 if (!hmac_buf) {
1150                         ret = -ENOMEM;
1151                         goto e_data;
1152                 }
1153                 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1154
1155                 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1156                 memcpy(hmac_buf + block_size, ctx.address, digest_size);
1157
1158                 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1159                 hmac_cmd.engine = CCP_ENGINE_SHA;
1160                 hmac_cmd.u.sha.type = sha->type;
1161                 hmac_cmd.u.sha.ctx = sha->ctx;
1162                 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1163                 hmac_cmd.u.sha.src = &sg;
1164                 hmac_cmd.u.sha.src_len = block_size + digest_size;
1165                 hmac_cmd.u.sha.opad = NULL;
1166                 hmac_cmd.u.sha.opad_len = 0;
1167                 hmac_cmd.u.sha.first = 1;
1168                 hmac_cmd.u.sha.final = 1;
1169                 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1170
1171                 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1172                 if (ret)
1173                         cmd->engine_error = hmac_cmd.engine_error;
1174
1175                 kfree(hmac_buf);
1176         }
1177
1178 e_data:
1179         ccp_free_data(&src, cmd_q);
1180
1181 e_ctx:
1182         ccp_dm_free(&ctx);
1183
1184         return ret;
1185 }
1186
1187 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1188 {
1189         struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1190         struct ccp_dm_workarea exp, src;
1191         struct ccp_data dst;
1192         struct ccp_op op;
1193         unsigned int ksb_count, i_len, o_len;
1194         int ret;
1195
1196         if (rsa->key_size > CCP_RSA_MAX_WIDTH)
1197                 return -EINVAL;
1198
1199         if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1200                 return -EINVAL;
1201
1202         /* The RSA modulus must precede the message being acted upon, so
1203          * it must be copied to a DMA area where the message and the
1204          * modulus can be concatenated.  Therefore the input buffer
1205          * length required is twice the output buffer length (which
1206          * must be a multiple of 256-bits).
1207          */
1208         o_len = ((rsa->key_size + 255) / 256) * 32;
1209         i_len = o_len * 2;
1210
1211         ksb_count = o_len / CCP_KSB_BYTES;
1212
1213         memset(&op, 0, sizeof(op));
1214         op.cmd_q = cmd_q;
1215         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1216         op.ksb_key = ccp_alloc_ksb(cmd_q->ccp, ksb_count);
1217         if (!op.ksb_key)
1218                 return -EIO;
1219
1220         /* The RSA exponent may span multiple (32-byte) KSB entries and must
1221          * be in little endian format. Reverse copy each 32-byte chunk
1222          * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
1223          * and each byte within that chunk and do not perform any byte swap
1224          * operations on the passthru operation.
1225          */
1226         ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1227         if (ret)
1228                 goto e_ksb;
1229
1230         ret = ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len,
1231                                       CCP_KSB_BYTES, false);
1232         if (ret)
1233                 goto e_exp;
1234         ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key,
1235                               CCP_PASSTHRU_BYTESWAP_NOOP);
1236         if (ret) {
1237                 cmd->engine_error = cmd_q->cmd_error;
1238                 goto e_exp;
1239         }
1240
1241         /* Concatenate the modulus and the message. Both the modulus and
1242          * the operands must be in little endian format.  Since the input
1243          * is in big endian format it must be converted.
1244          */
1245         ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1246         if (ret)
1247                 goto e_exp;
1248
1249         ret = ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len,
1250                                       CCP_KSB_BYTES, false);
1251         if (ret)
1252                 goto e_src;
1253         src.address += o_len;   /* Adjust the address for the copy operation */
1254         ret = ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len,
1255                                       CCP_KSB_BYTES, false);
1256         if (ret)
1257                 goto e_src;
1258         src.address -= o_len;   /* Reset the address to original value */
1259
1260         /* Prepare the output area for the operation */
1261         ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,
1262                             o_len, DMA_FROM_DEVICE);
1263         if (ret)
1264                 goto e_src;
1265
1266         op.soc = 1;
1267         op.src.u.dma.address = src.dma.address;
1268         op.src.u.dma.offset = 0;
1269         op.src.u.dma.length = i_len;
1270         op.dst.u.dma.address = dst.dm_wa.dma.address;
1271         op.dst.u.dma.offset = 0;
1272         op.dst.u.dma.length = o_len;
1273
1274         op.u.rsa.mod_size = rsa->key_size;
1275         op.u.rsa.input_len = i_len;
1276
1277         ret = cmd_q->ccp->vdata->perform->perform_rsa(&op);
1278         if (ret) {
1279                 cmd->engine_error = cmd_q->cmd_error;
1280                 goto e_dst;
1281         }
1282
1283         ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len);
1284
1285 e_dst:
1286         ccp_free_data(&dst, cmd_q);
1287
1288 e_src:
1289         ccp_dm_free(&src);
1290
1291 e_exp:
1292         ccp_dm_free(&exp);
1293
1294 e_ksb:
1295         ccp_free_ksb(cmd_q->ccp, op.ksb_key, ksb_count);
1296
1297         return ret;
1298 }
1299
1300 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1301                                 struct ccp_cmd *cmd)
1302 {
1303         struct ccp_passthru_engine *pt = &cmd->u.passthru;
1304         struct ccp_dm_workarea mask;
1305         struct ccp_data src, dst;
1306         struct ccp_op op;
1307         bool in_place = false;
1308         unsigned int i;
1309         int ret;
1310
1311         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1312                 return -EINVAL;
1313
1314         if (!pt->src || !pt->dst)
1315                 return -EINVAL;
1316
1317         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1318                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1319                         return -EINVAL;
1320                 if (!pt->mask)
1321                         return -EINVAL;
1322         }
1323
1324         BUILD_BUG_ON(CCP_PASSTHRU_KSB_COUNT != 1);
1325
1326         memset(&op, 0, sizeof(op));
1327         op.cmd_q = cmd_q;
1328         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1329
1330         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1331                 /* Load the mask */
1332                 op.ksb_key = cmd_q->ksb_key;
1333
1334                 ret = ccp_init_dm_workarea(&mask, cmd_q,
1335                                            CCP_PASSTHRU_KSB_COUNT *
1336                                            CCP_KSB_BYTES,
1337                                            DMA_TO_DEVICE);
1338                 if (ret)
1339                         return ret;
1340
1341                 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1342                 ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key,
1343                                       CCP_PASSTHRU_BYTESWAP_NOOP);
1344                 if (ret) {
1345                         cmd->engine_error = cmd_q->cmd_error;
1346                         goto e_mask;
1347                 }
1348         }
1349
1350         /* Prepare the input and output data workareas. For in-place
1351          * operations we need to set the dma direction to BIDIRECTIONAL
1352          * and copy the src workarea to the dst workarea.
1353          */
1354         if (sg_virt(pt->src) == sg_virt(pt->dst))
1355                 in_place = true;
1356
1357         ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1358                             CCP_PASSTHRU_MASKSIZE,
1359                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1360         if (ret)
1361                 goto e_mask;
1362
1363         if (in_place) {
1364                 dst = src;
1365         } else {
1366                 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1367                                     CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1368                 if (ret)
1369                         goto e_src;
1370         }
1371
1372         /* Send data to the CCP Passthru engine
1373          *   Because the CCP engine works on a single source and destination
1374          *   dma address at a time, each entry in the source scatterlist
1375          *   (after the dma_map_sg call) must be less than or equal to the
1376          *   (remaining) length in the destination scatterlist entry and the
1377          *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
1378          */
1379         dst.sg_wa.sg_used = 0;
1380         for (i = 1; i <= src.sg_wa.dma_count; i++) {
1381                 if (!dst.sg_wa.sg ||
1382                     (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
1383                         ret = -EINVAL;
1384                         goto e_dst;
1385                 }
1386
1387                 if (i == src.sg_wa.dma_count) {
1388                         op.eom = 1;
1389                         op.soc = 1;
1390                 }
1391
1392                 op.src.type = CCP_MEMTYPE_SYSTEM;
1393                 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
1394                 op.src.u.dma.offset = 0;
1395                 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
1396
1397                 op.dst.type = CCP_MEMTYPE_SYSTEM;
1398                 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
1399                 op.dst.u.dma.offset = dst.sg_wa.sg_used;
1400                 op.dst.u.dma.length = op.src.u.dma.length;
1401
1402                 ret = cmd_q->ccp->vdata->perform->perform_passthru(&op);
1403                 if (ret) {
1404                         cmd->engine_error = cmd_q->cmd_error;
1405                         goto e_dst;
1406                 }
1407
1408                 dst.sg_wa.sg_used += src.sg_wa.sg->length;
1409                 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
1410                         dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
1411                         dst.sg_wa.sg_used = 0;
1412                 }
1413                 src.sg_wa.sg = sg_next(src.sg_wa.sg);
1414         }
1415
1416 e_dst:
1417         if (!in_place)
1418                 ccp_free_data(&dst, cmd_q);
1419
1420 e_src:
1421         ccp_free_data(&src, cmd_q);
1422
1423 e_mask:
1424         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1425                 ccp_dm_free(&mask);
1426
1427         return ret;
1428 }
1429
1430 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1431 {
1432         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1433         struct ccp_dm_workarea src, dst;
1434         struct ccp_op op;
1435         int ret;
1436         u8 *save;
1437
1438         if (!ecc->u.mm.operand_1 ||
1439             (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
1440                 return -EINVAL;
1441
1442         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
1443                 if (!ecc->u.mm.operand_2 ||
1444                     (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
1445                         return -EINVAL;
1446
1447         if (!ecc->u.mm.result ||
1448             (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
1449                 return -EINVAL;
1450
1451         memset(&op, 0, sizeof(op));
1452         op.cmd_q = cmd_q;
1453         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1454
1455         /* Concatenate the modulus and the operands. Both the modulus and
1456          * the operands must be in little endian format.  Since the input
1457          * is in big endian format it must be converted and placed in a
1458          * fixed length buffer.
1459          */
1460         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1461                                    DMA_TO_DEVICE);
1462         if (ret)
1463                 return ret;
1464
1465         /* Save the workarea address since it is updated in order to perform
1466          * the concatenation
1467          */
1468         save = src.address;
1469
1470         /* Copy the ECC modulus */
1471         ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1472                                       CCP_ECC_OPERAND_SIZE, false);
1473         if (ret)
1474                 goto e_src;
1475         src.address += CCP_ECC_OPERAND_SIZE;
1476
1477         /* Copy the first operand */
1478         ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1,
1479                                       ecc->u.mm.operand_1_len,
1480                                       CCP_ECC_OPERAND_SIZE, false);
1481         if (ret)
1482                 goto e_src;
1483         src.address += CCP_ECC_OPERAND_SIZE;
1484
1485         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
1486                 /* Copy the second operand */
1487                 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2,
1488                                               ecc->u.mm.operand_2_len,
1489                                               CCP_ECC_OPERAND_SIZE, false);
1490                 if (ret)
1491                         goto e_src;
1492                 src.address += CCP_ECC_OPERAND_SIZE;
1493         }
1494
1495         /* Restore the workarea address */
1496         src.address = save;
1497
1498         /* Prepare the output area for the operation */
1499         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1500                                    DMA_FROM_DEVICE);
1501         if (ret)
1502                 goto e_src;
1503
1504         op.soc = 1;
1505         op.src.u.dma.address = src.dma.address;
1506         op.src.u.dma.offset = 0;
1507         op.src.u.dma.length = src.length;
1508         op.dst.u.dma.address = dst.dma.address;
1509         op.dst.u.dma.offset = 0;
1510         op.dst.u.dma.length = dst.length;
1511
1512         op.u.ecc.function = cmd->u.ecc.function;
1513
1514         ret = cmd_q->ccp->vdata->perform->perform_ecc(&op);
1515         if (ret) {
1516                 cmd->engine_error = cmd_q->cmd_error;
1517                 goto e_dst;
1518         }
1519
1520         ecc->ecc_result = le16_to_cpup(
1521                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1522         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1523                 ret = -EIO;
1524                 goto e_dst;
1525         }
1526
1527         /* Save the ECC result */
1528         ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES);
1529
1530 e_dst:
1531         ccp_dm_free(&dst);
1532
1533 e_src:
1534         ccp_dm_free(&src);
1535
1536         return ret;
1537 }
1538
1539 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1540 {
1541         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1542         struct ccp_dm_workarea src, dst;
1543         struct ccp_op op;
1544         int ret;
1545         u8 *save;
1546
1547         if (!ecc->u.pm.point_1.x ||
1548             (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
1549             !ecc->u.pm.point_1.y ||
1550             (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
1551                 return -EINVAL;
1552
1553         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1554                 if (!ecc->u.pm.point_2.x ||
1555                     (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
1556                     !ecc->u.pm.point_2.y ||
1557                     (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
1558                         return -EINVAL;
1559         } else {
1560                 if (!ecc->u.pm.domain_a ||
1561                     (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
1562                         return -EINVAL;
1563
1564                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
1565                         if (!ecc->u.pm.scalar ||
1566                             (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
1567                                 return -EINVAL;
1568         }
1569
1570         if (!ecc->u.pm.result.x ||
1571             (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
1572             !ecc->u.pm.result.y ||
1573             (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
1574                 return -EINVAL;
1575
1576         memset(&op, 0, sizeof(op));
1577         op.cmd_q = cmd_q;
1578         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1579
1580         /* Concatenate the modulus and the operands. Both the modulus and
1581          * the operands must be in little endian format.  Since the input
1582          * is in big endian format it must be converted and placed in a
1583          * fixed length buffer.
1584          */
1585         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1586                                    DMA_TO_DEVICE);
1587         if (ret)
1588                 return ret;
1589
1590         /* Save the workarea address since it is updated in order to perform
1591          * the concatenation
1592          */
1593         save = src.address;
1594
1595         /* Copy the ECC modulus */
1596         ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1597                                       CCP_ECC_OPERAND_SIZE, false);
1598         if (ret)
1599                 goto e_src;
1600         src.address += CCP_ECC_OPERAND_SIZE;
1601
1602         /* Copy the first point X and Y coordinate */
1603         ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x,
1604                                       ecc->u.pm.point_1.x_len,
1605                                       CCP_ECC_OPERAND_SIZE, false);
1606         if (ret)
1607                 goto e_src;
1608         src.address += CCP_ECC_OPERAND_SIZE;
1609         ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y,
1610                                       ecc->u.pm.point_1.y_len,
1611                                       CCP_ECC_OPERAND_SIZE, false);
1612         if (ret)
1613                 goto e_src;
1614         src.address += CCP_ECC_OPERAND_SIZE;
1615
1616         /* Set the first point Z coordianate to 1 */
1617         *src.address = 0x01;
1618         src.address += CCP_ECC_OPERAND_SIZE;
1619
1620         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1621                 /* Copy the second point X and Y coordinate */
1622                 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x,
1623                                               ecc->u.pm.point_2.x_len,
1624                                               CCP_ECC_OPERAND_SIZE, false);
1625                 if (ret)
1626                         goto e_src;
1627                 src.address += CCP_ECC_OPERAND_SIZE;
1628                 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y,
1629                                               ecc->u.pm.point_2.y_len,
1630                                               CCP_ECC_OPERAND_SIZE, false);
1631                 if (ret)
1632                         goto e_src;
1633                 src.address += CCP_ECC_OPERAND_SIZE;
1634
1635                 /* Set the second point Z coordianate to 1 */
1636                 *src.address = 0x01;
1637                 src.address += CCP_ECC_OPERAND_SIZE;
1638         } else {
1639                 /* Copy the Domain "a" parameter */
1640                 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a,
1641                                               ecc->u.pm.domain_a_len,
1642                                               CCP_ECC_OPERAND_SIZE, false);
1643                 if (ret)
1644                         goto e_src;
1645                 src.address += CCP_ECC_OPERAND_SIZE;
1646
1647                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
1648                         /* Copy the scalar value */
1649                         ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar,
1650                                                       ecc->u.pm.scalar_len,
1651                                                       CCP_ECC_OPERAND_SIZE,
1652                                                       false);
1653                         if (ret)
1654                                 goto e_src;
1655                         src.address += CCP_ECC_OPERAND_SIZE;
1656                 }
1657         }
1658
1659         /* Restore the workarea address */
1660         src.address = save;
1661
1662         /* Prepare the output area for the operation */
1663         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1664                                    DMA_FROM_DEVICE);
1665         if (ret)
1666                 goto e_src;
1667
1668         op.soc = 1;
1669         op.src.u.dma.address = src.dma.address;
1670         op.src.u.dma.offset = 0;
1671         op.src.u.dma.length = src.length;
1672         op.dst.u.dma.address = dst.dma.address;
1673         op.dst.u.dma.offset = 0;
1674         op.dst.u.dma.length = dst.length;
1675
1676         op.u.ecc.function = cmd->u.ecc.function;
1677
1678         ret = cmd_q->ccp->vdata->perform->perform_ecc(&op);
1679         if (ret) {
1680                 cmd->engine_error = cmd_q->cmd_error;
1681                 goto e_dst;
1682         }
1683
1684         ecc->ecc_result = le16_to_cpup(
1685                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1686         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1687                 ret = -EIO;
1688                 goto e_dst;
1689         }
1690
1691         /* Save the workarea address since it is updated as we walk through
1692          * to copy the point math result
1693          */
1694         save = dst.address;
1695
1696         /* Save the ECC result X and Y coordinates */
1697         ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x,
1698                                 CCP_ECC_MODULUS_BYTES);
1699         dst.address += CCP_ECC_OUTPUT_SIZE;
1700         ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y,
1701                                 CCP_ECC_MODULUS_BYTES);
1702         dst.address += CCP_ECC_OUTPUT_SIZE;
1703
1704         /* Restore the workarea address */
1705         dst.address = save;
1706
1707 e_dst:
1708         ccp_dm_free(&dst);
1709
1710 e_src:
1711         ccp_dm_free(&src);
1712
1713         return ret;
1714 }
1715
1716 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1717 {
1718         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1719
1720         ecc->ecc_result = 0;
1721
1722         if (!ecc->mod ||
1723             (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
1724                 return -EINVAL;
1725
1726         switch (ecc->function) {
1727         case CCP_ECC_FUNCTION_MMUL_384BIT:
1728         case CCP_ECC_FUNCTION_MADD_384BIT:
1729         case CCP_ECC_FUNCTION_MINV_384BIT:
1730                 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
1731
1732         case CCP_ECC_FUNCTION_PADD_384BIT:
1733         case CCP_ECC_FUNCTION_PMUL_384BIT:
1734         case CCP_ECC_FUNCTION_PDBL_384BIT:
1735                 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
1736
1737         default:
1738                 return -EINVAL;
1739         }
1740 }
1741
1742 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1743 {
1744         int ret;
1745
1746         cmd->engine_error = 0;
1747         cmd_q->cmd_error = 0;
1748         cmd_q->int_rcvd = 0;
1749         cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
1750
1751         switch (cmd->engine) {
1752         case CCP_ENGINE_AES:
1753                 ret = ccp_run_aes_cmd(cmd_q, cmd);
1754                 break;
1755         case CCP_ENGINE_XTS_AES_128:
1756                 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
1757                 break;
1758         case CCP_ENGINE_SHA:
1759                 ret = ccp_run_sha_cmd(cmd_q, cmd);
1760                 break;
1761         case CCP_ENGINE_RSA:
1762                 ret = ccp_run_rsa_cmd(cmd_q, cmd);
1763                 break;
1764         case CCP_ENGINE_PASSTHRU:
1765                 ret = ccp_run_passthru_cmd(cmd_q, cmd);
1766                 break;
1767         case CCP_ENGINE_ECC:
1768                 ret = ccp_run_ecc_cmd(cmd_q, cmd);
1769                 break;
1770         default:
1771                 ret = -EINVAL;
1772         }
1773
1774         return ret;
1775 }