drivers/crypto/ux500/cryp/cryp_core.c

   1 /**
   2  * Copyright (C) ST-Ericsson SA 2010
   3  * Author: Shujuan Chen <[email protected]> for ST-Ericsson.
   4  * Author: Joakim Bech <[email protected]> for ST-Ericsson.
   5  * Author: Berne Hebark <[email protected]> for ST-Ericsson.
   6  * Author: Niklas Hernaeus <[email protected]> for ST-Ericsson.
   7  * Author: Jonas Linde <[email protected]> for ST-Ericsson.
   8  * Author: Andreas Westin <[email protected]> for ST-Ericsson.
   9  * License terms: GNU General Public License (GPL) version 2
  10  */
  11
  12 #include <linux/clk.h>
  13 #include <linux/completion.h>
  14 #include <linux/crypto.h>
  15 #include <linux/dmaengine.h>
  16 #include <linux/err.h>
  17 #include <linux/errno.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/io.h>
  20 #include <linux/irqreturn.h>
  21 #include <linux/klist.h>
  22 #include <linux/module.h>
  23 #include <linux/platform_device.h>
  24 #include <linux/regulator/consumer.h>
  25 #include <linux/semaphore.h>
  26 #include <linux/platform_data/dma-ste-dma40.h>
  27
  28 #include <crypto/aes.h>
  29 #include <crypto/algapi.h>
  30 #include <crypto/ctr.h>
  31 #include <crypto/des.h>
  32 #include <crypto/scatterwalk.h>
  33
  34 #include <linux/platform_data/crypto-ux500.h>
  35
  36 #include "cryp_p.h"
  37 #include "cryp.h"
  38
  39 #define CRYP_MAX_KEY_SIZE       32
  40 #define BYTES_PER_WORD          4
  41
  42 static int cryp_mode;
  43 static atomic_t session_id;
  44
  45 static struct stedma40_chan_cfg *mem_to_engine;
  46 static struct stedma40_chan_cfg *engine_to_mem;
  47
  48 /**
  49  * struct cryp_driver_data - data specific to the driver.
  50  *
  51  * @device_list: A list of registered devices to choose from.
  52  * @device_allocation: A semaphore initialized with number of devices.
  53  */
  54 struct cryp_driver_data {
  55         struct klist device_list;
  56         struct semaphore device_allocation;
  57 };
  58
  59 /**
  60  * struct cryp_ctx - Crypto context
  61  * @config: Crypto mode.
  62  * @key[CRYP_MAX_KEY_SIZE]: Key.
  63  * @keylen: Length of key.
  64  * @iv: Pointer to initialization vector.
  65  * @indata: Pointer to indata.
  66  * @outdata: Pointer to outdata.
  67  * @datalen: Length of indata.
  68  * @outlen: Length of outdata.
  69  * @blocksize: Size of blocks.
  70  * @updated: Updated flag.
  71  * @dev_ctx: Device dependent context.
  72  * @device: Pointer to the device.
  73  */
  74 struct cryp_ctx {
  75         struct cryp_config config;
  76         u8 key[CRYP_MAX_KEY_SIZE];
  77         u32 keylen;
  78         u8 *iv;
  79         const u8 *indata;
  80         u8 *outdata;
  81         u32 datalen;
  82         u32 outlen;
  83         u32 blocksize;
  84         u8 updated;
  85         struct cryp_device_context dev_ctx;
  86         struct cryp_device_data *device;
  87         u32 session_id;
  88 };
  89
  90 static struct cryp_driver_data driver_data;
  91
  92 /**
  93  * uint8p_to_uint32_be - 4*uint8 to uint32 big endian
  94  * @in: Data to convert.
  95  */
  96 static inline u32 uint8p_to_uint32_be(u8 *in)
  97 {
  98         u32 *data = (u32 *)in;
  99
 100         return cpu_to_be32p(data);
 101 }
 102
 103 /**
 104  * swap_bits_in_byte - mirror the bits in a byte
 105  * @b: the byte to be mirrored
 106  *
 107  * The bits are swapped the following way:
 108  *  Byte b include bits 0-7, nibble 1 (n1) include bits 0-3 and
 109  *  nibble 2 (n2) bits 4-7.
 110  *
 111  *  Nibble 1 (n1):
 112  *  (The "old" (moved) bit is replaced with a zero)
 113  *  1. Move bit 6 and 7, 4 positions to the left.
 114  *  2. Move bit 3 and 5, 2 positions to the left.
 115  *  3. Move bit 1-4, 1 position to the left.
 116  *
 117  *  Nibble 2 (n2):
 118  *  1. Move bit 0 and 1, 4 positions to the right.
 119  *  2. Move bit 2 and 4, 2 positions to the right.
 120  *  3. Move bit 3-6, 1 position to the right.
 121  *
 122  *  Combine the two nibbles to a complete and swapped byte.
 123  */
 124
 125 static inline u8 swap_bits_in_byte(u8 b)
 126 {
 127 #define R_SHIFT_4_MASK  0xc0 /* Bits 6 and 7, right shift 4 */
 128 #define R_SHIFT_2_MASK  0x28 /* (After right shift 4) Bits 3 and 5,
 129                                   right shift 2 */
 130 #define R_SHIFT_1_MASK  0x1e /* (After right shift 2) Bits 1-4,
 131                                   right shift 1 */
 132 #define L_SHIFT_4_MASK  0x03 /* Bits 0 and 1, left shift 4 */
 133 #define L_SHIFT_2_MASK  0x14 /* (After left shift 4) Bits 2 and 4,
 134                                   left shift 2 */
 135 #define L_SHIFT_1_MASK  0x78 /* (After left shift 1) Bits 3-6,
 136                                   left shift 1 */
 137
 138         u8 n1;
 139         u8 n2;
 140
 141         /* Swap most significant nibble */
 142         /* Right shift 4, bits 6 and 7 */
 143         n1 = ((b  & R_SHIFT_4_MASK) >> 4) | (b  & ~(R_SHIFT_4_MASK >> 4));
 144         /* Right shift 2, bits 3 and 5 */
 145         n1 = ((n1 & R_SHIFT_2_MASK) >> 2) | (n1 & ~(R_SHIFT_2_MASK >> 2));
 146         /* Right shift 1, bits 1-4 */
 147         n1 = (n1  & R_SHIFT_1_MASK) >> 1;
 148
 149         /* Swap least significant nibble */
 150         /* Left shift 4, bits 0 and 1 */
 151         n2 = ((b  & L_SHIFT_4_MASK) << 4) | (b  & ~(L_SHIFT_4_MASK << 4));
 152         /* Left shift 2, bits 2 and 4 */
 153         n2 = ((n2 & L_SHIFT_2_MASK) << 2) | (n2 & ~(L_SHIFT_2_MASK << 2));
 154         /* Left shift 1, bits 3-6 */
 155         n2 = (n2  & L_SHIFT_1_MASK) << 1;
 156
 157         return n1 | n2;
 158 }
 159
 160 static inline void swap_words_in_key_and_bits_in_byte(const u8 *in,
 161                                                       u8 *out, u32 len)
 162 {
 163         unsigned int i = 0;
 164         int j;
 165         int index = 0;
 166
 167         j = len - BYTES_PER_WORD;
 168         while (j >= 0) {
 169                 for (i = 0; i < BYTES_PER_WORD; i++) {
 170                         index = len - j - BYTES_PER_WORD + i;
 171                         out[j + i] =
 172                                 swap_bits_in_byte(in[index]);
 173                 }
 174                 j -= BYTES_PER_WORD;
 175         }
 176 }
 177
 178 static void add_session_id(struct cryp_ctx *ctx)
 179 {
 180         /*
 181          * We never want 0 to be a valid value, since this is the default value
 182          * for the software context.
 183          */
 184         if (unlikely(atomic_inc_and_test(&session_id)))
 185                 atomic_inc(&session_id);
 186
 187         ctx->session_id = atomic_read(&session_id);
 188 }
 189
 190 static irqreturn_t cryp_interrupt_handler(int irq, void *param)
 191 {
 192         struct cryp_ctx *ctx;
 193         int count;
 194         struct cryp_device_data *device_data;
 195
 196         if (param == NULL) {
 197                 BUG_ON(!param);
 198                 return IRQ_HANDLED;
 199         }
 200
 201         /* The device is coming from the one found in hw_crypt_noxts. */
 202         device_data = (struct cryp_device_data *)param;
 203
 204         ctx = device_data->current_ctx;
 205
 206         if (ctx == NULL) {
 207                 BUG_ON(!ctx);
 208                 return IRQ_HANDLED;
 209         }
 210
 211         dev_dbg(ctx->device->dev, "[%s] (len: %d) %s, ", __func__, ctx->outlen,
 212                 cryp_pending_irq_src(device_data, CRYP_IRQ_SRC_OUTPUT_FIFO) ?
 213                 "out" : "in");
 214
 215         if (cryp_pending_irq_src(device_data,
 216                                  CRYP_IRQ_SRC_OUTPUT_FIFO)) {
 217                 if (ctx->outlen / ctx->blocksize > 0) {
 218                         count = ctx->blocksize / 4;
 219
 220                         readsl(&device_data->base->dout, ctx->outdata, count);
 221                         ctx->outdata += count;
 222                         ctx->outlen -= count;
 223
 224                         if (ctx->outlen == 0) {
 225                                 cryp_disable_irq_src(device_data,
 226                                                      CRYP_IRQ_SRC_OUTPUT_FIFO);
 227                         }
 228                 }
 229         } else if (cryp_pending_irq_src(device_data,
 230                                         CRYP_IRQ_SRC_INPUT_FIFO)) {
 231                 if (ctx->datalen / ctx->blocksize > 0) {
 232                         count = ctx->blocksize / 4;
 233
 234                         writesl(&device_data->base->din, ctx->indata, count);
 235
 236                         ctx->indata += count;
 237                         ctx->datalen -= count;
 238
 239                         if (ctx->datalen == 0)
 240                                 cryp_disable_irq_src(device_data,
 241                                                    CRYP_IRQ_SRC_INPUT_FIFO);
 242
 243                         if (ctx->config.algomode == CRYP_ALGO_AES_XTS) {
 244                                 CRYP_PUT_BITS(&device_data->base->cr,
 245                                               CRYP_START_ENABLE,
 246                                               CRYP_CR_START_POS,
 247                                               CRYP_CR_START_MASK);
 248
 249                                 cryp_wait_until_done(device_data);
 250                         }
 251                 }
 252         }
 253
 254         return IRQ_HANDLED;
 255 }
 256
 257 static int mode_is_aes(enum cryp_algo_mode mode)
 258 {
 259         return  CRYP_ALGO_AES_ECB == mode ||
 260                 CRYP_ALGO_AES_CBC == mode ||
 261                 CRYP_ALGO_AES_CTR == mode ||
 262                 CRYP_ALGO_AES_XTS == mode;
 263 }
 264
 265 static int cfg_iv(struct cryp_device_data *device_data, u32 left, u32 right,
 266                   enum cryp_init_vector_index index)
 267 {
 268         struct cryp_init_vector_value vector_value;
 269
 270         dev_dbg(device_data->dev, "[%s]", __func__);
 271
 272         vector_value.init_value_left = left;
 273         vector_value.init_value_right = right;
 274
 275         return cryp_configure_init_vector(device_data,
 276                                           index,
 277                                           vector_value);
 278 }
 279
 280 static int cfg_ivs(struct cryp_device_data *device_data, struct cryp_ctx *ctx)
 281 {
 282         int i;
 283         int status = 0;
 284         int num_of_regs = ctx->blocksize / 8;
 285         u32 iv[AES_BLOCK_SIZE / 4];
 286
 287         dev_dbg(device_data->dev, "[%s]", __func__);
 288
 289         /*
 290          * Since we loop on num_of_regs we need to have a check in case
 291          * someone provides an incorrect blocksize which would force calling
 292          * cfg_iv with i greater than 2 which is an error.
 293          */
 294         if (num_of_regs > 2) {
 295                 dev_err(device_data->dev, "[%s] Incorrect blocksize %d",
 296                         __func__, ctx->blocksize);
 297                 return -EINVAL;
 298         }
 299
 300         for (i = 0; i < ctx->blocksize / 4; i++)
 301                 iv[i] = uint8p_to_uint32_be(ctx->iv + i*4);
 302
 303         for (i = 0; i < num_of_regs; i++) {
 304                 status = cfg_iv(device_data, iv[i*2], iv[i*2+1],
 305                                 (enum cryp_init_vector_index) i);
 306                 if (status != 0)
 307                         return status;
 308         }
 309         return status;
 310 }
 311
 312 static int set_key(struct cryp_device_data *device_data,
 313                    u32 left_key,
 314                    u32 right_key,
 315                    enum cryp_key_reg_index index)
 316 {
 317         struct cryp_key_value key_value;
 318         int cryp_error;
 319
 320         dev_dbg(device_data->dev, "[%s]", __func__);
 321
 322         key_value.key_value_left = left_key;
 323         key_value.key_value_right = right_key;
 324
 325         cryp_error = cryp_configure_key_values(device_data,
 326                                                index,
 327                                                key_value);
 328         if (cryp_error != 0)
 329                 dev_err(device_data->dev, "[%s]: "
 330                         "cryp_configure_key_values() failed!", __func__);
 331
 332         return cryp_error;
 333 }
 334
 335 static int cfg_keys(struct cryp_ctx *ctx)
 336 {
 337         int i;
 338         int num_of_regs = ctx->keylen / 8;
 339         u32 swapped_key[CRYP_MAX_KEY_SIZE / 4];
 340         int cryp_error = 0;
 341
 342         dev_dbg(ctx->device->dev, "[%s]", __func__);
 343
 344         if (mode_is_aes(ctx->config.algomode)) {
 345                 swap_words_in_key_and_bits_in_byte((u8 *)ctx->key,
 346                                                    (u8 *)swapped_key,
 347                                                    ctx->keylen);
 348         } else {
 349                 for (i = 0; i < ctx->keylen / 4; i++)
 350                         swapped_key[i] = uint8p_to_uint32_be(ctx->key + i*4);
 351         }
 352
 353         for (i = 0; i < num_of_regs; i++) {
 354                 cryp_error = set_key(ctx->device,
 355                                      *(((u32 *)swapped_key)+i*2),
 356                                      *(((u32 *)swapped_key)+i*2+1),
 357                                      (enum cryp_key_reg_index) i);
 358
 359                 if (cryp_error != 0) {
 360                         dev_err(ctx->device->dev, "[%s]: set_key() failed!",
 361                                         __func__);
 362                         return cryp_error;
 363                 }
 364         }
 365         return cryp_error;
 366 }
 367
 368 static int cryp_setup_context(struct cryp_ctx *ctx,
 369                               struct cryp_device_data *device_data)
 370 {
 371         u32 control_register = CRYP_CR_DEFAULT;
 372
 373         switch (cryp_mode) {
 374         case CRYP_MODE_INTERRUPT:
 375                 writel_relaxed(CRYP_IMSC_DEFAULT, &device_data->base->imsc);
 376                 break;
 377
 378         case CRYP_MODE_DMA:
 379                 writel_relaxed(CRYP_DMACR_DEFAULT, &device_data->base->dmacr);
 380                 break;
 381
 382         default:
 383                 break;
 384         }
 385
 386         if (ctx->updated == 0) {
 387                 cryp_flush_inoutfifo(device_data);
 388                 if (cfg_keys(ctx) != 0) {
 389                         dev_err(ctx->device->dev, "[%s]: cfg_keys failed!",
 390                                 __func__);
 391                         return -EINVAL;
 392                 }
 393
 394                 if (ctx->iv &&
 395                     CRYP_ALGO_AES_ECB != ctx->config.algomode &&
 396                     CRYP_ALGO_DES_ECB != ctx->config.algomode &&
 397                     CRYP_ALGO_TDES_ECB != ctx->config.algomode) {
 398                         if (cfg_ivs(device_data, ctx) != 0)
 399                                 return -EPERM;
 400                 }
 401
 402                 cryp_set_configuration(device_data, &ctx->config,
 403                                        &control_register);
 404                 add_session_id(ctx);
 405         } else if (ctx->updated == 1 &&
 406                    ctx->session_id != atomic_read(&session_id)) {
 407                 cryp_flush_inoutfifo(device_data);
 408                 cryp_restore_device_context(device_data, &ctx->dev_ctx);
 409
 410                 add_session_id(ctx);
 411                 control_register = ctx->dev_ctx.cr;
 412         } else
 413                 control_register = ctx->dev_ctx.cr;
 414
 415         writel(control_register |
 416                (CRYP_CRYPEN_ENABLE << CRYP_CR_CRYPEN_POS),
 417                &device_data->base->cr);
 418
 419         return 0;
 420 }
 421
 422 static int cryp_get_device_data(struct cryp_ctx *ctx,
 423                                 struct cryp_device_data **device_data)
 424 {
 425         int ret;
 426         struct klist_iter device_iterator;
 427         struct klist_node *device_node;
 428         struct cryp_device_data *local_device_data = NULL;
 429         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 430
 431         /* Wait until a device is available */
 432         ret = down_interruptible(&driver_data.device_allocation);
 433         if (ret)
 434                 return ret;  /* Interrupted */
 435
 436         /* Select a device */
 437         klist_iter_init(&driver_data.device_list, &device_iterator);
 438
 439         device_node = klist_next(&device_iterator);
 440         while (device_node) {
 441                 local_device_data = container_of(device_node,
 442                                            struct cryp_device_data, list_node);
 443                 spin_lock(&local_device_data->ctx_lock);
 444                 /* current_ctx allocates a device, NULL = unallocated */
 445                 if (local_device_data->current_ctx) {
 446                         device_node = klist_next(&device_iterator);
 447                 } else {
 448                         local_device_data->current_ctx = ctx;
 449                         ctx->device = local_device_data;
 450                         spin_unlock(&local_device_data->ctx_lock);
 451                         break;
 452                 }
 453                 spin_unlock(&local_device_data->ctx_lock);
 454         }
 455         klist_iter_exit(&device_iterator);
 456
 457         if (!device_node) {
 458                 /**
 459                  * No free device found.
 460                  * Since we allocated a device with down_interruptible, this
 461                  * should not be able to happen.
 462                  * Number of available devices, which are contained in
 463                  * device_allocation, is therefore decremented by not doing
 464                  * an up(device_allocation).
 465                  */
 466                 return -EBUSY;
 467         }
 468
 469         *device_data = local_device_data;
 470
 471         return 0;
 472 }
 473
 474 static void cryp_dma_setup_channel(struct cryp_device_data *device_data,
 475                                    struct device *dev)
 476 {
 477         struct dma_slave_config mem2cryp = {
 478                 .direction = DMA_MEM_TO_DEV,
 479                 .dst_addr = device_data->phybase + CRYP_DMA_TX_FIFO,
 480                 .dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
 481                 .dst_maxburst = 4,
 482         };
 483         struct dma_slave_config cryp2mem = {
 484                 .direction = DMA_DEV_TO_MEM,
 485                 .src_addr = device_data->phybase + CRYP_DMA_RX_FIFO,
 486                 .src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
 487                 .src_maxburst = 4,
 488         };
 489
 490         dma_cap_zero(device_data->dma.mask);
 491         dma_cap_set(DMA_SLAVE, device_data->dma.mask);
 492
 493         device_data->dma.cfg_mem2cryp = mem_to_engine;
 494         device_data->dma.chan_mem2cryp =
 495                 dma_request_channel(device_data->dma.mask,
 496                                     stedma40_filter,
 497                                     device_data->dma.cfg_mem2cryp);
 498
 499         device_data->dma.cfg_cryp2mem = engine_to_mem;
 500         device_data->dma.chan_cryp2mem =
 501                 dma_request_channel(device_data->dma.mask,
 502                                     stedma40_filter,
 503                                     device_data->dma.cfg_cryp2mem);
 504
 505         dmaengine_slave_config(device_data->dma.chan_mem2cryp, &mem2cryp);
 506         dmaengine_slave_config(device_data->dma.chan_cryp2mem, &cryp2mem);
 507
 508         init_completion(&device_data->dma.cryp_dma_complete);
 509 }
 510
 511 static void cryp_dma_out_callback(void *data)
 512 {
 513         struct cryp_ctx *ctx = (struct cryp_ctx *) data;
 514         dev_dbg(ctx->device->dev, "[%s]: ", __func__);
 515
 516         complete(&ctx->device->dma.cryp_dma_complete);
 517 }
 518
 519 static int cryp_set_dma_transfer(struct cryp_ctx *ctx,
 520                                  struct scatterlist *sg,
 521                                  int len,
 522                                  enum dma_data_direction direction)
 523 {
 524         struct dma_async_tx_descriptor *desc;
 525         struct dma_chan *channel = NULL;
 526         dma_cookie_t cookie;
 527
 528         dev_dbg(ctx->device->dev, "[%s]: ", __func__);
 529
 530         if (unlikely(!IS_ALIGNED((u32)sg, 4))) {
 531                 dev_err(ctx->device->dev, "[%s]: Data in sg list isn't "
 532                         "aligned! Addr: 0x%08x", __func__, (u32)sg);
 533                 return -EFAULT;
 534         }
 535
 536         switch (direction) {
 537         case DMA_TO_DEVICE:
 538                 channel = ctx->device->dma.chan_mem2cryp;
 539                 ctx->device->dma.sg_src = sg;
 540                 ctx->device->dma.sg_src_len = dma_map_sg(channel->device->dev,
 541                                                  ctx->device->dma.sg_src,
 542                                                  ctx->device->dma.nents_src,
 543                                                  direction);
 544
 545                 if (!ctx->device->dma.sg_src_len) {
 546                         dev_dbg(ctx->device->dev,
 547                                 "[%s]: Could not map the sg list (TO_DEVICE)",
 548                                 __func__);
 549                         return -EFAULT;
 550                 }
 551
 552                 dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer "
 553                         "(TO_DEVICE)", __func__);
 554
 555                 desc = dmaengine_prep_slave_sg(channel,
 556                                 ctx->device->dma.sg_src,
 557                                 ctx->device->dma.sg_src_len,
 558                                 direction, DMA_CTRL_ACK);
 559                 break;
 560
 561         case DMA_FROM_DEVICE:
 562                 channel = ctx->device->dma.chan_cryp2mem;
 563                 ctx->device->dma.sg_dst = sg;
 564                 ctx->device->dma.sg_dst_len = dma_map_sg(channel->device->dev,
 565                                                  ctx->device->dma.sg_dst,
 566                                                  ctx->device->dma.nents_dst,
 567                                                  direction);
 568
 569                 if (!ctx->device->dma.sg_dst_len) {
 570                         dev_dbg(ctx->device->dev,
 571                                 "[%s]: Could not map the sg list (FROM_DEVICE)",
 572                                 __func__);
 573                         return -EFAULT;
 574                 }
 575
 576                 dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer "
 577                         "(FROM_DEVICE)", __func__);
 578
 579                 desc = dmaengine_prep_slave_sg(channel,
 580                                 ctx->device->dma.sg_dst,
 581                                 ctx->device->dma.sg_dst_len,
 582                                 direction,
 583                                 DMA_CTRL_ACK |
 584                                 DMA_PREP_INTERRUPT);
 585
 586                 desc->callback = cryp_dma_out_callback;
 587                 desc->callback_param = ctx;
 588                 break;
 589
 590         default:
 591                 dev_dbg(ctx->device->dev, "[%s]: Invalid DMA direction",
 592                         __func__);
 593                 return -EFAULT;
 594         }
 595
 596         cookie = dmaengine_submit(desc);
 597         dma_async_issue_pending(channel);
 598
 599         return 0;
 600 }
 601
 602 static void cryp_dma_done(struct cryp_ctx *ctx)
 603 {
 604         struct dma_chan *chan;
 605
 606         dev_dbg(ctx->device->dev, "[%s]: ", __func__);
 607
 608         chan = ctx->device->dma.chan_mem2cryp;
 609         dmaengine_terminate_all(chan);
 610         dma_unmap_sg(chan->device->dev, ctx->device->dma.sg_src,
 611                      ctx->device->dma.sg_src_len, DMA_TO_DEVICE);
 612
 613         chan = ctx->device->dma.chan_cryp2mem;
 614         dmaengine_terminate_all(chan);
 615         dma_unmap_sg(chan->device->dev, ctx->device->dma.sg_dst,
 616                      ctx->device->dma.sg_dst_len, DMA_FROM_DEVICE);
 617 }
 618
 619 static int cryp_dma_write(struct cryp_ctx *ctx, struct scatterlist *sg,
 620                           int len)
 621 {
 622         int error = cryp_set_dma_transfer(ctx, sg, len, DMA_TO_DEVICE);
 623         dev_dbg(ctx->device->dev, "[%s]: ", __func__);
 624
 625         if (error) {
 626                 dev_dbg(ctx->device->dev, "[%s]: cryp_set_dma_transfer() "
 627                         "failed", __func__);
 628                 return error;
 629         }
 630
 631         return len;
 632 }
 633
 634 static int cryp_dma_read(struct cryp_ctx *ctx, struct scatterlist *sg, int len)
 635 {
 636         int error = cryp_set_dma_transfer(ctx, sg, len, DMA_FROM_DEVICE);
 637         if (error) {
 638                 dev_dbg(ctx->device->dev, "[%s]: cryp_set_dma_transfer() "
 639                         "failed", __func__);
 640                 return error;
 641         }
 642
 643         return len;
 644 }
 645
 646 static void cryp_polling_mode(struct cryp_ctx *ctx,
 647                               struct cryp_device_data *device_data)
 648 {
 649         int len = ctx->blocksize / BYTES_PER_WORD;
 650         int remaining_length = ctx->datalen;
 651         u32 *indata = (u32 *)ctx->indata;
 652         u32 *outdata = (u32 *)ctx->outdata;
 653
 654         while (remaining_length > 0) {
 655                 writesl(&device_data->base->din, indata, len);
 656                 indata += len;
 657                 remaining_length -= (len * BYTES_PER_WORD);
 658                 cryp_wait_until_done(device_data);
 659
 660                 readsl(&device_data->base->dout, outdata, len);
 661                 outdata += len;
 662                 cryp_wait_until_done(device_data);
 663         }
 664 }
 665
 666 static int cryp_disable_power(struct device *dev,
 667                               struct cryp_device_data *device_data,
 668                               bool save_device_context)
 669 {
 670         int ret = 0;
 671
 672         dev_dbg(dev, "[%s]", __func__);
 673
 674         spin_lock(&device_data->power_state_spinlock);
 675         if (!device_data->power_state)
 676                 goto out;
 677
 678         spin_lock(&device_data->ctx_lock);
 679         if (save_device_context && device_data->current_ctx) {
 680                 cryp_save_device_context(device_data,
 681                                 &device_data->current_ctx->dev_ctx,
 682                                 cryp_mode);
 683                 device_data->restore_dev_ctx = true;
 684         }
 685         spin_unlock(&device_data->ctx_lock);
 686
 687         clk_disable(device_data->clk);
 688         ret = regulator_disable(device_data->pwr_regulator);
 689         if (ret)
 690                 dev_err(dev, "[%s]: "
 691                                 "regulator_disable() failed!",
 692                                 __func__);
 693
 694         device_data->power_state = false;
 695
 696 out:
 697         spin_unlock(&device_data->power_state_spinlock);
 698
 699         return ret;
 700 }
 701
 702 static int cryp_enable_power(
 703                 struct device *dev,
 704                 struct cryp_device_data *device_data,
 705                 bool restore_device_context)
 706 {
 707         int ret = 0;
 708
 709         dev_dbg(dev, "[%s]", __func__);
 710
 711         spin_lock(&device_data->power_state_spinlock);
 712         if (!device_data->power_state) {
 713                 ret = regulator_enable(device_data->pwr_regulator);
 714                 if (ret) {
 715                         dev_err(dev, "[%s]: regulator_enable() failed!",
 716                                         __func__);
 717                         goto out;
 718                 }
 719
 720                 ret = clk_enable(device_data->clk);
 721                 if (ret) {
 722                         dev_err(dev, "[%s]: clk_enable() failed!",
 723                                         __func__);
 724                         regulator_disable(device_data->pwr_regulator);
 725                         goto out;
 726                 }
 727                 device_data->power_state = true;
 728         }
 729
 730         if (device_data->restore_dev_ctx) {
 731                 spin_lock(&device_data->ctx_lock);
 732                 if (restore_device_context && device_data->current_ctx) {
 733                         device_data->restore_dev_ctx = false;
 734                         cryp_restore_device_context(device_data,
 735                                         &device_data->current_ctx->dev_ctx);
 736                 }
 737                 spin_unlock(&device_data->ctx_lock);
 738         }
 739 out:
 740         spin_unlock(&device_data->power_state_spinlock);
 741
 742         return ret;
 743 }
 744
 745 static int hw_crypt_noxts(struct cryp_ctx *ctx,
 746                           struct cryp_device_data *device_data)
 747 {
 748         int ret = 0;
 749
 750         const u8 *indata = ctx->indata;
 751         u8 *outdata = ctx->outdata;
 752         u32 datalen = ctx->datalen;
 753         u32 outlen = datalen;
 754
 755         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 756
 757         ctx->outlen = ctx->datalen;
 758
 759         if (unlikely(!IS_ALIGNED((u32)indata, 4))) {
 760                 pr_debug(DEV_DBG_NAME " [%s]: Data isn't aligned! Addr: "
 761                          "0x%08x", __func__, (u32)indata);
 762                 return -EINVAL;
 763         }
 764
 765         ret = cryp_setup_context(ctx, device_data);
 766
 767         if (ret)
 768                 goto out;
 769
 770         if (cryp_mode == CRYP_MODE_INTERRUPT) {
 771                 cryp_enable_irq_src(device_data, CRYP_IRQ_SRC_INPUT_FIFO |
 772                                     CRYP_IRQ_SRC_OUTPUT_FIFO);
 773
 774                 /*
 775                  * ctx->outlen is decremented in the cryp_interrupt_handler
 776                  * function. We had to add cpu_relax() (barrier) to make sure
 777                  * that gcc didn't optimze away this variable.
 778                  */
 779                 while (ctx->outlen > 0)
 780                         cpu_relax();
 781         } else if (cryp_mode == CRYP_MODE_POLLING ||
 782                    cryp_mode == CRYP_MODE_DMA) {
 783                 /*
 784                  * The reason for having DMA in this if case is that if we are
 785                  * running cryp_mode = 2, then we separate DMA routines for
 786                  * handling cipher/plaintext > blocksize, except when
 787                  * running the normal CRYPTO_ALG_TYPE_CIPHER, then we still use
 788                  * the polling mode. Overhead of doing DMA setup eats up the
 789                  * benefits using it.
 790                  */
 791                 cryp_polling_mode(ctx, device_data);
 792         } else {
 793                 dev_err(ctx->device->dev, "[%s]: Invalid operation mode!",
 794                         __func__);
 795                 ret = -EPERM;
 796                 goto out;
 797         }
 798
 799         cryp_save_device_context(device_data, &ctx->dev_ctx, cryp_mode);
 800         ctx->updated = 1;
 801
 802 out:
 803         ctx->indata = indata;
 804         ctx->outdata = outdata;
 805         ctx->datalen = datalen;
 806         ctx->outlen = outlen;
 807
 808         return ret;
 809 }
 810
 811 static int get_nents(struct scatterlist *sg, int nbytes)
 812 {
 813         int nents = 0;
 814
 815         while (nbytes > 0) {
 816                 nbytes -= sg->length;
 817                 sg = sg_next(sg);
 818                 nents++;
 819         }
 820
 821         return nents;
 822 }
 823
 824 static int ablk_dma_crypt(struct ablkcipher_request *areq)
 825 {
 826         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
 827         struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
 828         struct cryp_device_data *device_data;
 829
 830         int bytes_written = 0;
 831         int bytes_read = 0;
 832         int ret;
 833
 834         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 835
 836         ctx->datalen = areq->nbytes;
 837         ctx->outlen = areq->nbytes;
 838
 839         ret = cryp_get_device_data(ctx, &device_data);
 840         if (ret)
 841                 return ret;
 842
 843         ret = cryp_setup_context(ctx, device_data);
 844         if (ret)
 845                 goto out;
 846
 847         /* We have the device now, so store the nents in the dma struct. */
 848         ctx->device->dma.nents_src = get_nents(areq->src, ctx->datalen);
 849         ctx->device->dma.nents_dst = get_nents(areq->dst, ctx->outlen);
 850
 851         /* Enable DMA in- and output. */
 852         cryp_configure_for_dma(device_data, CRYP_DMA_ENABLE_BOTH_DIRECTIONS);
 853
 854         bytes_written = cryp_dma_write(ctx, areq->src, ctx->datalen);
 855         bytes_read = cryp_dma_read(ctx, areq->dst, bytes_written);
 856
 857         wait_for_completion(&ctx->device->dma.cryp_dma_complete);
 858         cryp_dma_done(ctx);
 859
 860         cryp_save_device_context(device_data, &ctx->dev_ctx, cryp_mode);
 861         ctx->updated = 1;
 862
 863 out:
 864         spin_lock(&device_data->ctx_lock);
 865         device_data->current_ctx = NULL;
 866         ctx->device = NULL;
 867         spin_unlock(&device_data->ctx_lock);
 868
 869         /*
 870          * The down_interruptible part for this semaphore is called in
 871          * cryp_get_device_data.
 872          */
 873         up(&driver_data.device_allocation);
 874
 875         if (unlikely(bytes_written != bytes_read))
 876                 return -EPERM;
 877
 878         return 0;
 879 }
 880
 881 static int ablk_crypt(struct ablkcipher_request *areq)
 882 {
 883         struct ablkcipher_walk walk;
 884         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
 885         struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
 886         struct cryp_device_data *device_data;
 887         unsigned long src_paddr;
 888         unsigned long dst_paddr;
 889         int ret;
 890         int nbytes;
 891
 892         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 893
 894         ret = cryp_get_device_data(ctx, &device_data);
 895         if (ret)
 896                 goto out;
 897
 898         ablkcipher_walk_init(&walk, areq->dst, areq->src, areq->nbytes);
 899         ret = ablkcipher_walk_phys(areq, &walk);
 900
 901         if (ret) {
 902                 pr_err(DEV_DBG_NAME "[%s]: ablkcipher_walk_phys() failed!",
 903                         __func__);
 904                 goto out;
 905         }
 906
 907         while ((nbytes = walk.nbytes) > 0) {
 908                 ctx->iv = walk.iv;
 909                 src_paddr = (page_to_phys(walk.src.page) + walk.src.offset);
 910                 ctx->indata = phys_to_virt(src_paddr);
 911
 912                 dst_paddr = (page_to_phys(walk.dst.page) + walk.dst.offset);
 913                 ctx->outdata = phys_to_virt(dst_paddr);
 914
 915                 ctx->datalen = nbytes - (nbytes % ctx->blocksize);
 916
 917                 ret = hw_crypt_noxts(ctx, device_data);
 918                 if (ret)
 919                         goto out;
 920
 921                 nbytes -= ctx->datalen;
 922                 ret = ablkcipher_walk_done(areq, &walk, nbytes);
 923                 if (ret)
 924                         goto out;
 925         }
 926         ablkcipher_walk_complete(&walk);
 927
 928 out:
 929         /* Release the device */
 930         spin_lock(&device_data->ctx_lock);
 931         device_data->current_ctx = NULL;
 932         ctx->device = NULL;
 933         spin_unlock(&device_data->ctx_lock);
 934
 935         /*
 936          * The down_interruptible part for this semaphore is called in
 937          * cryp_get_device_data.
 938          */
 939         up(&driver_data.device_allocation);
 940
 941         return ret;
 942 }
 943
 944 static int aes_ablkcipher_setkey(struct crypto_ablkcipher *cipher,
 945                                  const u8 *key, unsigned int keylen)
 946 {
 947         struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
 948         u32 *flags = &cipher->base.crt_flags;
 949
 950         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 951
 952         switch (keylen) {
 953         case AES_KEYSIZE_128:
 954                 ctx->config.keysize = CRYP_KEY_SIZE_128;
 955                 break;
 956
 957         case AES_KEYSIZE_192:
 958                 ctx->config.keysize = CRYP_KEY_SIZE_192;
 959                 break;
 960
 961         case AES_KEYSIZE_256:
 962                 ctx->config.keysize = CRYP_KEY_SIZE_256;
 963                 break;
 964
 965         default:
 966                 pr_err(DEV_DBG_NAME "[%s]: Unknown keylen!", __func__);
 967                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 968                 return -EINVAL;
 969         }
 970
 971         memcpy(ctx->key, key, keylen);
 972         ctx->keylen = keylen;
 973
 974         ctx->updated = 0;
 975
 976         return 0;
 977 }
 978
 979 static int des_ablkcipher_setkey(struct crypto_ablkcipher *cipher,
 980                                  const u8 *key, unsigned int keylen)
 981 {
 982         struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
 983         u32 *flags = &cipher->base.crt_flags;
 984         u32 tmp[DES_EXPKEY_WORDS];
 985         int ret;
 986
 987         pr_debug(DEV_DBG_NAME " [%s]", __func__);
 988         if (keylen != DES_KEY_SIZE) {
 989                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 990                 pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_RES_BAD_KEY_LEN",
 991                                 __func__);
 992                 return -EINVAL;
 993         }
 994
 995         ret = des_ekey(tmp, key);
 996         if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
 997                 *flags |= CRYPTO_TFM_RES_WEAK_KEY;
 998                 pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_REQ_WEAK_KEY",
 999                                 __func__);
1000                 return -EINVAL;
1001         }
1002
1003         memcpy(ctx->key, key, keylen);
1004         ctx->keylen = keylen;
1005
1006         ctx->updated = 0;
1007         return 0;
1008 }
1009
1010 static int des3_ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1011                                   const u8 *key, unsigned int keylen)
1012 {
1013         struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1014         u32 *flags = &cipher->base.crt_flags;
1015         const u32 *K = (const u32 *)key;
1016         u32 tmp[DES3_EDE_EXPKEY_WORDS];
1017         int i, ret;
1018
1019         pr_debug(DEV_DBG_NAME " [%s]", __func__);
1020         if (keylen != DES3_EDE_KEY_SIZE) {
1021                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
1022                 pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_RES_BAD_KEY_LEN",
1023                                 __func__);
1024                 return -EINVAL;
1025         }
1026
1027         /* Checking key interdependency for weak key detection. */
1028         if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) ||
1029                                 !((K[2] ^ K[4]) | (K[3] ^ K[5]))) &&
1030                         (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
1031                 *flags |= CRYPTO_TFM_RES_WEAK_KEY;
1032                 pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_REQ_WEAK_KEY",
1033                                 __func__);
1034                 return -EINVAL;
1035         }
1036         for (i = 0; i < 3; i++) {
1037                 ret = des_ekey(tmp, key + i*DES_KEY_SIZE);
1038                 if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
1039                         *flags |= CRYPTO_TFM_RES_WEAK_KEY;
1040                         pr_debug(DEV_DBG_NAME " [%s]: "
1041                                         "CRYPTO_TFM_REQ_WEAK_KEY", __func__);
1042                         return -EINVAL;
1043                 }
1044         }
1045
1046         memcpy(ctx->key, key, keylen);
1047         ctx->keylen = keylen;
1048
1049         ctx->updated = 0;
1050         return 0;
1051 }
1052
1053 static int cryp_blk_encrypt(struct ablkcipher_request *areq)
1054 {
1055         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1056         struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1057
1058         pr_debug(DEV_DBG_NAME " [%s]", __func__);
1059
1060         ctx->config.algodir = CRYP_ALGORITHM_ENCRYPT;
1061
1062         /*
1063          * DMA does not work for DES due to a hw bug */
1064         if (cryp_mode == CRYP_MODE_DMA && mode_is_aes(ctx->config.algomode))
1065                 return ablk_dma_crypt(areq);
1066
1067         /* For everything except DMA, we run the non DMA version. */
1068         return ablk_crypt(areq);
1069 }
1070
1071 static int cryp_blk_decrypt(struct ablkcipher_request *areq)
1072 {
1073         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1074         struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1075
1076         pr_debug(DEV_DBG_NAME " [%s]", __func__);
1077
1078         ctx->config.algodir = CRYP_ALGORITHM_DECRYPT;
1079
1080         /* DMA does not work for DES due to a hw bug */
1081         if (cryp_mode == CRYP_MODE_DMA && mode_is_aes(ctx->config.algomode))
1082                 return ablk_dma_crypt(areq);
1083
1084         /* For everything except DMA, we run the non DMA version. */
1085         return ablk_crypt(areq);
1086 }
1087
1088 struct cryp_algo_template {
1089         enum cryp_algo_mode algomode;
1090         struct crypto_alg crypto;
1091 };
1092
1093 static int cryp_cra_init(struct crypto_tfm *tfm)
1094 {
1095         struct cryp_ctx *ctx = crypto_tfm_ctx(tfm);
1096         struct crypto_alg *alg = tfm->__crt_alg;
1097         struct cryp_algo_template *cryp_alg = container_of(alg,
1098                         struct cryp_algo_template,
1099                         crypto);
1100
1101         ctx->config.algomode = cryp_alg->algomode;
1102         ctx->blocksize = crypto_tfm_alg_blocksize(tfm);
1103
1104         return 0;
1105 }
1106
1107 static struct cryp_algo_template cryp_algs[] = {
1108         {
1109                 .algomode = CRYP_ALGO_AES_ECB,
1110                 .crypto = {
1111                         .cra_name = "aes",
1112                         .cra_driver_name = "aes-ux500",
1113                         .cra_priority = 300,
1114                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1115                                         CRYPTO_ALG_ASYNC,
1116                         .cra_blocksize = AES_BLOCK_SIZE,
1117                         .cra_ctxsize = sizeof(struct cryp_ctx),
1118                         .cra_alignmask = 3,
1119                         .cra_type = &crypto_ablkcipher_type,
1120                         .cra_init = cryp_cra_init,
1121                         .cra_module = THIS_MODULE,
1122                         .cra_u = {
1123                                 .ablkcipher = {
1124                                         .min_keysize = AES_MIN_KEY_SIZE,
1125                                         .max_keysize = AES_MAX_KEY_SIZE,
1126                                         .setkey = aes_ablkcipher_setkey,
1127                                         .encrypt = cryp_blk_encrypt,
1128                                         .decrypt = cryp_blk_decrypt
1129                                 }
1130                         }
1131                 }
1132         },
1133         {
1134                 .algomode = CRYP_ALGO_AES_ECB,
1135                 .crypto = {
1136                         .cra_name = "ecb(aes)",
1137                         .cra_driver_name = "ecb-aes-ux500",
1138                         .cra_priority = 300,
1139                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1140                                         CRYPTO_ALG_ASYNC,
1141                         .cra_blocksize = AES_BLOCK_SIZE,
1142                         .cra_ctxsize = sizeof(struct cryp_ctx),
1143                         .cra_alignmask = 3,
1144                         .cra_type = &crypto_ablkcipher_type,
1145                         .cra_init = cryp_cra_init,
1146                         .cra_module = THIS_MODULE,
1147                         .cra_u = {
1148                                 .ablkcipher = {
1149                                         .min_keysize = AES_MIN_KEY_SIZE,
1150                                         .max_keysize = AES_MAX_KEY_SIZE,
1151                                         .setkey = aes_ablkcipher_setkey,
1152                                         .encrypt = cryp_blk_encrypt,
1153                                         .decrypt = cryp_blk_decrypt,
1154                                 }
1155                         }
1156                 }
1157         },
1158         {
1159                 .algomode = CRYP_ALGO_AES_CBC,
1160                 .crypto = {
1161                         .cra_name = "cbc(aes)",
1162                         .cra_driver_name = "cbc-aes-ux500",
1163                         .cra_priority = 300,
1164                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1165                                         CRYPTO_ALG_ASYNC,
1166                         .cra_blocksize = AES_BLOCK_SIZE,
1167                         .cra_ctxsize = sizeof(struct cryp_ctx),
1168                         .cra_alignmask = 3,
1169                         .cra_type = &crypto_ablkcipher_type,
1170                         .cra_init = cryp_cra_init,
1171                         .cra_module = THIS_MODULE,
1172                         .cra_u = {
1173                                 .ablkcipher = {
1174                                         .min_keysize = AES_MIN_KEY_SIZE,
1175                                         .max_keysize = AES_MAX_KEY_SIZE,
1176                                         .setkey = aes_ablkcipher_setkey,
1177                                         .encrypt = cryp_blk_encrypt,
1178                                         .decrypt = cryp_blk_decrypt,
1179                                         .ivsize = AES_BLOCK_SIZE,
1180                                 }
1181                         }
1182                 }
1183         },
1184         {
1185                 .algomode = CRYP_ALGO_AES_CTR,
1186                 .crypto = {
1187                         .cra_name = "ctr(aes)",
1188                         .cra_driver_name = "ctr-aes-ux500",
1189                         .cra_priority = 300,
1190                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1191                                                 CRYPTO_ALG_ASYNC,
1192                         .cra_blocksize = AES_BLOCK_SIZE,
1193                         .cra_ctxsize = sizeof(struct cryp_ctx),
1194                         .cra_alignmask = 3,
1195                         .cra_type = &crypto_ablkcipher_type,
1196                         .cra_init = cryp_cra_init,
1197                         .cra_module = THIS_MODULE,
1198                         .cra_u = {
1199                                 .ablkcipher = {
1200                                         .min_keysize = AES_MIN_KEY_SIZE,
1201                                         .max_keysize = AES_MAX_KEY_SIZE,
1202                                         .setkey = aes_ablkcipher_setkey,
1203                                         .encrypt = cryp_blk_encrypt,
1204                                         .decrypt = cryp_blk_decrypt,
1205                                         .ivsize = AES_BLOCK_SIZE,
1206                                 }
1207                         }
1208                 }
1209         },
1210         {
1211                 .algomode = CRYP_ALGO_DES_ECB,
1212                 .crypto = {
1213                         .cra_name = "des",
1214                         .cra_driver_name = "des-ux500",
1215                         .cra_priority = 300,
1216                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1217                                                 CRYPTO_ALG_ASYNC,
1218                         .cra_blocksize = DES_BLOCK_SIZE,
1219                         .cra_ctxsize = sizeof(struct cryp_ctx),
1220                         .cra_alignmask = 3,
1221                         .cra_type = &crypto_ablkcipher_type,
1222                         .cra_init = cryp_cra_init,
1223                         .cra_module = THIS_MODULE,
1224                         .cra_u = {
1225                                 .ablkcipher = {
1226                                         .min_keysize = DES_KEY_SIZE,
1227                                         .max_keysize = DES_KEY_SIZE,
1228                                         .setkey = des_ablkcipher_setkey,
1229                                         .encrypt = cryp_blk_encrypt,
1230                                         .decrypt = cryp_blk_decrypt
1231                                 }
1232                         }
1233                 }
1234
1235         },
1236         {
1237                 .algomode = CRYP_ALGO_TDES_ECB,
1238                 .crypto = {
1239                         .cra_name = "des3_ede",
1240                         .cra_driver_name = "des3_ede-ux500",
1241                         .cra_priority = 300,
1242                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1243                                                 CRYPTO_ALG_ASYNC,
1244                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1245                         .cra_ctxsize = sizeof(struct cryp_ctx),
1246                         .cra_alignmask = 3,
1247                         .cra_type = &crypto_ablkcipher_type,
1248                         .cra_init = cryp_cra_init,
1249                         .cra_module = THIS_MODULE,
1250                         .cra_u = {
1251                                 .ablkcipher = {
1252                                         .min_keysize = DES3_EDE_KEY_SIZE,
1253                                         .max_keysize = DES3_EDE_KEY_SIZE,
1254                                         .setkey = des_ablkcipher_setkey,
1255                                         .encrypt = cryp_blk_encrypt,
1256                                         .decrypt = cryp_blk_decrypt
1257                                 }
1258                         }
1259                 }
1260         },
1261         {
1262                 .algomode = CRYP_ALGO_DES_ECB,
1263                 .crypto = {
1264                         .cra_name = "ecb(des)",
1265                         .cra_driver_name = "ecb-des-ux500",
1266                         .cra_priority = 300,
1267                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1268                                         CRYPTO_ALG_ASYNC,
1269                         .cra_blocksize = DES_BLOCK_SIZE,
1270                         .cra_ctxsize = sizeof(struct cryp_ctx),
1271                         .cra_alignmask = 3,
1272                         .cra_type = &crypto_ablkcipher_type,
1273                         .cra_init = cryp_cra_init,
1274                         .cra_module = THIS_MODULE,
1275                         .cra_u = {
1276                                 .ablkcipher = {
1277                                         .min_keysize = DES_KEY_SIZE,
1278                                         .max_keysize = DES_KEY_SIZE,
1279                                         .setkey = des_ablkcipher_setkey,
1280                                         .encrypt = cryp_blk_encrypt,
1281                                         .decrypt = cryp_blk_decrypt,
1282                                 }
1283                         }
1284                 }
1285         },
1286         {
1287                 .algomode = CRYP_ALGO_TDES_ECB,
1288                 .crypto = {
1289                         .cra_name = "ecb(des3_ede)",
1290                         .cra_driver_name = "ecb-des3_ede-ux500",
1291                         .cra_priority = 300,
1292                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1293                                         CRYPTO_ALG_ASYNC,
1294                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1295                         .cra_ctxsize = sizeof(struct cryp_ctx),
1296                         .cra_alignmask = 3,
1297                         .cra_type = &crypto_ablkcipher_type,
1298                         .cra_init = cryp_cra_init,
1299                         .cra_module = THIS_MODULE,
1300                         .cra_u = {
1301                                 .ablkcipher = {
1302                                         .min_keysize = DES3_EDE_KEY_SIZE,
1303                                         .max_keysize = DES3_EDE_KEY_SIZE,
1304                                         .setkey = des3_ablkcipher_setkey,
1305                                         .encrypt = cryp_blk_encrypt,
1306                                         .decrypt = cryp_blk_decrypt,
1307                                 }
1308                         }
1309                 }
1310         },
1311         {
1312                 .algomode = CRYP_ALGO_DES_CBC,
1313                 .crypto = {
1314                         .cra_name = "cbc(des)",
1315                         .cra_driver_name = "cbc-des-ux500",
1316                         .cra_priority = 300,
1317                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1318                                         CRYPTO_ALG_ASYNC,
1319                         .cra_blocksize = DES_BLOCK_SIZE,
1320                         .cra_ctxsize = sizeof(struct cryp_ctx),
1321                         .cra_alignmask = 3,
1322                         .cra_type = &crypto_ablkcipher_type,
1323                         .cra_init = cryp_cra_init,
1324                         .cra_module = THIS_MODULE,
1325                         .cra_u = {
1326                                 .ablkcipher = {
1327                                         .min_keysize = DES_KEY_SIZE,
1328                                         .max_keysize = DES_KEY_SIZE,
1329                                         .setkey = des_ablkcipher_setkey,
1330                                         .encrypt = cryp_blk_encrypt,
1331                                         .decrypt = cryp_blk_decrypt,
1332                                 }
1333                         }
1334                 }
1335         },
1336         {
1337                 .algomode = CRYP_ALGO_TDES_CBC,
1338                 .crypto = {
1339                         .cra_name = "cbc(des3_ede)",
1340                         .cra_driver_name = "cbc-des3_ede-ux500",
1341                         .cra_priority = 300,
1342                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1343                                         CRYPTO_ALG_ASYNC,
1344                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1345                         .cra_ctxsize = sizeof(struct cryp_ctx),
1346                         .cra_alignmask = 3,
1347                         .cra_type = &crypto_ablkcipher_type,
1348                         .cra_init = cryp_cra_init,
1349                         .cra_module = THIS_MODULE,
1350                         .cra_u = {
1351                                 .ablkcipher = {
1352                                         .min_keysize = DES3_EDE_KEY_SIZE,
1353                                         .max_keysize = DES3_EDE_KEY_SIZE,
1354                                         .setkey = des3_ablkcipher_setkey,
1355                                         .encrypt = cryp_blk_encrypt,
1356                                         .decrypt = cryp_blk_decrypt,
1357                                         .ivsize = DES3_EDE_BLOCK_SIZE,
1358                                 }
1359                         }
1360                 }
1361         }
1362 };
1363
1364 /**
1365  * cryp_algs_register_all -
1366  */
1367 static int cryp_algs_register_all(void)
1368 {
1369         int ret;
1370         int i;
1371         int count;
1372
1373         pr_debug("[%s]", __func__);
1374
1375         for (i = 0; i < ARRAY_SIZE(cryp_algs); i++) {
1376                 ret = crypto_register_alg(&cryp_algs[i].crypto);
1377                 if (ret) {
1378                         count = i;
1379                         pr_err("[%s] alg registration failed",
1380                                         cryp_algs[i].crypto.cra_driver_name);
1381                         goto unreg;
1382                 }
1383         }
1384         return 0;
1385 unreg:
1386         for (i = 0; i < count; i++)
1387                 crypto_unregister_alg(&cryp_algs[i].crypto);
1388         return ret;
1389 }
1390
1391 /**
1392  * cryp_algs_unregister_all -
1393  */
1394 static void cryp_algs_unregister_all(void)
1395 {
1396         int i;
1397
1398         pr_debug(DEV_DBG_NAME " [%s]", __func__);
1399
1400         for (i = 0; i < ARRAY_SIZE(cryp_algs); i++)
1401                 crypto_unregister_alg(&cryp_algs[i].crypto);
1402 }
1403
1404 static int ux500_cryp_probe(struct platform_device *pdev)
1405 {
1406         int ret;
1407         int cryp_error = 0;
1408         struct resource *res = NULL;
1409         struct resource *res_irq = NULL;
1410         struct cryp_device_data *device_data;
1411         struct cryp_protection_config prot = {
1412                 .privilege_access = CRYP_STATE_ENABLE
1413         };
1414         struct device *dev = &pdev->dev;
1415
1416         dev_dbg(dev, "[%s]", __func__);
1417         device_data = devm_kzalloc(dev, sizeof(*device_data), GFP_ATOMIC);
1418         if (!device_data) {
1419                 dev_err(dev, "[%s]: kzalloc() failed!", __func__);
1420                 ret = -ENOMEM;
1421                 goto out;
1422         }
1423
1424         device_data->dev = dev;
1425         device_data->current_ctx = NULL;
1426
1427         /* Grab the DMA configuration from platform data. */
1428         mem_to_engine = &((struct cryp_platform_data *)
1429                          dev->platform_data)->mem_to_engine;
1430         engine_to_mem = &((struct cryp_platform_data *)
1431                          dev->platform_data)->engine_to_mem;
1432
1433         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1434         if (!res) {
1435                 dev_err(dev, "[%s]: platform_get_resource() failed",
1436                                 __func__);
1437                 ret = -ENODEV;
1438                 goto out;
1439         }
1440
1441         device_data->phybase = res->start;
1442         device_data->base = devm_ioremap_resource(dev, res);
1443         if (IS_ERR(device_data->base)) {
1444                 dev_err(dev, "[%s]: ioremap failed!", __func__);
1445                 ret = PTR_ERR(device_data->base);
1446                 goto out;
1447         }
1448
1449         spin_lock_init(&device_data->ctx_lock);
1450         spin_lock_init(&device_data->power_state_spinlock);
1451
1452         /* Enable power for CRYP hardware block */
1453         device_data->pwr_regulator = regulator_get(&pdev->dev, "v-ape");
1454         if (IS_ERR(device_data->pwr_regulator)) {
1455                 dev_err(dev, "[%s]: could not get cryp regulator", __func__);
1456                 ret = PTR_ERR(device_data->pwr_regulator);
1457                 device_data->pwr_regulator = NULL;
1458                 goto out;
1459         }
1460
1461         /* Enable the clk for CRYP hardware block */
1462         device_data->clk = devm_clk_get(&pdev->dev, NULL);
1463         if (IS_ERR(device_data->clk)) {
1464                 dev_err(dev, "[%s]: clk_get() failed!", __func__);
1465                 ret = PTR_ERR(device_data->clk);
1466                 goto out_regulator;
1467         }
1468
1469         ret = clk_prepare(device_data->clk);
1470         if (ret) {
1471                 dev_err(dev, "[%s]: clk_prepare() failed!", __func__);
1472                 goto out_regulator;
1473         }
1474
1475         /* Enable device power (and clock) */
1476         ret = cryp_enable_power(device_data->dev, device_data, false);
1477         if (ret) {
1478                 dev_err(dev, "[%s]: cryp_enable_power() failed!", __func__);
1479                 goto out_clk_unprepare;
1480         }
1481
1482         cryp_error = cryp_check(device_data);
1483         if (cryp_error != 0) {
1484                 dev_err(dev, "[%s]: cryp_init() failed!", __func__);
1485                 ret = -EINVAL;
1486                 goto out_power;
1487         }
1488
1489         cryp_error = cryp_configure_protection(device_data, &prot);
1490         if (cryp_error != 0) {
1491                 dev_err(dev, "[%s]: cryp_configure_protection() failed!",
1492                         __func__);
1493                 ret = -EINVAL;
1494                 goto out_power;
1495         }
1496
1497         res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1498         if (!res_irq) {
1499                 dev_err(dev, "[%s]: IORESOURCE_IRQ unavailable",
1500                         __func__);
1501                 ret = -ENODEV;
1502                 goto out_power;
1503         }
1504
1505         ret = devm_request_irq(&pdev->dev, res_irq->start,
1506                                cryp_interrupt_handler, 0, "cryp1", device_data);
1507         if (ret) {
1508                 dev_err(dev, "[%s]: Unable to request IRQ", __func__);
1509                 goto out_power;
1510         }
1511
1512         if (cryp_mode == CRYP_MODE_DMA)
1513                 cryp_dma_setup_channel(device_data, dev);
1514
1515         platform_set_drvdata(pdev, device_data);
1516
1517         /* Put the new device into the device list... */
1518         klist_add_tail(&device_data->list_node, &driver_data.device_list);
1519
1520         /* ... and signal that a new device is available. */
1521         up(&driver_data.device_allocation);
1522
1523         atomic_set(&session_id, 1);
1524
1525         ret = cryp_algs_register_all();
1526         if (ret) {
1527                 dev_err(dev, "[%s]: cryp_algs_register_all() failed!",
1528                         __func__);
1529                 goto out_power;
1530         }
1531
1532         dev_info(dev, "successfully registered\n");
1533
1534         return 0;
1535
1536 out_power:
1537         cryp_disable_power(device_data->dev, device_data, false);
1538
1539 out_clk_unprepare:
1540         clk_unprepare(device_data->clk);
1541
1542 out_regulator:
1543         regulator_put(device_data->pwr_regulator);
1544
1545 out:
1546         return ret;
1547 }
1548
1549 static int ux500_cryp_remove(struct platform_device *pdev)
1550 {
1551         struct cryp_device_data *device_data;
1552
1553         dev_dbg(&pdev->dev, "[%s]", __func__);
1554         device_data = platform_get_drvdata(pdev);
1555         if (!device_data) {
1556                 dev_err(&pdev->dev, "[%s]: platform_get_drvdata() failed!",
1557                         __func__);
1558                 return -ENOMEM;
1559         }
1560
1561         /* Try to decrease the number of available devices. */
1562         if (down_trylock(&driver_data.device_allocation))
1563                 return -EBUSY;
1564
1565         /* Check that the device is free */
1566         spin_lock(&device_data->ctx_lock);
1567         /* current_ctx allocates a device, NULL = unallocated */
1568         if (device_data->current_ctx) {
1569                 /* The device is busy */
1570                 spin_unlock(&device_data->ctx_lock);
1571                 /* Return the device to the pool. */
1572                 up(&driver_data.device_allocation);
1573                 return -EBUSY;
1574         }
1575
1576         spin_unlock(&device_data->ctx_lock);
1577
1578         /* Remove the device from the list */
1579         if (klist_node_attached(&device_data->list_node))
1580                 klist_remove(&device_data->list_node);
1581
1582         /* If this was the last device, remove the services */
1583         if (list_empty(&driver_data.device_list.k_list))
1584                 cryp_algs_unregister_all();
1585
1586         if (cryp_disable_power(&pdev->dev, device_data, false))
1587                 dev_err(&pdev->dev, "[%s]: cryp_disable_power() failed",
1588                         __func__);
1589
1590         clk_unprepare(device_data->clk);
1591         regulator_put(device_data->pwr_regulator);
1592
1593         return 0;
1594 }
1595
1596 static void ux500_cryp_shutdown(struct platform_device *pdev)
1597 {
1598         struct cryp_device_data *device_data;
1599
1600         dev_dbg(&pdev->dev, "[%s]", __func__);
1601
1602         device_data = platform_get_drvdata(pdev);
1603         if (!device_data) {
1604                 dev_err(&pdev->dev, "[%s]: platform_get_drvdata() failed!",
1605                         __func__);
1606                 return;
1607         }
1608
1609         /* Check that the device is free */
1610         spin_lock(&device_data->ctx_lock);
1611         /* current_ctx allocates a device, NULL = unallocated */
1612         if (!device_data->current_ctx) {
1613                 if (down_trylock(&driver_data.device_allocation))
1614                         dev_dbg(&pdev->dev, "[%s]: Cryp still in use!"
1615                                 "Shutting down anyway...", __func__);
1616                 /**
1617                  * (Allocate the device)
1618                  * Need to set this to non-null (dummy) value,
1619                  * to avoid usage if context switching.
1620                  */
1621                 device_data->current_ctx++;
1622         }
1623         spin_unlock(&device_data->ctx_lock);
1624
1625         /* Remove the device from the list */
1626         if (klist_node_attached(&device_data->list_node))
1627                 klist_remove(&device_data->list_node);
1628
1629         /* If this was the last device, remove the services */
1630         if (list_empty(&driver_data.device_list.k_list))
1631                 cryp_algs_unregister_all();
1632
1633         if (cryp_disable_power(&pdev->dev, device_data, false))
1634                 dev_err(&pdev->dev, "[%s]: cryp_disable_power() failed",
1635                         __func__);
1636
1637 }
1638
1639 #ifdef CONFIG_PM_SLEEP
1640 static int ux500_cryp_suspend(struct device *dev)
1641 {
1642         int ret;
1643         struct platform_device *pdev = to_platform_device(dev);
1644         struct cryp_device_data *device_data;
1645         struct resource *res_irq;
1646         struct cryp_ctx *temp_ctx = NULL;
1647
1648         dev_dbg(dev, "[%s]", __func__);
1649
1650         /* Handle state? */
1651         device_data = platform_get_drvdata(pdev);
1652         if (!device_data) {
1653                 dev_err(dev, "[%s]: platform_get_drvdata() failed!", __func__);
1654                 return -ENOMEM;
1655         }
1656
1657         res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1658         if (!res_irq)
1659                 dev_err(dev, "[%s]: IORESOURCE_IRQ, unavailable", __func__);
1660         else
1661                 disable_irq(res_irq->start);
1662
1663         spin_lock(&device_data->ctx_lock);
1664         if (!device_data->current_ctx)
1665                 device_data->current_ctx++;
1666         spin_unlock(&device_data->ctx_lock);
1667
1668         if (device_data->current_ctx == ++temp_ctx) {
1669                 if (down_interruptible(&driver_data.device_allocation))
1670                         dev_dbg(dev, "[%s]: down_interruptible() failed",
1671                                 __func__);
1672                 ret = cryp_disable_power(dev, device_data, false);
1673
1674         } else
1675                 ret = cryp_disable_power(dev, device_data, true);
1676
1677         if (ret)
1678                 dev_err(dev, "[%s]: cryp_disable_power()", __func__);
1679
1680         return ret;
1681 }
1682
1683 static int ux500_cryp_resume(struct device *dev)
1684 {
1685         int ret = 0;
1686         struct platform_device *pdev = to_platform_device(dev);
1687         struct cryp_device_data *device_data;
1688         struct resource *res_irq;
1689         struct cryp_ctx *temp_ctx = NULL;
1690
1691         dev_dbg(dev, "[%s]", __func__);
1692
1693         device_data = platform_get_drvdata(pdev);
1694         if (!device_data) {
1695                 dev_err(dev, "[%s]: platform_get_drvdata() failed!", __func__);
1696                 return -ENOMEM;
1697         }
1698
1699         spin_lock(&device_data->ctx_lock);
1700         if (device_data->current_ctx == ++temp_ctx)
1701                 device_data->current_ctx = NULL;
1702         spin_unlock(&device_data->ctx_lock);
1703
1704
1705         if (!device_data->current_ctx)
1706                 up(&driver_data.device_allocation);
1707         else
1708                 ret = cryp_enable_power(dev, device_data, true);
1709
1710         if (ret)
1711                 dev_err(dev, "[%s]: cryp_enable_power() failed!", __func__);
1712         else {
1713                 res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1714                 if (res_irq)
1715                         enable_irq(res_irq->start);
1716         }
1717
1718         return ret;
1719 }
1720 #endif
1721
1722 static SIMPLE_DEV_PM_OPS(ux500_cryp_pm, ux500_cryp_suspend, ux500_cryp_resume);
1723
1724 static const struct of_device_id ux500_cryp_match[] = {
1725         { .compatible = "stericsson,ux500-cryp" },
1726         { },
1727 };
1728 MODULE_DEVICE_TABLE(of, ux500_cryp_match);
1729
1730 static struct platform_driver cryp_driver = {
1731         .probe  = ux500_cryp_probe,
1732         .remove = ux500_cryp_remove,
1733         .shutdown = ux500_cryp_shutdown,
1734         .driver = {
1735                 .name  = "cryp1",
1736                 .of_match_table = ux500_cryp_match,
1737                 .pm    = &ux500_cryp_pm,
1738         }
1739 };
1740
1741 static int __init ux500_cryp_mod_init(void)
1742 {
1743         pr_debug("[%s] is called!", __func__);
1744         klist_init(&driver_data.device_list, NULL, NULL);
1745         /* Initialize the semaphore to 0 devices (locked state) */
1746         sema_init(&driver_data.device_allocation, 0);
1747         return platform_driver_register(&cryp_driver);
1748 }
1749
1750 static void __exit ux500_cryp_mod_fini(void)
1751 {
1752         pr_debug("[%s] is called!", __func__);
1753         platform_driver_unregister(&cryp_driver);
1754         return;
1755 }
1756
1757 module_init(ux500_cryp_mod_init);
1758 module_exit(ux500_cryp_mod_fini);
1759
1760 module_param(cryp_mode, int, 0);
1761
1762 MODULE_DESCRIPTION("Driver for ST-Ericsson UX500 CRYP crypto engine.");
1763 MODULE_ALIAS_CRYPTO("aes-all");
1764 MODULE_ALIAS_CRYPTO("des-all");
1765
1766 MODULE_LICENSE("GPL");