drivers/crypto/stm32/stm32-cryp.c

   1 /*
   2  * Copyright (C) STMicroelectronics SA 2017
   3  * Author: Fabien Dessenne <[email protected]>
   4  * License terms:  GNU General Public License (GPL), version 2
   5  */
   6
   7 #include <linux/clk.h>
   8 #include <linux/delay.h>
   9 #include <linux/interrupt.h>
  10 #include <linux/iopoll.h>
  11 #include <linux/module.h>
  12 #include <linux/of_device.h>
  13 #include <linux/platform_device.h>
  14 #include <linux/pm_runtime.h>
  15 #include <linux/reset.h>
  16
  17 #include <crypto/aes.h>
  18 #include <crypto/des.h>
  19 #include <crypto/engine.h>
  20 #include <crypto/scatterwalk.h>
  21 #include <crypto/internal/aead.h>
  22
  23 #define DRIVER_NAME             "stm32-cryp"
  24
  25 /* Bit [0] encrypt / decrypt */
  26 #define FLG_ENCRYPT             BIT(0)
  27 /* Bit [8..1] algo & operation mode */
  28 #define FLG_AES                 BIT(1)
  29 #define FLG_DES                 BIT(2)
  30 #define FLG_TDES                BIT(3)
  31 #define FLG_ECB                 BIT(4)
  32 #define FLG_CBC                 BIT(5)
  33 #define FLG_CTR                 BIT(6)
  34 #define FLG_GCM                 BIT(7)
  35 #define FLG_CCM                 BIT(8)
  36 /* Mode mask = bits [15..0] */
  37 #define FLG_MODE_MASK           GENMASK(15, 0)
  38 /* Bit [31..16] status  */
  39 #define FLG_CCM_PADDED_WA       BIT(16)
  40
  41 /* Registers */
  42 #define CRYP_CR                 0x00000000
  43 #define CRYP_SR                 0x00000004
  44 #define CRYP_DIN                0x00000008
  45 #define CRYP_DOUT               0x0000000C
  46 #define CRYP_DMACR              0x00000010
  47 #define CRYP_IMSCR              0x00000014
  48 #define CRYP_RISR               0x00000018
  49 #define CRYP_MISR               0x0000001C
  50 #define CRYP_K0LR               0x00000020
  51 #define CRYP_K0RR               0x00000024
  52 #define CRYP_K1LR               0x00000028
  53 #define CRYP_K1RR               0x0000002C
  54 #define CRYP_K2LR               0x00000030
  55 #define CRYP_K2RR               0x00000034
  56 #define CRYP_K3LR               0x00000038
  57 #define CRYP_K3RR               0x0000003C
  58 #define CRYP_IV0LR              0x00000040
  59 #define CRYP_IV0RR              0x00000044
  60 #define CRYP_IV1LR              0x00000048
  61 #define CRYP_IV1RR              0x0000004C
  62 #define CRYP_CSGCMCCM0R         0x00000050
  63 #define CRYP_CSGCM0R            0x00000070
  64
  65 /* Registers values */
  66 #define CR_DEC_NOT_ENC          0x00000004
  67 #define CR_TDES_ECB             0x00000000
  68 #define CR_TDES_CBC             0x00000008
  69 #define CR_DES_ECB              0x00000010
  70 #define CR_DES_CBC              0x00000018
  71 #define CR_AES_ECB              0x00000020
  72 #define CR_AES_CBC              0x00000028
  73 #define CR_AES_CTR              0x00000030
  74 #define CR_AES_KP               0x00000038
  75 #define CR_AES_GCM              0x00080000
  76 #define CR_AES_CCM              0x00080008
  77 #define CR_AES_UNKNOWN          0xFFFFFFFF
  78 #define CR_ALGO_MASK            0x00080038
  79 #define CR_DATA32               0x00000000
  80 #define CR_DATA16               0x00000040
  81 #define CR_DATA8                0x00000080
  82 #define CR_DATA1                0x000000C0
  83 #define CR_KEY128               0x00000000
  84 #define CR_KEY192               0x00000100
  85 #define CR_KEY256               0x00000200
  86 #define CR_FFLUSH               0x00004000
  87 #define CR_CRYPEN               0x00008000
  88 #define CR_PH_INIT              0x00000000
  89 #define CR_PH_HEADER            0x00010000
  90 #define CR_PH_PAYLOAD           0x00020000
  91 #define CR_PH_FINAL             0x00030000
  92 #define CR_PH_MASK              0x00030000
  93 #define CR_NBPBL_SHIFT          20
  94
  95 #define SR_BUSY                 0x00000010
  96 #define SR_OFNE                 0x00000004
  97
  98 #define IMSCR_IN                BIT(0)
  99 #define IMSCR_OUT               BIT(1)
 100
 101 #define MISR_IN                 BIT(0)
 102 #define MISR_OUT                BIT(1)
 103
 104 /* Misc */
 105 #define AES_BLOCK_32            (AES_BLOCK_SIZE / sizeof(u32))
 106 #define GCM_CTR_INIT            2
 107 #define _walked_in              (cryp->in_walk.offset - cryp->in_sg->offset)
 108 #define _walked_out             (cryp->out_walk.offset - cryp->out_sg->offset)
 109 #define CRYP_AUTOSUSPEND_DELAY  50
 110
 111 struct stm32_cryp_caps {
 112         bool                    swap_final;
 113         bool                    padding_wa;
 114 };
 115
 116 struct stm32_cryp_ctx {
 117         struct crypto_engine_ctx enginectx;
 118         struct stm32_cryp       *cryp;
 119         int                     keylen;
 120         u32                     key[AES_KEYSIZE_256 / sizeof(u32)];
 121         unsigned long           flags;
 122 };
 123
 124 struct stm32_cryp_reqctx {
 125         unsigned long mode;
 126 };
 127
 128 struct stm32_cryp {
 129         struct list_head        list;
 130         struct device           *dev;
 131         void __iomem            *regs;
 132         struct clk              *clk;
 133         unsigned long           flags;
 134         u32                     irq_status;
 135         const struct stm32_cryp_caps *caps;
 136         struct stm32_cryp_ctx   *ctx;
 137
 138         struct crypto_engine    *engine;
 139
 140         struct mutex            lock; /* protects req / areq */
 141         struct ablkcipher_request *req;
 142         struct aead_request     *areq;
 143
 144         size_t                  authsize;
 145         size_t                  hw_blocksize;
 146
 147         size_t                  total_in;
 148         size_t                  total_in_save;
 149         size_t                  total_out;
 150         size_t                  total_out_save;
 151
 152         struct scatterlist      *in_sg;
 153         struct scatterlist      *out_sg;
 154         struct scatterlist      *out_sg_save;
 155
 156         struct scatterlist      in_sgl;
 157         struct scatterlist      out_sgl;
 158         bool                    sgs_copied;
 159
 160         int                     in_sg_len;
 161         int                     out_sg_len;
 162
 163         struct scatter_walk     in_walk;
 164         struct scatter_walk     out_walk;
 165
 166         u32                     last_ctr[4];
 167         u32                     gcm_ctr;
 168 };
 169
 170 struct stm32_cryp_list {
 171         struct list_head        dev_list;
 172         spinlock_t              lock; /* protect dev_list */
 173 };
 174
 175 static struct stm32_cryp_list cryp_list = {
 176         .dev_list = LIST_HEAD_INIT(cryp_list.dev_list),
 177         .lock     = __SPIN_LOCK_UNLOCKED(cryp_list.lock),
 178 };
 179
 180 static inline bool is_aes(struct stm32_cryp *cryp)
 181 {
 182         return cryp->flags & FLG_AES;
 183 }
 184
 185 static inline bool is_des(struct stm32_cryp *cryp)
 186 {
 187         return cryp->flags & FLG_DES;
 188 }
 189
 190 static inline bool is_tdes(struct stm32_cryp *cryp)
 191 {
 192         return cryp->flags & FLG_TDES;
 193 }
 194
 195 static inline bool is_ecb(struct stm32_cryp *cryp)
 196 {
 197         return cryp->flags & FLG_ECB;
 198 }
 199
 200 static inline bool is_cbc(struct stm32_cryp *cryp)
 201 {
 202         return cryp->flags & FLG_CBC;
 203 }
 204
 205 static inline bool is_ctr(struct stm32_cryp *cryp)
 206 {
 207         return cryp->flags & FLG_CTR;
 208 }
 209
 210 static inline bool is_gcm(struct stm32_cryp *cryp)
 211 {
 212         return cryp->flags & FLG_GCM;
 213 }
 214
 215 static inline bool is_ccm(struct stm32_cryp *cryp)
 216 {
 217         return cryp->flags & FLG_CCM;
 218 }
 219
 220 static inline bool is_encrypt(struct stm32_cryp *cryp)
 221 {
 222         return cryp->flags & FLG_ENCRYPT;
 223 }
 224
 225 static inline bool is_decrypt(struct stm32_cryp *cryp)
 226 {
 227         return !is_encrypt(cryp);
 228 }
 229
 230 static inline u32 stm32_cryp_read(struct stm32_cryp *cryp, u32 ofst)
 231 {
 232         return readl_relaxed(cryp->regs + ofst);
 233 }
 234
 235 static inline void stm32_cryp_write(struct stm32_cryp *cryp, u32 ofst, u32 val)
 236 {
 237         writel_relaxed(val, cryp->regs + ofst);
 238 }
 239
 240 static inline int stm32_cryp_wait_busy(struct stm32_cryp *cryp)
 241 {
 242         u32 status;
 243
 244         return readl_relaxed_poll_timeout(cryp->regs + CRYP_SR, status,
 245                         !(status & SR_BUSY), 10, 100000);
 246 }
 247
 248 static inline int stm32_cryp_wait_enable(struct stm32_cryp *cryp)
 249 {
 250         u32 status;
 251
 252         return readl_relaxed_poll_timeout(cryp->regs + CRYP_CR, status,
 253                         !(status & CR_CRYPEN), 10, 100000);
 254 }
 255
 256 static inline int stm32_cryp_wait_output(struct stm32_cryp *cryp)
 257 {
 258         u32 status;
 259
 260         return readl_relaxed_poll_timeout(cryp->regs + CRYP_SR, status,
 261                         status & SR_OFNE, 10, 100000);
 262 }
 263
 264 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp);
 265
 266 static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx)
 267 {
 268         struct stm32_cryp *tmp, *cryp = NULL;
 269
 270         spin_lock_bh(&cryp_list.lock);
 271         if (!ctx->cryp) {
 272                 list_for_each_entry(tmp, &cryp_list.dev_list, list) {
 273                         cryp = tmp;
 274                         break;
 275                 }
 276                 ctx->cryp = cryp;
 277         } else {
 278                 cryp = ctx->cryp;
 279         }
 280
 281         spin_unlock_bh(&cryp_list.lock);
 282
 283         return cryp;
 284 }
 285
 286 static int stm32_cryp_check_aligned(struct scatterlist *sg, size_t total,
 287                                     size_t align)
 288 {
 289         int len = 0;
 290
 291         if (!total)
 292                 return 0;
 293
 294         if (!IS_ALIGNED(total, align))
 295                 return -EINVAL;
 296
 297         while (sg) {
 298                 if (!IS_ALIGNED(sg->offset, sizeof(u32)))
 299                         return -EINVAL;
 300
 301                 if (!IS_ALIGNED(sg->length, align))
 302                         return -EINVAL;
 303
 304                 len += sg->length;
 305                 sg = sg_next(sg);
 306         }
 307
 308         if (len != total)
 309                 return -EINVAL;
 310
 311         return 0;
 312 }
 313
 314 static int stm32_cryp_check_io_aligned(struct stm32_cryp *cryp)
 315 {
 316         int ret;
 317
 318         ret = stm32_cryp_check_aligned(cryp->in_sg, cryp->total_in,
 319                                        cryp->hw_blocksize);
 320         if (ret)
 321                 return ret;
 322
 323         ret = stm32_cryp_check_aligned(cryp->out_sg, cryp->total_out,
 324                                        cryp->hw_blocksize);
 325
 326         return ret;
 327 }
 328
 329 static void sg_copy_buf(void *buf, struct scatterlist *sg,
 330                         unsigned int start, unsigned int nbytes, int out)
 331 {
 332         struct scatter_walk walk;
 333
 334         if (!nbytes)
 335                 return;
 336
 337         scatterwalk_start(&walk, sg);
 338         scatterwalk_advance(&walk, start);
 339         scatterwalk_copychunks(buf, &walk, nbytes, out);
 340         scatterwalk_done(&walk, out, 0);
 341 }
 342
 343 static int stm32_cryp_copy_sgs(struct stm32_cryp *cryp)
 344 {
 345         void *buf_in, *buf_out;
 346         int pages, total_in, total_out;
 347
 348         if (!stm32_cryp_check_io_aligned(cryp)) {
 349                 cryp->sgs_copied = 0;
 350                 return 0;
 351         }
 352
 353         total_in = ALIGN(cryp->total_in, cryp->hw_blocksize);
 354         pages = total_in ? get_order(total_in) : 1;
 355         buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages);
 356
 357         total_out = ALIGN(cryp->total_out, cryp->hw_blocksize);
 358         pages = total_out ? get_order(total_out) : 1;
 359         buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages);
 360
 361         if (!buf_in || !buf_out) {
 362                 dev_err(cryp->dev, "Can't allocate pages when unaligned\n");
 363                 cryp->sgs_copied = 0;
 364                 return -EFAULT;
 365         }
 366
 367         sg_copy_buf(buf_in, cryp->in_sg, 0, cryp->total_in, 0);
 368
 369         sg_init_one(&cryp->in_sgl, buf_in, total_in);
 370         cryp->in_sg = &cryp->in_sgl;
 371         cryp->in_sg_len = 1;
 372
 373         sg_init_one(&cryp->out_sgl, buf_out, total_out);
 374         cryp->out_sg_save = cryp->out_sg;
 375         cryp->out_sg = &cryp->out_sgl;
 376         cryp->out_sg_len = 1;
 377
 378         cryp->sgs_copied = 1;
 379
 380         return 0;
 381 }
 382
 383 static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, u32 *iv)
 384 {
 385         if (!iv)
 386                 return;
 387
 388         stm32_cryp_write(cryp, CRYP_IV0LR, cpu_to_be32(*iv++));
 389         stm32_cryp_write(cryp, CRYP_IV0RR, cpu_to_be32(*iv++));
 390
 391         if (is_aes(cryp)) {
 392                 stm32_cryp_write(cryp, CRYP_IV1LR, cpu_to_be32(*iv++));
 393                 stm32_cryp_write(cryp, CRYP_IV1RR, cpu_to_be32(*iv++));
 394         }
 395 }
 396
 397 static void stm32_cryp_hw_write_key(struct stm32_cryp *c)
 398 {
 399         unsigned int i;
 400         int r_id;
 401
 402         if (is_des(c)) {
 403                 stm32_cryp_write(c, CRYP_K1LR, cpu_to_be32(c->ctx->key[0]));
 404                 stm32_cryp_write(c, CRYP_K1RR, cpu_to_be32(c->ctx->key[1]));
 405         } else {
 406                 r_id = CRYP_K3RR;
 407                 for (i = c->ctx->keylen / sizeof(u32); i > 0; i--, r_id -= 4)
 408                         stm32_cryp_write(c, r_id,
 409                                          cpu_to_be32(c->ctx->key[i - 1]));
 410         }
 411 }
 412
 413 static u32 stm32_cryp_get_hw_mode(struct stm32_cryp *cryp)
 414 {
 415         if (is_aes(cryp) && is_ecb(cryp))
 416                 return CR_AES_ECB;
 417
 418         if (is_aes(cryp) && is_cbc(cryp))
 419                 return CR_AES_CBC;
 420
 421         if (is_aes(cryp) && is_ctr(cryp))
 422                 return CR_AES_CTR;
 423
 424         if (is_aes(cryp) && is_gcm(cryp))
 425                 return CR_AES_GCM;
 426
 427         if (is_aes(cryp) && is_ccm(cryp))
 428                 return CR_AES_CCM;
 429
 430         if (is_des(cryp) && is_ecb(cryp))
 431                 return CR_DES_ECB;
 432
 433         if (is_des(cryp) && is_cbc(cryp))
 434                 return CR_DES_CBC;
 435
 436         if (is_tdes(cryp) && is_ecb(cryp))
 437                 return CR_TDES_ECB;
 438
 439         if (is_tdes(cryp) && is_cbc(cryp))
 440                 return CR_TDES_CBC;
 441
 442         dev_err(cryp->dev, "Unknown mode\n");
 443         return CR_AES_UNKNOWN;
 444 }
 445
 446 static unsigned int stm32_cryp_get_input_text_len(struct stm32_cryp *cryp)
 447 {
 448         return is_encrypt(cryp) ? cryp->areq->cryptlen :
 449                                   cryp->areq->cryptlen - cryp->authsize;
 450 }
 451
 452 static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg)
 453 {
 454         int ret;
 455         u32 iv[4];
 456
 457         /* Phase 1 : init */
 458         memcpy(iv, cryp->areq->iv, 12);
 459         iv[3] = cpu_to_be32(GCM_CTR_INIT);
 460         cryp->gcm_ctr = GCM_CTR_INIT;
 461         stm32_cryp_hw_write_iv(cryp, iv);
 462
 463         stm32_cryp_write(cryp, CRYP_CR, cfg | CR_PH_INIT | CR_CRYPEN);
 464
 465         /* Wait for end of processing */
 466         ret = stm32_cryp_wait_enable(cryp);
 467         if (ret)
 468                 dev_err(cryp->dev, "Timeout (gcm init)\n");
 469
 470         return ret;
 471 }
 472
 473 static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg)
 474 {
 475         int ret;
 476         u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE];
 477         u32 *d;
 478         unsigned int i, textlen;
 479
 480         /* Phase 1 : init. Firstly set the CTR value to 1 (not 0) */
 481         memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE);
 482         memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
 483         iv[AES_BLOCK_SIZE - 1] = 1;
 484         stm32_cryp_hw_write_iv(cryp, (u32 *)iv);
 485
 486         /* Build B0 */
 487         memcpy(b0, iv, AES_BLOCK_SIZE);
 488
 489         b0[0] |= (8 * ((cryp->authsize - 2) / 2));
 490
 491         if (cryp->areq->assoclen)
 492                 b0[0] |= 0x40;
 493
 494         textlen = stm32_cryp_get_input_text_len(cryp);
 495
 496         b0[AES_BLOCK_SIZE - 2] = textlen >> 8;
 497         b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF;
 498
 499         /* Enable HW */
 500         stm32_cryp_write(cryp, CRYP_CR, cfg | CR_PH_INIT | CR_CRYPEN);
 501
 502         /* Write B0 */
 503         d = (u32 *)b0;
 504
 505         for (i = 0; i < AES_BLOCK_32; i++) {
 506                 if (!cryp->caps->padding_wa)
 507                         *d = cpu_to_be32(*d);
 508                 stm32_cryp_write(cryp, CRYP_DIN, *d++);
 509         }
 510
 511         /* Wait for end of processing */
 512         ret = stm32_cryp_wait_enable(cryp);
 513         if (ret)
 514                 dev_err(cryp->dev, "Timeout (ccm init)\n");
 515
 516         return ret;
 517 }
 518
 519 static int stm32_cryp_hw_init(struct stm32_cryp *cryp)
 520 {
 521         int ret;
 522         u32 cfg, hw_mode;
 523
 524         pm_runtime_get_sync(cryp->dev);
 525
 526         /* Disable interrupt */
 527         stm32_cryp_write(cryp, CRYP_IMSCR, 0);
 528
 529         /* Set key */
 530         stm32_cryp_hw_write_key(cryp);
 531
 532         /* Set configuration */
 533         cfg = CR_DATA8 | CR_FFLUSH;
 534
 535         switch (cryp->ctx->keylen) {
 536         case AES_KEYSIZE_128:
 537                 cfg |= CR_KEY128;
 538                 break;
 539
 540         case AES_KEYSIZE_192:
 541                 cfg |= CR_KEY192;
 542                 break;
 543
 544         default:
 545         case AES_KEYSIZE_256:
 546                 cfg |= CR_KEY256;
 547                 break;
 548         }
 549
 550         hw_mode = stm32_cryp_get_hw_mode(cryp);
 551         if (hw_mode == CR_AES_UNKNOWN)
 552                 return -EINVAL;
 553
 554         /* AES ECB/CBC decrypt: run key preparation first */
 555         if (is_decrypt(cryp) &&
 556             ((hw_mode == CR_AES_ECB) || (hw_mode == CR_AES_CBC))) {
 557                 stm32_cryp_write(cryp, CRYP_CR, cfg | CR_AES_KP | CR_CRYPEN);
 558
 559                 /* Wait for end of processing */
 560                 ret = stm32_cryp_wait_busy(cryp);
 561                 if (ret) {
 562                         dev_err(cryp->dev, "Timeout (key preparation)\n");
 563                         return ret;
 564                 }
 565         }
 566
 567         cfg |= hw_mode;
 568
 569         if (is_decrypt(cryp))
 570                 cfg |= CR_DEC_NOT_ENC;
 571
 572         /* Apply config and flush (valid when CRYPEN = 0) */
 573         stm32_cryp_write(cryp, CRYP_CR, cfg);
 574
 575         switch (hw_mode) {
 576         case CR_AES_GCM:
 577         case CR_AES_CCM:
 578                 /* Phase 1 : init */
 579                 if (hw_mode == CR_AES_CCM)
 580                         ret = stm32_cryp_ccm_init(cryp, cfg);
 581                 else
 582                         ret = stm32_cryp_gcm_init(cryp, cfg);
 583
 584                 if (ret)
 585                         return ret;
 586
 587                 /* Phase 2 : header (authenticated data) */
 588                 if (cryp->areq->assoclen) {
 589                         cfg |= CR_PH_HEADER;
 590                 } else if (stm32_cryp_get_input_text_len(cryp)) {
 591                         cfg |= CR_PH_PAYLOAD;
 592                         stm32_cryp_write(cryp, CRYP_CR, cfg);
 593                 } else {
 594                         cfg |= CR_PH_INIT;
 595                 }
 596
 597                 break;
 598
 599         case CR_DES_CBC:
 600         case CR_TDES_CBC:
 601         case CR_AES_CBC:
 602         case CR_AES_CTR:
 603                 stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->req->info);
 604                 break;
 605
 606         default:
 607                 break;
 608         }
 609
 610         /* Enable now */
 611         cfg |= CR_CRYPEN;
 612
 613         stm32_cryp_write(cryp, CRYP_CR, cfg);
 614
 615         cryp->flags &= ~FLG_CCM_PADDED_WA;
 616
 617         return 0;
 618 }
 619
 620 static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err)
 621 {
 622         if (!err && (is_gcm(cryp) || is_ccm(cryp)))
 623                 /* Phase 4 : output tag */
 624                 err = stm32_cryp_read_auth_tag(cryp);
 625
 626         if (cryp->sgs_copied) {
 627                 void *buf_in, *buf_out;
 628                 int pages, len;
 629
 630                 buf_in = sg_virt(&cryp->in_sgl);
 631                 buf_out = sg_virt(&cryp->out_sgl);
 632
 633                 sg_copy_buf(buf_out, cryp->out_sg_save, 0,
 634                             cryp->total_out_save, 1);
 635
 636                 len = ALIGN(cryp->total_in_save, cryp->hw_blocksize);
 637                 pages = len ? get_order(len) : 1;
 638                 free_pages((unsigned long)buf_in, pages);
 639
 640                 len = ALIGN(cryp->total_out_save, cryp->hw_blocksize);
 641                 pages = len ? get_order(len) : 1;
 642                 free_pages((unsigned long)buf_out, pages);
 643         }
 644
 645         pm_runtime_mark_last_busy(cryp->dev);
 646         pm_runtime_put_autosuspend(cryp->dev);
 647
 648         if (is_gcm(cryp) || is_ccm(cryp)) {
 649                 crypto_finalize_aead_request(cryp->engine, cryp->areq, err);
 650                 cryp->areq = NULL;
 651         } else {
 652                 crypto_finalize_ablkcipher_request(cryp->engine, cryp->req,
 653                                                    err);
 654                 cryp->req = NULL;
 655         }
 656
 657         memset(cryp->ctx->key, 0, cryp->ctx->keylen);
 658
 659         mutex_unlock(&cryp->lock);
 660 }
 661
 662 static int stm32_cryp_cpu_start(struct stm32_cryp *cryp)
 663 {
 664         /* Enable interrupt and let the IRQ handler do everything */
 665         stm32_cryp_write(cryp, CRYP_IMSCR, IMSCR_IN | IMSCR_OUT);
 666
 667         return 0;
 668 }
 669
 670 static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq);
 671 static int stm32_cryp_prepare_cipher_req(struct crypto_engine *engine,
 672                                          void *areq);
 673
 674 static int stm32_cryp_cra_init(struct crypto_tfm *tfm)
 675 {
 676         struct stm32_cryp_ctx *ctx = crypto_tfm_ctx(tfm);
 677
 678         tfm->crt_ablkcipher.reqsize = sizeof(struct stm32_cryp_reqctx);
 679
 680         ctx->enginectx.op.do_one_request = stm32_cryp_cipher_one_req;
 681         ctx->enginectx.op.prepare_request = stm32_cryp_prepare_cipher_req;
 682         ctx->enginectx.op.unprepare_request = NULL;
 683         return 0;
 684 }
 685
 686 static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq);
 687 static int stm32_cryp_prepare_aead_req(struct crypto_engine *engine,
 688                                        void *areq);
 689
 690 static int stm32_cryp_aes_aead_init(struct crypto_aead *tfm)
 691 {
 692         struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm);
 693
 694         tfm->reqsize = sizeof(struct stm32_cryp_reqctx);
 695
 696         ctx->enginectx.op.do_one_request = stm32_cryp_aead_one_req;
 697         ctx->enginectx.op.prepare_request = stm32_cryp_prepare_aead_req;
 698         ctx->enginectx.op.unprepare_request = NULL;
 699
 700         return 0;
 701 }
 702
 703 static int stm32_cryp_crypt(struct ablkcipher_request *req, unsigned long mode)
 704 {
 705         struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx(
 706                         crypto_ablkcipher_reqtfm(req));
 707         struct stm32_cryp_reqctx *rctx = ablkcipher_request_ctx(req);
 708         struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx);
 709
 710         if (!cryp)
 711                 return -ENODEV;
 712
 713         rctx->mode = mode;
 714
 715         return crypto_transfer_ablkcipher_request_to_engine(cryp->engine, req);
 716 }
 717
 718 static int stm32_cryp_aead_crypt(struct aead_request *req, unsigned long mode)
 719 {
 720         struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 721         struct stm32_cryp_reqctx *rctx = aead_request_ctx(req);
 722         struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx);
 723
 724         if (!cryp)
 725                 return -ENODEV;
 726
 727         rctx->mode = mode;
 728
 729         return crypto_transfer_aead_request_to_engine(cryp->engine, req);
 730 }
 731
 732 static int stm32_cryp_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 733                              unsigned int keylen)
 734 {
 735         struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 736
 737         memcpy(ctx->key, key, keylen);
 738         ctx->keylen = keylen;
 739
 740         return 0;
 741 }
 742
 743 static int stm32_cryp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 744                                  unsigned int keylen)
 745 {
 746         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 747             keylen != AES_KEYSIZE_256)
 748                 return -EINVAL;
 749         else
 750                 return stm32_cryp_setkey(tfm, key, keylen);
 751 }
 752
 753 static int stm32_cryp_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 754                                  unsigned int keylen)
 755 {
 756         if (keylen != DES_KEY_SIZE)
 757                 return -EINVAL;
 758         else
 759                 return stm32_cryp_setkey(tfm, key, keylen);
 760 }
 761
 762 static int stm32_cryp_tdes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 763                                   unsigned int keylen)
 764 {
 765         if (keylen != (3 * DES_KEY_SIZE))
 766                 return -EINVAL;
 767         else
 768                 return stm32_cryp_setkey(tfm, key, keylen);
 769 }
 770
 771 static int stm32_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key,
 772                                       unsigned int keylen)
 773 {
 774         struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm);
 775
 776         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 777             keylen != AES_KEYSIZE_256)
 778                 return -EINVAL;
 779
 780         memcpy(ctx->key, key, keylen);
 781         ctx->keylen = keylen;
 782
 783         return 0;
 784 }
 785
 786 static int stm32_cryp_aes_gcm_setauthsize(struct crypto_aead *tfm,
 787                                           unsigned int authsize)
 788 {
 789         return authsize == AES_BLOCK_SIZE ? 0 : -EINVAL;
 790 }
 791
 792 static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm,
 793                                           unsigned int authsize)
 794 {
 795         switch (authsize) {
 796         case 4:
 797         case 6:
 798         case 8:
 799         case 10:
 800         case 12:
 801         case 14:
 802         case 16:
 803                 break;
 804         default:
 805                 return -EINVAL;
 806         }
 807
 808         return 0;
 809 }
 810
 811 static int stm32_cryp_aes_ecb_encrypt(struct ablkcipher_request *req)
 812 {
 813         return stm32_cryp_crypt(req, FLG_AES | FLG_ECB | FLG_ENCRYPT);
 814 }
 815
 816 static int stm32_cryp_aes_ecb_decrypt(struct ablkcipher_request *req)
 817 {
 818         return stm32_cryp_crypt(req, FLG_AES | FLG_ECB);
 819 }
 820
 821 static int stm32_cryp_aes_cbc_encrypt(struct ablkcipher_request *req)
 822 {
 823         return stm32_cryp_crypt(req, FLG_AES | FLG_CBC | FLG_ENCRYPT);
 824 }
 825
 826 static int stm32_cryp_aes_cbc_decrypt(struct ablkcipher_request *req)
 827 {
 828         return stm32_cryp_crypt(req, FLG_AES | FLG_CBC);
 829 }
 830
 831 static int stm32_cryp_aes_ctr_encrypt(struct ablkcipher_request *req)
 832 {
 833         return stm32_cryp_crypt(req, FLG_AES | FLG_CTR | FLG_ENCRYPT);
 834 }
 835
 836 static int stm32_cryp_aes_ctr_decrypt(struct ablkcipher_request *req)
 837 {
 838         return stm32_cryp_crypt(req, FLG_AES | FLG_CTR);
 839 }
 840
 841 static int stm32_cryp_aes_gcm_encrypt(struct aead_request *req)
 842 {
 843         return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM | FLG_ENCRYPT);
 844 }
 845
 846 static int stm32_cryp_aes_gcm_decrypt(struct aead_request *req)
 847 {
 848         return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM);
 849 }
 850
 851 static int stm32_cryp_aes_ccm_encrypt(struct aead_request *req)
 852 {
 853         return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM | FLG_ENCRYPT);
 854 }
 855
 856 static int stm32_cryp_aes_ccm_decrypt(struct aead_request *req)
 857 {
 858         return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM);
 859 }
 860
 861 static int stm32_cryp_des_ecb_encrypt(struct ablkcipher_request *req)
 862 {
 863         return stm32_cryp_crypt(req, FLG_DES | FLG_ECB | FLG_ENCRYPT);
 864 }
 865
 866 static int stm32_cryp_des_ecb_decrypt(struct ablkcipher_request *req)
 867 {
 868         return stm32_cryp_crypt(req, FLG_DES | FLG_ECB);
 869 }
 870
 871 static int stm32_cryp_des_cbc_encrypt(struct ablkcipher_request *req)
 872 {
 873         return stm32_cryp_crypt(req, FLG_DES | FLG_CBC | FLG_ENCRYPT);
 874 }
 875
 876 static int stm32_cryp_des_cbc_decrypt(struct ablkcipher_request *req)
 877 {
 878         return stm32_cryp_crypt(req, FLG_DES | FLG_CBC);
 879 }
 880
 881 static int stm32_cryp_tdes_ecb_encrypt(struct ablkcipher_request *req)
 882 {
 883         return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB | FLG_ENCRYPT);
 884 }
 885
 886 static int stm32_cryp_tdes_ecb_decrypt(struct ablkcipher_request *req)
 887 {
 888         return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB);
 889 }
 890
 891 static int stm32_cryp_tdes_cbc_encrypt(struct ablkcipher_request *req)
 892 {
 893         return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC | FLG_ENCRYPT);
 894 }
 895
 896 static int stm32_cryp_tdes_cbc_decrypt(struct ablkcipher_request *req)
 897 {
 898         return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC);
 899 }
 900
 901 static int stm32_cryp_prepare_req(struct ablkcipher_request *req,
 902                                   struct aead_request *areq)
 903 {
 904         struct stm32_cryp_ctx *ctx;
 905         struct stm32_cryp *cryp;
 906         struct stm32_cryp_reqctx *rctx;
 907         int ret;
 908
 909         if (!req && !areq)
 910                 return -EINVAL;
 911
 912         ctx = req ? crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)) :
 913                     crypto_aead_ctx(crypto_aead_reqtfm(areq));
 914
 915         cryp = ctx->cryp;
 916
 917         if (!cryp)
 918                 return -ENODEV;
 919
 920         mutex_lock(&cryp->lock);
 921
 922         rctx = req ? ablkcipher_request_ctx(req) : aead_request_ctx(areq);
 923         rctx->mode &= FLG_MODE_MASK;
 924
 925         ctx->cryp = cryp;
 926
 927         cryp->flags = (cryp->flags & ~FLG_MODE_MASK) | rctx->mode;
 928         cryp->hw_blocksize = is_aes(cryp) ? AES_BLOCK_SIZE : DES_BLOCK_SIZE;
 929         cryp->ctx = ctx;
 930
 931         if (req) {
 932                 cryp->req = req;
 933                 cryp->total_in = req->nbytes;
 934                 cryp->total_out = cryp->total_in;
 935         } else {
 936                 /*
 937                  * Length of input and output data:
 938                  * Encryption case:
 939                  *  INPUT  =   AssocData  ||   PlainText
 940                  *          <- assoclen ->  <- cryptlen ->
 941                  *          <------- total_in ----------->
 942                  *
 943                  *  OUTPUT =   AssocData  ||  CipherText  ||   AuthTag
 944                  *          <- assoclen ->  <- cryptlen ->  <- authsize ->
 945                  *          <---------------- total_out ----------------->
 946                  *
 947                  * Decryption case:
 948                  *  INPUT  =   AssocData  ||  CipherText  ||  AuthTag
 949                  *          <- assoclen ->  <--------- cryptlen --------->
 950                  *                                          <- authsize ->
 951                  *          <---------------- total_in ------------------>
 952                  *
 953                  *  OUTPUT =   AssocData  ||   PlainText
 954                  *          <- assoclen ->  <- crypten - authsize ->
 955                  *          <---------- total_out ----------------->
 956                  */
 957                 cryp->areq = areq;
 958                 cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq));
 959                 cryp->total_in = areq->assoclen + areq->cryptlen;
 960                 if (is_encrypt(cryp))
 961                         /* Append auth tag to output */
 962                         cryp->total_out = cryp->total_in + cryp->authsize;
 963                 else
 964                         /* No auth tag in output */
 965                         cryp->total_out = cryp->total_in - cryp->authsize;
 966         }
 967
 968         cryp->total_in_save = cryp->total_in;
 969         cryp->total_out_save = cryp->total_out;
 970
 971         cryp->in_sg = req ? req->src : areq->src;
 972         cryp->out_sg = req ? req->dst : areq->dst;
 973         cryp->out_sg_save = cryp->out_sg;
 974
 975         cryp->in_sg_len = sg_nents_for_len(cryp->in_sg, cryp->total_in);
 976         if (cryp->in_sg_len < 0) {
 977                 dev_err(cryp->dev, "Cannot get in_sg_len\n");
 978                 ret = cryp->in_sg_len;
 979                 goto out;
 980         }
 981
 982         cryp->out_sg_len = sg_nents_for_len(cryp->out_sg, cryp->total_out);
 983         if (cryp->out_sg_len < 0) {
 984                 dev_err(cryp->dev, "Cannot get out_sg_len\n");
 985                 ret = cryp->out_sg_len;
 986                 goto out;
 987         }
 988
 989         ret = stm32_cryp_copy_sgs(cryp);
 990         if (ret)
 991                 goto out;
 992
 993         scatterwalk_start(&cryp->in_walk, cryp->in_sg);
 994         scatterwalk_start(&cryp->out_walk, cryp->out_sg);
 995
 996         if (is_gcm(cryp) || is_ccm(cryp)) {
 997                 /* In output, jump after assoc data */
 998                 scatterwalk_advance(&cryp->out_walk, cryp->areq->assoclen);
 999                 cryp->total_out -= cryp->areq->assoclen;
1000         }
1001
1002         ret = stm32_cryp_hw_init(cryp);
1003 out:
1004         if (ret)
1005                 mutex_unlock(&cryp->lock);
1006
1007         return ret;
1008 }
1009
1010 static int stm32_cryp_prepare_cipher_req(struct crypto_engine *engine,
1011                                          void *areq)
1012 {
1013         struct ablkcipher_request *req = container_of(areq,
1014                                                       struct ablkcipher_request,
1015                                                       base);
1016
1017         return stm32_cryp_prepare_req(req, NULL);
1018 }
1019
1020 static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq)
1021 {
1022         struct ablkcipher_request *req = container_of(areq,
1023                                                       struct ablkcipher_request,
1024                                                       base);
1025         struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx(
1026                         crypto_ablkcipher_reqtfm(req));
1027         struct stm32_cryp *cryp = ctx->cryp;
1028
1029         if (!cryp)
1030                 return -ENODEV;
1031
1032         return stm32_cryp_cpu_start(cryp);
1033 }
1034
1035 static int stm32_cryp_prepare_aead_req(struct crypto_engine *engine, void *areq)
1036 {
1037         struct aead_request *req = container_of(areq, struct aead_request,
1038                                                 base);
1039
1040         return stm32_cryp_prepare_req(NULL, req);
1041 }
1042
1043 static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq)
1044 {
1045         struct aead_request *req = container_of(areq, struct aead_request,
1046                                                 base);
1047         struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
1048         struct stm32_cryp *cryp = ctx->cryp;
1049
1050         if (!cryp)
1051                 return -ENODEV;
1052
1053         if (unlikely(!cryp->areq->assoclen &&
1054                      !stm32_cryp_get_input_text_len(cryp))) {
1055                 /* No input data to process: get tag and finish */
1056                 stm32_cryp_finish_req(cryp, 0);
1057                 return 0;
1058         }
1059
1060         return stm32_cryp_cpu_start(cryp);
1061 }
1062
1063 static u32 *stm32_cryp_next_out(struct stm32_cryp *cryp, u32 *dst,
1064                                 unsigned int n)
1065 {
1066         scatterwalk_advance(&cryp->out_walk, n);
1067
1068         if (unlikely(cryp->out_sg->length == _walked_out)) {
1069                 cryp->out_sg = sg_next(cryp->out_sg);
1070                 if (cryp->out_sg) {
1071                         scatterwalk_start(&cryp->out_walk, cryp->out_sg);
1072                         return (sg_virt(cryp->out_sg) + _walked_out);
1073                 }
1074         }
1075
1076         return (u32 *)((u8 *)dst + n);
1077 }
1078
1079 static u32 *stm32_cryp_next_in(struct stm32_cryp *cryp, u32 *src,
1080                                unsigned int n)
1081 {
1082         scatterwalk_advance(&cryp->in_walk, n);
1083
1084         if (unlikely(cryp->in_sg->length == _walked_in)) {
1085                 cryp->in_sg = sg_next(cryp->in_sg);
1086                 if (cryp->in_sg) {
1087                         scatterwalk_start(&cryp->in_walk, cryp->in_sg);
1088                         return (sg_virt(cryp->in_sg) + _walked_in);
1089                 }
1090         }
1091
1092         return (u32 *)((u8 *)src + n);
1093 }
1094
1095 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp)
1096 {
1097         u32 cfg, size_bit, *dst, d32;
1098         u8 *d8;
1099         unsigned int i, j;
1100         int ret = 0;
1101
1102         /* Update Config */
1103         cfg = stm32_cryp_read(cryp, CRYP_CR);
1104
1105         cfg &= ~CR_PH_MASK;
1106         cfg |= CR_PH_FINAL;
1107         cfg &= ~CR_DEC_NOT_ENC;
1108         cfg |= CR_CRYPEN;
1109
1110         stm32_cryp_write(cryp, CRYP_CR, cfg);
1111
1112         if (is_gcm(cryp)) {
1113                 /* GCM: write aad and payload size (in bits) */
1114                 size_bit = cryp->areq->assoclen * 8;
1115                 if (cryp->caps->swap_final)
1116                         size_bit = cpu_to_be32(size_bit);
1117
1118                 stm32_cryp_write(cryp, CRYP_DIN, 0);
1119                 stm32_cryp_write(cryp, CRYP_DIN, size_bit);
1120
1121                 size_bit = is_encrypt(cryp) ? cryp->areq->cryptlen :
1122                                 cryp->areq->cryptlen - AES_BLOCK_SIZE;
1123                 size_bit *= 8;
1124                 if (cryp->caps->swap_final)
1125                         size_bit = cpu_to_be32(size_bit);
1126
1127                 stm32_cryp_write(cryp, CRYP_DIN, 0);
1128                 stm32_cryp_write(cryp, CRYP_DIN, size_bit);
1129         } else {
1130                 /* CCM: write CTR0 */
1131                 u8 iv[AES_BLOCK_SIZE];
1132                 u32 *iv32 = (u32 *)iv;
1133
1134                 memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE);
1135                 memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
1136
1137                 for (i = 0; i < AES_BLOCK_32; i++) {
1138                         if (!cryp->caps->padding_wa)
1139                                 *iv32 = cpu_to_be32(*iv32);
1140                         stm32_cryp_write(cryp, CRYP_DIN, *iv32++);
1141                 }
1142         }
1143
1144         /* Wait for output data */
1145         ret = stm32_cryp_wait_output(cryp);
1146         if (ret) {
1147                 dev_err(cryp->dev, "Timeout (read tag)\n");
1148                 return ret;
1149         }
1150
1151         if (is_encrypt(cryp)) {
1152                 /* Get and write tag */
1153                 dst = sg_virt(cryp->out_sg) + _walked_out;
1154
1155                 for (i = 0; i < AES_BLOCK_32; i++) {
1156                         if (cryp->total_out >= sizeof(u32)) {
1157                                 /* Read a full u32 */
1158                                 *dst = stm32_cryp_read(cryp, CRYP_DOUT);
1159
1160                                 dst = stm32_cryp_next_out(cryp, dst,
1161                                                           sizeof(u32));
1162                                 cryp->total_out -= sizeof(u32);
1163                         } else if (!cryp->total_out) {
1164                                 /* Empty fifo out (data from input padding) */
1165                                 stm32_cryp_read(cryp, CRYP_DOUT);
1166                         } else {
1167                                 /* Read less than an u32 */
1168                                 d32 = stm32_cryp_read(cryp, CRYP_DOUT);
1169                                 d8 = (u8 *)&d32;
1170
1171                                 for (j = 0; j < cryp->total_out; j++) {
1172                                         *((u8 *)dst) = *(d8++);
1173                                         dst = stm32_cryp_next_out(cryp, dst, 1);
1174                                 }
1175                                 cryp->total_out = 0;
1176                         }
1177                 }
1178         } else {
1179                 /* Get and check tag */
1180                 u32 in_tag[AES_BLOCK_32], out_tag[AES_BLOCK_32];
1181
1182                 scatterwalk_map_and_copy(in_tag, cryp->in_sg,
1183                                          cryp->total_in_save - cryp->authsize,
1184                                          cryp->authsize, 0);
1185
1186                 for (i = 0; i < AES_BLOCK_32; i++)
1187                         out_tag[i] = stm32_cryp_read(cryp, CRYP_DOUT);
1188
1189                 if (crypto_memneq(in_tag, out_tag, cryp->authsize))
1190                         ret = -EBADMSG;
1191         }
1192
1193         /* Disable cryp */
1194         cfg &= ~CR_CRYPEN;
1195         stm32_cryp_write(cryp, CRYP_CR, cfg);
1196
1197         return ret;
1198 }
1199
1200 static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp)
1201 {
1202         u32 cr;
1203
1204         if (unlikely(cryp->last_ctr[3] == 0xFFFFFFFF)) {
1205                 cryp->last_ctr[3] = 0;
1206                 cryp->last_ctr[2]++;
1207                 if (!cryp->last_ctr[2]) {
1208                         cryp->last_ctr[1]++;
1209                         if (!cryp->last_ctr[1])
1210                                 cryp->last_ctr[0]++;
1211                 }
1212
1213                 cr = stm32_cryp_read(cryp, CRYP_CR);
1214                 stm32_cryp_write(cryp, CRYP_CR, cr & ~CR_CRYPEN);
1215
1216                 stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->last_ctr);
1217
1218                 stm32_cryp_write(cryp, CRYP_CR, cr);
1219         }
1220
1221         cryp->last_ctr[0] = stm32_cryp_read(cryp, CRYP_IV0LR);
1222         cryp->last_ctr[1] = stm32_cryp_read(cryp, CRYP_IV0RR);
1223         cryp->last_ctr[2] = stm32_cryp_read(cryp, CRYP_IV1LR);
1224         cryp->last_ctr[3] = stm32_cryp_read(cryp, CRYP_IV1RR);
1225 }
1226
1227 static bool stm32_cryp_irq_read_data(struct stm32_cryp *cryp)
1228 {
1229         unsigned int i, j;
1230         u32 d32, *dst;
1231         u8 *d8;
1232         size_t tag_size;
1233
1234         /* Do no read tag now (if any) */
1235         if (is_encrypt(cryp) && (is_gcm(cryp) || is_ccm(cryp)))
1236                 tag_size = cryp->authsize;
1237         else
1238                 tag_size = 0;
1239
1240         dst = sg_virt(cryp->out_sg) + _walked_out;
1241
1242         for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) {
1243                 if (likely(cryp->total_out - tag_size >= sizeof(u32))) {
1244                         /* Read a full u32 */
1245                         *dst = stm32_cryp_read(cryp, CRYP_DOUT);
1246
1247                         dst = stm32_cryp_next_out(cryp, dst, sizeof(u32));
1248                         cryp->total_out -= sizeof(u32);
1249                 } else if (cryp->total_out == tag_size) {
1250                         /* Empty fifo out (data from input padding) */
1251                         d32 = stm32_cryp_read(cryp, CRYP_DOUT);
1252                 } else {
1253                         /* Read less than an u32 */
1254                         d32 = stm32_cryp_read(cryp, CRYP_DOUT);
1255                         d8 = (u8 *)&d32;
1256
1257                         for (j = 0; j < cryp->total_out - tag_size; j++) {
1258                                 *((u8 *)dst) = *(d8++);
1259                                 dst = stm32_cryp_next_out(cryp, dst, 1);
1260                         }
1261                         cryp->total_out = tag_size;
1262                 }
1263         }
1264
1265         return !(cryp->total_out - tag_size) || !cryp->total_in;
1266 }
1267
1268 static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp)
1269 {
1270         unsigned int i, j;
1271         u32 *src;
1272         u8 d8[4];
1273         size_t tag_size;
1274
1275         /* Do no write tag (if any) */
1276         if (is_decrypt(cryp) && (is_gcm(cryp) || is_ccm(cryp)))
1277                 tag_size = cryp->authsize;
1278         else
1279                 tag_size = 0;
1280
1281         src = sg_virt(cryp->in_sg) + _walked_in;
1282
1283         for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) {
1284                 if (likely(cryp->total_in - tag_size >= sizeof(u32))) {
1285                         /* Write a full u32 */
1286                         stm32_cryp_write(cryp, CRYP_DIN, *src);
1287
1288                         src = stm32_cryp_next_in(cryp, src, sizeof(u32));
1289                         cryp->total_in -= sizeof(u32);
1290                 } else if (cryp->total_in == tag_size) {
1291                         /* Write padding data */
1292                         stm32_cryp_write(cryp, CRYP_DIN, 0);
1293                 } else {
1294                         /* Write less than an u32 */
1295                         memset(d8, 0, sizeof(u32));
1296                         for (j = 0; j < cryp->total_in - tag_size; j++) {
1297                                 d8[j] = *((u8 *)src);
1298                                 src = stm32_cryp_next_in(cryp, src, 1);
1299                         }
1300
1301                         stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
1302                         cryp->total_in = tag_size;
1303                 }
1304         }
1305 }
1306
1307 static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp)
1308 {
1309         int err;
1310         u32 cfg, tmp[AES_BLOCK_32];
1311         size_t total_in_ori = cryp->total_in;
1312         struct scatterlist *out_sg_ori = cryp->out_sg;
1313         unsigned int i;
1314
1315         /* 'Special workaround' procedure described in the datasheet */
1316
1317         /* a) disable ip */
1318         stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1319         cfg = stm32_cryp_read(cryp, CRYP_CR);
1320         cfg &= ~CR_CRYPEN;
1321         stm32_cryp_write(cryp, CRYP_CR, cfg);
1322
1323         /* b) Update IV1R */
1324         stm32_cryp_write(cryp, CRYP_IV1RR, cryp->gcm_ctr - 2);
1325
1326         /* c) change mode to CTR */
1327         cfg &= ~CR_ALGO_MASK;
1328         cfg |= CR_AES_CTR;
1329         stm32_cryp_write(cryp, CRYP_CR, cfg);
1330
1331         /* a) enable IP */
1332         cfg |= CR_CRYPEN;
1333         stm32_cryp_write(cryp, CRYP_CR, cfg);
1334
1335         /* b) pad and write the last block */
1336         stm32_cryp_irq_write_block(cryp);
1337         cryp->total_in = total_in_ori;
1338         err = stm32_cryp_wait_output(cryp);
1339         if (err) {
1340                 dev_err(cryp->dev, "Timeout (write gcm header)\n");
1341                 return stm32_cryp_finish_req(cryp, err);
1342         }
1343
1344         /* c) get and store encrypted data */
1345         stm32_cryp_irq_read_data(cryp);
1346         scatterwalk_map_and_copy(tmp, out_sg_ori,
1347                                  cryp->total_in_save - total_in_ori,
1348                                  total_in_ori, 0);
1349
1350         /* d) change mode back to AES GCM */
1351         cfg &= ~CR_ALGO_MASK;
1352         cfg |= CR_AES_GCM;
1353         stm32_cryp_write(cryp, CRYP_CR, cfg);
1354
1355         /* e) change phase to Final */
1356         cfg &= ~CR_PH_MASK;
1357         cfg |= CR_PH_FINAL;
1358         stm32_cryp_write(cryp, CRYP_CR, cfg);
1359
1360         /* f) write padded data */
1361         for (i = 0; i < AES_BLOCK_32; i++) {
1362                 if (cryp->total_in)
1363                         stm32_cryp_write(cryp, CRYP_DIN, tmp[i]);
1364                 else
1365                         stm32_cryp_write(cryp, CRYP_DIN, 0);
1366
1367                 cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in);
1368         }
1369
1370         /* g) Empty fifo out */
1371         err = stm32_cryp_wait_output(cryp);
1372         if (err) {
1373                 dev_err(cryp->dev, "Timeout (write gcm header)\n");
1374                 return stm32_cryp_finish_req(cryp, err);
1375         }
1376
1377         for (i = 0; i < AES_BLOCK_32; i++)
1378                 stm32_cryp_read(cryp, CRYP_DOUT);
1379
1380         /* h) run the he normal Final phase */
1381         stm32_cryp_finish_req(cryp, 0);
1382 }
1383
1384 static void stm32_cryp_irq_set_npblb(struct stm32_cryp *cryp)
1385 {
1386         u32 cfg, payload_bytes;
1387
1388         /* disable ip, set NPBLB and reneable ip */
1389         cfg = stm32_cryp_read(cryp, CRYP_CR);
1390         cfg &= ~CR_CRYPEN;
1391         stm32_cryp_write(cryp, CRYP_CR, cfg);
1392
1393         payload_bytes = is_decrypt(cryp) ? cryp->total_in - cryp->authsize :
1394                                            cryp->total_in;
1395         cfg |= (cryp->hw_blocksize - payload_bytes) << CR_NBPBL_SHIFT;
1396         cfg |= CR_CRYPEN;
1397         stm32_cryp_write(cryp, CRYP_CR, cfg);
1398 }
1399
1400 static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp)
1401 {
1402         int err = 0;
1403         u32 cfg, iv1tmp;
1404         u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32], tmp[AES_BLOCK_32];
1405         size_t last_total_out, total_in_ori = cryp->total_in;
1406         struct scatterlist *out_sg_ori = cryp->out_sg;
1407         unsigned int i;
1408
1409         /* 'Special workaround' procedure described in the datasheet */
1410         cryp->flags |= FLG_CCM_PADDED_WA;
1411
1412         /* a) disable ip */
1413         stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1414
1415         cfg = stm32_cryp_read(cryp, CRYP_CR);
1416         cfg &= ~CR_CRYPEN;
1417         stm32_cryp_write(cryp, CRYP_CR, cfg);
1418
1419         /* b) get IV1 from CRYP_CSGCMCCM7 */
1420         iv1tmp = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + 7 * 4);
1421
1422         /* c) Load CRYP_CSGCMCCMxR */
1423         for (i = 0; i < ARRAY_SIZE(cstmp1); i++)
1424                 cstmp1[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4);
1425
1426         /* d) Write IV1R */
1427         stm32_cryp_write(cryp, CRYP_IV1RR, iv1tmp);
1428
1429         /* e) change mode to CTR */
1430         cfg &= ~CR_ALGO_MASK;
1431         cfg |= CR_AES_CTR;
1432         stm32_cryp_write(cryp, CRYP_CR, cfg);
1433
1434         /* a) enable IP */
1435         cfg |= CR_CRYPEN;
1436         stm32_cryp_write(cryp, CRYP_CR, cfg);
1437
1438         /* b) pad and write the last block */
1439         stm32_cryp_irq_write_block(cryp);
1440         cryp->total_in = total_in_ori;
1441         err = stm32_cryp_wait_output(cryp);
1442         if (err) {
1443                 dev_err(cryp->dev, "Timeout (wite ccm padded data)\n");
1444                 return stm32_cryp_finish_req(cryp, err);
1445         }
1446
1447         /* c) get and store decrypted data */
1448         last_total_out = cryp->total_out;
1449         stm32_cryp_irq_read_data(cryp);
1450
1451         memset(tmp, 0, sizeof(tmp));
1452         scatterwalk_map_and_copy(tmp, out_sg_ori,
1453                                  cryp->total_out_save - last_total_out,
1454                                  last_total_out, 0);
1455
1456         /* d) Load again CRYP_CSGCMCCMxR */
1457         for (i = 0; i < ARRAY_SIZE(cstmp2); i++)
1458                 cstmp2[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4);
1459
1460         /* e) change mode back to AES CCM */
1461         cfg &= ~CR_ALGO_MASK;
1462         cfg |= CR_AES_CCM;
1463         stm32_cryp_write(cryp, CRYP_CR, cfg);
1464
1465         /* f) change phase to header */
1466         cfg &= ~CR_PH_MASK;
1467         cfg |= CR_PH_HEADER;
1468         stm32_cryp_write(cryp, CRYP_CR, cfg);
1469
1470         /* g) XOR and write padded data */
1471         for (i = 0; i < ARRAY_SIZE(tmp); i++) {
1472                 tmp[i] ^= cstmp1[i];
1473                 tmp[i] ^= cstmp2[i];
1474                 stm32_cryp_write(cryp, CRYP_DIN, tmp[i]);
1475         }
1476
1477         /* h) wait for completion */
1478         err = stm32_cryp_wait_busy(cryp);
1479         if (err)
1480                 dev_err(cryp->dev, "Timeout (wite ccm padded data)\n");
1481
1482         /* i) run the he normal Final phase */
1483         stm32_cryp_finish_req(cryp, err);
1484 }
1485
1486 static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp)
1487 {
1488         if (unlikely(!cryp->total_in)) {
1489                 dev_warn(cryp->dev, "No more data to process\n");
1490                 return;
1491         }
1492
1493         if (unlikely(cryp->total_in < AES_BLOCK_SIZE &&
1494                      (stm32_cryp_get_hw_mode(cryp) == CR_AES_GCM) &&
1495                      is_encrypt(cryp))) {
1496                 /* Padding for AES GCM encryption */
1497                 if (cryp->caps->padding_wa)
1498                         /* Special case 1 */
1499                         return stm32_cryp_irq_write_gcm_padded_data(cryp);
1500
1501                 /* Setting padding bytes (NBBLB) */
1502                 stm32_cryp_irq_set_npblb(cryp);
1503         }
1504
1505         if (unlikely((cryp->total_in - cryp->authsize < AES_BLOCK_SIZE) &&
1506                      (stm32_cryp_get_hw_mode(cryp) == CR_AES_CCM) &&
1507                      is_decrypt(cryp))) {
1508                 /* Padding for AES CCM decryption */
1509                 if (cryp->caps->padding_wa)
1510                         /* Special case 2 */
1511                         return stm32_cryp_irq_write_ccm_padded_data(cryp);
1512
1513                 /* Setting padding bytes (NBBLB) */
1514                 stm32_cryp_irq_set_npblb(cryp);
1515         }
1516
1517         if (is_aes(cryp) && is_ctr(cryp))
1518                 stm32_cryp_check_ctr_counter(cryp);
1519
1520         stm32_cryp_irq_write_block(cryp);
1521 }
1522
1523 static void stm32_cryp_irq_write_gcm_header(struct stm32_cryp *cryp)
1524 {
1525         int err;
1526         unsigned int i, j;
1527         u32 cfg, *src;
1528
1529         src = sg_virt(cryp->in_sg) + _walked_in;
1530
1531         for (i = 0; i < AES_BLOCK_32; i++) {
1532                 stm32_cryp_write(cryp, CRYP_DIN, *src);
1533
1534                 src = stm32_cryp_next_in(cryp, src, sizeof(u32));
1535                 cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in);
1536
1537                 /* Check if whole header written */
1538                 if ((cryp->total_in_save - cryp->total_in) ==
1539                                 cryp->areq->assoclen) {
1540                         /* Write padding if needed */
1541                         for (j = i + 1; j < AES_BLOCK_32; j++)
1542                                 stm32_cryp_write(cryp, CRYP_DIN, 0);
1543
1544                         /* Wait for completion */
1545                         err = stm32_cryp_wait_busy(cryp);
1546                         if (err) {
1547                                 dev_err(cryp->dev, "Timeout (gcm header)\n");
1548                                 return stm32_cryp_finish_req(cryp, err);
1549                         }
1550
1551                         if (stm32_cryp_get_input_text_len(cryp)) {
1552                                 /* Phase 3 : payload */
1553                                 cfg = stm32_cryp_read(cryp, CRYP_CR);
1554                                 cfg &= ~CR_CRYPEN;
1555                                 stm32_cryp_write(cryp, CRYP_CR, cfg);
1556
1557                                 cfg &= ~CR_PH_MASK;
1558                                 cfg |= CR_PH_PAYLOAD;
1559                                 cfg |= CR_CRYPEN;
1560                                 stm32_cryp_write(cryp, CRYP_CR, cfg);
1561                         } else {
1562                                 /* Phase 4 : tag */
1563                                 stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1564                                 stm32_cryp_finish_req(cryp, 0);
1565                         }
1566
1567                         break;
1568                 }
1569
1570                 if (!cryp->total_in)
1571                         break;
1572         }
1573 }
1574
1575 static void stm32_cryp_irq_write_ccm_header(struct stm32_cryp *cryp)
1576 {
1577         int err;
1578         unsigned int i = 0, j, k;
1579         u32 alen, cfg, *src;
1580         u8 d8[4];
1581
1582         src = sg_virt(cryp->in_sg) + _walked_in;
1583         alen = cryp->areq->assoclen;
1584
1585         if (!_walked_in) {
1586                 if (cryp->areq->assoclen <= 65280) {
1587                         /* Write first u32 of B1 */
1588                         d8[0] = (alen >> 8) & 0xFF;
1589                         d8[1] = alen & 0xFF;
1590                         d8[2] = *((u8 *)src);
1591                         src = stm32_cryp_next_in(cryp, src, 1);
1592                         d8[3] = *((u8 *)src);
1593                         src = stm32_cryp_next_in(cryp, src, 1);
1594
1595                         stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
1596                         i++;
1597
1598                         cryp->total_in -= min_t(size_t, 2, cryp->total_in);
1599                 } else {
1600                         /* Build the two first u32 of B1 */
1601                         d8[0] = 0xFF;
1602                         d8[1] = 0xFE;
1603                         d8[2] = alen & 0xFF000000;
1604                         d8[3] = alen & 0x00FF0000;
1605
1606                         stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
1607                         i++;
1608
1609                         d8[0] = alen & 0x0000FF00;
1610                         d8[1] = alen & 0x000000FF;
1611                         d8[2] = *((u8 *)src);
1612                         src = stm32_cryp_next_in(cryp, src, 1);
1613                         d8[3] = *((u8 *)src);
1614                         src = stm32_cryp_next_in(cryp, src, 1);
1615
1616                         stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
1617                         i++;
1618
1619                         cryp->total_in -= min_t(size_t, 2, cryp->total_in);
1620                 }
1621         }
1622
1623         /* Write next u32 */
1624         for (; i < AES_BLOCK_32; i++) {
1625                 /* Build an u32 */
1626                 memset(d8, 0, sizeof(u32));
1627                 for (k = 0; k < sizeof(u32); k++) {
1628                         d8[k] = *((u8 *)src);
1629                         src = stm32_cryp_next_in(cryp, src, 1);
1630
1631                         cryp->total_in -= min_t(size_t, 1, cryp->total_in);
1632                         if ((cryp->total_in_save - cryp->total_in) == alen)
1633                                 break;
1634                 }
1635
1636                 stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
1637
1638                 if ((cryp->total_in_save - cryp->total_in) == alen) {
1639                         /* Write padding if needed */
1640                         for (j = i + 1; j < AES_BLOCK_32; j++)
1641                                 stm32_cryp_write(cryp, CRYP_DIN, 0);
1642
1643                         /* Wait for completion */
1644                         err = stm32_cryp_wait_busy(cryp);
1645                         if (err) {
1646                                 dev_err(cryp->dev, "Timeout (ccm header)\n");
1647                                 return stm32_cryp_finish_req(cryp, err);
1648                         }
1649
1650                         if (stm32_cryp_get_input_text_len(cryp)) {
1651                                 /* Phase 3 : payload */
1652                                 cfg = stm32_cryp_read(cryp, CRYP_CR);
1653                                 cfg &= ~CR_CRYPEN;
1654                                 stm32_cryp_write(cryp, CRYP_CR, cfg);
1655
1656                                 cfg &= ~CR_PH_MASK;
1657                                 cfg |= CR_PH_PAYLOAD;
1658                                 cfg |= CR_CRYPEN;
1659                                 stm32_cryp_write(cryp, CRYP_CR, cfg);
1660                         } else {
1661                                 /* Phase 4 : tag */
1662                                 stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1663                                 stm32_cryp_finish_req(cryp, 0);
1664                         }
1665
1666                         break;
1667                 }
1668         }
1669 }
1670
1671 static irqreturn_t stm32_cryp_irq_thread(int irq, void *arg)
1672 {
1673         struct stm32_cryp *cryp = arg;
1674         u32 ph;
1675
1676         if (cryp->irq_status & MISR_OUT)
1677                 /* Output FIFO IRQ: read data */
1678                 if (unlikely(stm32_cryp_irq_read_data(cryp))) {
1679                         /* All bytes processed, finish */
1680                         stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1681                         stm32_cryp_finish_req(cryp, 0);
1682                         return IRQ_HANDLED;
1683                 }
1684
1685         if (cryp->irq_status & MISR_IN) {
1686                 if (is_gcm(cryp)) {
1687                         ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK;
1688                         if (unlikely(ph == CR_PH_HEADER))
1689                                 /* Write Header */
1690                                 stm32_cryp_irq_write_gcm_header(cryp);
1691                         else
1692                                 /* Input FIFO IRQ: write data */
1693                                 stm32_cryp_irq_write_data(cryp);
1694                         cryp->gcm_ctr++;
1695                 } else if (is_ccm(cryp)) {
1696                         ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK;
1697                         if (unlikely(ph == CR_PH_HEADER))
1698                                 /* Write Header */
1699                                 stm32_cryp_irq_write_ccm_header(cryp);
1700                         else
1701                                 /* Input FIFO IRQ: write data */
1702                                 stm32_cryp_irq_write_data(cryp);
1703                 } else {
1704                         /* Input FIFO IRQ: write data */
1705                         stm32_cryp_irq_write_data(cryp);
1706                 }
1707         }
1708
1709         return IRQ_HANDLED;
1710 }
1711
1712 static irqreturn_t stm32_cryp_irq(int irq, void *arg)
1713 {
1714         struct stm32_cryp *cryp = arg;
1715
1716         cryp->irq_status = stm32_cryp_read(cryp, CRYP_MISR);
1717
1718         return IRQ_WAKE_THREAD;
1719 }
1720
1721 static struct crypto_alg crypto_algs[] = {
1722 {
1723         .cra_name               = "ecb(aes)",
1724         .cra_driver_name        = "stm32-ecb-aes",
1725         .cra_priority           = 200,
1726         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
1727                                   CRYPTO_ALG_ASYNC,
1728         .cra_blocksize          = AES_BLOCK_SIZE,
1729         .cra_ctxsize            = sizeof(struct stm32_cryp_ctx),
1730         .cra_alignmask          = 0xf,
1731         .cra_type               = &crypto_ablkcipher_type,
1732         .cra_module             = THIS_MODULE,
1733         .cra_init               = stm32_cryp_cra_init,
1734         .cra_ablkcipher = {
1735                 .min_keysize    = AES_MIN_KEY_SIZE,
1736                 .max_keysize    = AES_MAX_KEY_SIZE,
1737                 .setkey         = stm32_cryp_aes_setkey,
1738                 .encrypt        = stm32_cryp_aes_ecb_encrypt,
1739                 .decrypt        = stm32_cryp_aes_ecb_decrypt,
1740         }
1741 },
1742 {
1743         .cra_name               = "cbc(aes)",
1744         .cra_driver_name        = "stm32-cbc-aes",
1745         .cra_priority           = 200,
1746         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
1747                                   CRYPTO_ALG_ASYNC,
1748         .cra_blocksize          = AES_BLOCK_SIZE,
1749         .cra_ctxsize            = sizeof(struct stm32_cryp_ctx),
1750         .cra_alignmask          = 0xf,
1751         .cra_type               = &crypto_ablkcipher_type,
1752         .cra_module             = THIS_MODULE,
1753         .cra_init               = stm32_cryp_cra_init,
1754         .cra_ablkcipher = {
1755                 .min_keysize    = AES_MIN_KEY_SIZE,
1756                 .max_keysize    = AES_MAX_KEY_SIZE,
1757                 .ivsize         = AES_BLOCK_SIZE,
1758                 .setkey         = stm32_cryp_aes_setkey,
1759                 .encrypt        = stm32_cryp_aes_cbc_encrypt,
1760                 .decrypt        = stm32_cryp_aes_cbc_decrypt,
1761         }
1762 },
1763 {
1764         .cra_name               = "ctr(aes)",
1765         .cra_driver_name        = "stm32-ctr-aes",
1766         .cra_priority           = 200,
1767         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
1768                                   CRYPTO_ALG_ASYNC,
1769         .cra_blocksize          = 1,
1770         .cra_ctxsize            = sizeof(struct stm32_cryp_ctx),
1771         .cra_alignmask          = 0xf,
1772         .cra_type               = &crypto_ablkcipher_type,
1773         .cra_module             = THIS_MODULE,
1774         .cra_init               = stm32_cryp_cra_init,
1775         .cra_ablkcipher = {
1776                 .min_keysize    = AES_MIN_KEY_SIZE,
1777                 .max_keysize    = AES_MAX_KEY_SIZE,
1778                 .ivsize         = AES_BLOCK_SIZE,
1779                 .setkey         = stm32_cryp_aes_setkey,
1780                 .encrypt        = stm32_cryp_aes_ctr_encrypt,
1781                 .decrypt        = stm32_cryp_aes_ctr_decrypt,
1782         }
1783 },
1784 {
1785         .cra_name               = "ecb(des)",
1786         .cra_driver_name        = "stm32-ecb-des",
1787         .cra_priority           = 200,
1788         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
1789                                   CRYPTO_ALG_ASYNC,
1790         .cra_blocksize          = DES_BLOCK_SIZE,
1791         .cra_ctxsize            = sizeof(struct stm32_cryp_ctx),
1792         .cra_alignmask          = 0xf,
1793         .cra_type               = &crypto_ablkcipher_type,
1794         .cra_module             = THIS_MODULE,
1795         .cra_init               = stm32_cryp_cra_init,
1796         .cra_ablkcipher = {
1797                 .min_keysize    = DES_BLOCK_SIZE,
1798                 .max_keysize    = DES_BLOCK_SIZE,
1799                 .setkey         = stm32_cryp_des_setkey,
1800                 .encrypt        = stm32_cryp_des_ecb_encrypt,
1801                 .decrypt        = stm32_cryp_des_ecb_decrypt,
1802         }
1803 },
1804 {
1805         .cra_name               = "cbc(des)",
1806         .cra_driver_name        = "stm32-cbc-des",
1807         .cra_priority           = 200,
1808         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
1809                                   CRYPTO_ALG_ASYNC,
1810         .cra_blocksize          = DES_BLOCK_SIZE,
1811         .cra_ctxsize            = sizeof(struct stm32_cryp_ctx),
1812         .cra_alignmask          = 0xf,
1813         .cra_type               = &crypto_ablkcipher_type,
1814         .cra_module             = THIS_MODULE,
1815         .cra_init               = stm32_cryp_cra_init,
1816         .cra_ablkcipher = {
1817                 .min_keysize    = DES_BLOCK_SIZE,
1818                 .max_keysize    = DES_BLOCK_SIZE,
1819                 .ivsize         = DES_BLOCK_SIZE,
1820                 .setkey         = stm32_cryp_des_setkey,
1821                 .encrypt        = stm32_cryp_des_cbc_encrypt,
1822                 .decrypt        = stm32_cryp_des_cbc_decrypt,
1823         }
1824 },
1825 {
1826         .cra_name               = "ecb(des3_ede)",
1827         .cra_driver_name        = "stm32-ecb-des3",
1828         .cra_priority           = 200,
1829         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
1830                                   CRYPTO_ALG_ASYNC,
1831         .cra_blocksize          = DES_BLOCK_SIZE,
1832         .cra_ctxsize            = sizeof(struct stm32_cryp_ctx),
1833         .cra_alignmask          = 0xf,
1834         .cra_type               = &crypto_ablkcipher_type,
1835         .cra_module             = THIS_MODULE,
1836         .cra_init               = stm32_cryp_cra_init,
1837         .cra_ablkcipher = {
1838                 .min_keysize    = 3 * DES_BLOCK_SIZE,
1839                 .max_keysize    = 3 * DES_BLOCK_SIZE,
1840                 .setkey         = stm32_cryp_tdes_setkey,
1841                 .encrypt        = stm32_cryp_tdes_ecb_encrypt,
1842                 .decrypt        = stm32_cryp_tdes_ecb_decrypt,
1843         }
1844 },
1845 {
1846         .cra_name               = "cbc(des3_ede)",
1847         .cra_driver_name        = "stm32-cbc-des3",
1848         .cra_priority           = 200,
1849         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
1850                                   CRYPTO_ALG_ASYNC,
1851         .cra_blocksize          = DES_BLOCK_SIZE,
1852         .cra_ctxsize            = sizeof(struct stm32_cryp_ctx),
1853         .cra_alignmask          = 0xf,
1854         .cra_type               = &crypto_ablkcipher_type,
1855         .cra_module             = THIS_MODULE,
1856         .cra_init               = stm32_cryp_cra_init,
1857         .cra_ablkcipher = {
1858                 .min_keysize    = 3 * DES_BLOCK_SIZE,
1859                 .max_keysize    = 3 * DES_BLOCK_SIZE,
1860                 .ivsize         = DES_BLOCK_SIZE,
1861                 .setkey         = stm32_cryp_tdes_setkey,
1862                 .encrypt        = stm32_cryp_tdes_cbc_encrypt,
1863                 .decrypt        = stm32_cryp_tdes_cbc_decrypt,
1864         }
1865 },
1866 };
1867
1868 static struct aead_alg aead_algs[] = {
1869 {
1870         .setkey         = stm32_cryp_aes_aead_setkey,
1871         .setauthsize    = stm32_cryp_aes_gcm_setauthsize,
1872         .encrypt        = stm32_cryp_aes_gcm_encrypt,
1873         .decrypt        = stm32_cryp_aes_gcm_decrypt,
1874         .init           = stm32_cryp_aes_aead_init,
1875         .ivsize         = 12,
1876         .maxauthsize    = AES_BLOCK_SIZE,
1877
1878         .base = {
1879                 .cra_name               = "gcm(aes)",
1880                 .cra_driver_name        = "stm32-gcm-aes",
1881                 .cra_priority           = 200,
1882                 .cra_flags              = CRYPTO_ALG_ASYNC,
1883                 .cra_blocksize          = 1,
1884                 .cra_ctxsize            = sizeof(struct stm32_cryp_ctx),
1885                 .cra_alignmask          = 0xf,
1886                 .cra_module             = THIS_MODULE,
1887         },
1888 },
1889 {
1890         .setkey         = stm32_cryp_aes_aead_setkey,
1891         .setauthsize    = stm32_cryp_aes_ccm_setauthsize,
1892         .encrypt        = stm32_cryp_aes_ccm_encrypt,
1893         .decrypt        = stm32_cryp_aes_ccm_decrypt,
1894         .init           = stm32_cryp_aes_aead_init,
1895         .ivsize         = AES_BLOCK_SIZE,
1896         .maxauthsize    = AES_BLOCK_SIZE,
1897
1898         .base = {
1899                 .cra_name               = "ccm(aes)",
1900                 .cra_driver_name        = "stm32-ccm-aes",
1901                 .cra_priority           = 200,
1902                 .cra_flags              = CRYPTO_ALG_ASYNC,
1903                 .cra_blocksize          = 1,
1904                 .cra_ctxsize            = sizeof(struct stm32_cryp_ctx),
1905                 .cra_alignmask          = 0xf,
1906                 .cra_module             = THIS_MODULE,
1907         },
1908 },
1909 };
1910
1911 static const struct stm32_cryp_caps f7_data = {
1912         .swap_final = true,
1913         .padding_wa = true,
1914 };
1915
1916 static const struct stm32_cryp_caps mp1_data = {
1917         .swap_final = false,
1918         .padding_wa = false,
1919 };
1920
1921 static const struct of_device_id stm32_dt_ids[] = {
1922         { .compatible = "st,stm32f756-cryp", .data = &f7_data},
1923         { .compatible = "st,stm32mp1-cryp", .data = &mp1_data},
1924         {},
1925 };
1926 MODULE_DEVICE_TABLE(of, stm32_dt_ids);
1927
1928 static int stm32_cryp_probe(struct platform_device *pdev)
1929 {
1930         struct device *dev = &pdev->dev;
1931         struct stm32_cryp *cryp;
1932         struct resource *res;
1933         struct reset_control *rst;
1934         int irq, ret;
1935
1936         cryp = devm_kzalloc(dev, sizeof(*cryp), GFP_KERNEL);
1937         if (!cryp)
1938                 return -ENOMEM;
1939
1940         cryp->caps = of_device_get_match_data(dev);
1941         if (!cryp->caps)
1942                 return -ENODEV;
1943
1944         cryp->dev = dev;
1945
1946         mutex_init(&cryp->lock);
1947
1948         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1949         cryp->regs = devm_ioremap_resource(dev, res);
1950         if (IS_ERR(cryp->regs))
1951                 return PTR_ERR(cryp->regs);
1952
1953         irq = platform_get_irq(pdev, 0);
1954         if (irq < 0) {
1955                 dev_err(dev, "Cannot get IRQ resource\n");
1956                 return irq;
1957         }
1958
1959         ret = devm_request_threaded_irq(dev, irq, stm32_cryp_irq,
1960                                         stm32_cryp_irq_thread, IRQF_ONESHOT,
1961                                         dev_name(dev), cryp);
1962         if (ret) {
1963                 dev_err(dev, "Cannot grab IRQ\n");
1964                 return ret;
1965         }
1966
1967         cryp->clk = devm_clk_get(dev, NULL);
1968         if (IS_ERR(cryp->clk)) {
1969                 dev_err(dev, "Could not get clock\n");
1970                 return PTR_ERR(cryp->clk);
1971         }
1972
1973         ret = clk_prepare_enable(cryp->clk);
1974         if (ret) {
1975                 dev_err(cryp->dev, "Failed to enable clock\n");
1976                 return ret;
1977         }
1978
1979         pm_runtime_set_autosuspend_delay(dev, CRYP_AUTOSUSPEND_DELAY);
1980         pm_runtime_use_autosuspend(dev);
1981
1982         pm_runtime_get_noresume(dev);
1983         pm_runtime_set_active(dev);
1984         pm_runtime_enable(dev);
1985
1986         rst = devm_reset_control_get(dev, NULL);
1987         if (!IS_ERR(rst)) {
1988                 reset_control_assert(rst);
1989                 udelay(2);
1990                 reset_control_deassert(rst);
1991         }
1992
1993         platform_set_drvdata(pdev, cryp);
1994
1995         spin_lock(&cryp_list.lock);
1996         list_add(&cryp->list, &cryp_list.dev_list);
1997         spin_unlock(&cryp_list.lock);
1998
1999         /* Initialize crypto engine */
2000         cryp->engine = crypto_engine_alloc_init(dev, 1);
2001         if (!cryp->engine) {
2002                 dev_err(dev, "Could not init crypto engine\n");
2003                 ret = -ENOMEM;
2004                 goto err_engine1;
2005         }
2006
2007         ret = crypto_engine_start(cryp->engine);
2008         if (ret) {
2009                 dev_err(dev, "Could not start crypto engine\n");
2010                 goto err_engine2;
2011         }
2012
2013         ret = crypto_register_algs(crypto_algs, ARRAY_SIZE(crypto_algs));
2014         if (ret) {
2015                 dev_err(dev, "Could not register algs\n");
2016                 goto err_algs;
2017         }
2018
2019         ret = crypto_register_aeads(aead_algs, ARRAY_SIZE(aead_algs));
2020         if (ret)
2021                 goto err_aead_algs;
2022
2023         dev_info(dev, "Initialized\n");
2024
2025         pm_runtime_put_sync(dev);
2026
2027         return 0;
2028
2029 err_aead_algs:
2030         crypto_unregister_algs(crypto_algs, ARRAY_SIZE(crypto_algs));
2031 err_algs:
2032 err_engine2:
2033         crypto_engine_exit(cryp->engine);
2034 err_engine1:
2035         spin_lock(&cryp_list.lock);
2036         list_del(&cryp->list);
2037         spin_unlock(&cryp_list.lock);
2038
2039         pm_runtime_disable(dev);
2040         pm_runtime_put_noidle(dev);
2041         pm_runtime_disable(dev);
2042         pm_runtime_put_noidle(dev);
2043
2044         clk_disable_unprepare(cryp->clk);
2045
2046         return ret;
2047 }
2048
2049 static int stm32_cryp_remove(struct platform_device *pdev)
2050 {
2051         struct stm32_cryp *cryp = platform_get_drvdata(pdev);
2052         int ret;
2053
2054         if (!cryp)
2055                 return -ENODEV;
2056
2057         ret = pm_runtime_get_sync(cryp->dev);
2058         if (ret < 0)
2059                 return ret;
2060
2061         crypto_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs));
2062         crypto_unregister_algs(crypto_algs, ARRAY_SIZE(crypto_algs));
2063
2064         crypto_engine_exit(cryp->engine);
2065
2066         spin_lock(&cryp_list.lock);
2067         list_del(&cryp->list);
2068         spin_unlock(&cryp_list.lock);
2069
2070         pm_runtime_disable(cryp->dev);
2071         pm_runtime_put_noidle(cryp->dev);
2072
2073         clk_disable_unprepare(cryp->clk);
2074
2075         return 0;
2076 }
2077
2078 #ifdef CONFIG_PM
2079 static int stm32_cryp_runtime_suspend(struct device *dev)
2080 {
2081         struct stm32_cryp *cryp = dev_get_drvdata(dev);
2082
2083         clk_disable_unprepare(cryp->clk);
2084
2085         return 0;
2086 }
2087
2088 static int stm32_cryp_runtime_resume(struct device *dev)
2089 {
2090         struct stm32_cryp *cryp = dev_get_drvdata(dev);
2091         int ret;
2092
2093         ret = clk_prepare_enable(cryp->clk);
2094         if (ret) {
2095                 dev_err(cryp->dev, "Failed to prepare_enable clock\n");
2096                 return ret;
2097         }
2098
2099         return 0;
2100 }
2101 #endif
2102
2103 static const struct dev_pm_ops stm32_cryp_pm_ops = {
2104         SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
2105                                 pm_runtime_force_resume)
2106         SET_RUNTIME_PM_OPS(stm32_cryp_runtime_suspend,
2107                            stm32_cryp_runtime_resume, NULL)
2108 };
2109
2110 static struct platform_driver stm32_cryp_driver = {
2111         .probe  = stm32_cryp_probe,
2112         .remove = stm32_cryp_remove,
2113         .driver = {
2114                 .name           = DRIVER_NAME,
2115                 .pm             = &stm32_cryp_pm_ops,
2116                 .of_match_table = stm32_dt_ids,
2117         },
2118 };
2119
2120 module_platform_driver(stm32_cryp_driver);
2121
2122 MODULE_AUTHOR("Fabien Dessenne <[email protected]>");
2123 MODULE_DESCRIPTION("STMicrolectronics STM32 CRYP hardware driver");
2124 MODULE_LICENSE("GPL");