drivers/crypto/stm32/stm32-hash.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * This file is part of STM32 Crypto driver for Linux.
   4  *
   5  * Copyright (C) 2017, STMicroelectronics - All Rights Reserved
   6  * Author(s): Lionel DEBIEVE <[email protected]> for STMicroelectronics.
   7  */
   8
   9 #include <crypto/engine.h>
  10 #include <crypto/internal/hash.h>
  11 #include <crypto/md5.h>
  12 #include <crypto/scatterwalk.h>
  13 #include <crypto/sha1.h>
  14 #include <crypto/sha2.h>
  15 #include <crypto/sha3.h>
  16 #include <linux/clk.h>
  17 #include <linux/delay.h>
  18 #include <linux/dma-mapping.h>
  19 #include <linux/dmaengine.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/iopoll.h>
  22 #include <linux/kernel.h>
  23 #include <linux/module.h>
  24 #include <linux/of.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/pm_runtime.h>
  27 #include <linux/reset.h>
  28 #include <linux/string.h>
  29
  30 #define HASH_CR                         0x00
  31 #define HASH_DIN                        0x04
  32 #define HASH_STR                        0x08
  33 #define HASH_UX500_HREG(x)              (0x0c + ((x) * 0x04))
  34 #define HASH_IMR                        0x20
  35 #define HASH_SR                         0x24
  36 #define HASH_CSR(x)                     (0x0F8 + ((x) * 0x04))
  37 #define HASH_HREG(x)                    (0x310 + ((x) * 0x04))
  38 #define HASH_HWCFGR                     0x3F0
  39 #define HASH_VER                        0x3F4
  40 #define HASH_ID                         0x3F8
  41
  42 /* Control Register */
  43 #define HASH_CR_INIT                    BIT(2)
  44 #define HASH_CR_DMAE                    BIT(3)
  45 #define HASH_CR_DATATYPE_POS            4
  46 #define HASH_CR_MODE                    BIT(6)
  47 #define HASH_CR_ALGO_POS                7
  48 #define HASH_CR_MDMAT                   BIT(13)
  49 #define HASH_CR_DMAA                    BIT(14)
  50 #define HASH_CR_LKEY                    BIT(16)
  51
  52 /* Interrupt */
  53 #define HASH_DINIE                      BIT(0)
  54 #define HASH_DCIE                       BIT(1)
  55
  56 /* Interrupt Mask */
  57 #define HASH_MASK_CALC_COMPLETION       BIT(0)
  58 #define HASH_MASK_DATA_INPUT            BIT(1)
  59
  60 /* Status Flags */
  61 #define HASH_SR_DATA_INPUT_READY        BIT(0)
  62 #define HASH_SR_OUTPUT_READY            BIT(1)
  63 #define HASH_SR_DMA_ACTIVE              BIT(2)
  64 #define HASH_SR_BUSY                    BIT(3)
  65
  66 /* STR Register */
  67 #define HASH_STR_NBLW_MASK              GENMASK(4, 0)
  68 #define HASH_STR_DCAL                   BIT(8)
  69
  70 /* HWCFGR Register */
  71 #define HASH_HWCFG_DMA_MASK             GENMASK(3, 0)
  72
  73 /* Context swap register */
  74 #define HASH_CSR_NB_SHA256_HMAC         54
  75 #define HASH_CSR_NB_SHA256              38
  76 #define HASH_CSR_NB_SHA512_HMAC         103
  77 #define HASH_CSR_NB_SHA512              91
  78 #define HASH_CSR_NB_SHA3_HMAC           88
  79 #define HASH_CSR_NB_SHA3                72
  80 #define HASH_CSR_NB_MAX                 HASH_CSR_NB_SHA512_HMAC
  81
  82 #define HASH_FLAGS_INIT                 BIT(0)
  83 #define HASH_FLAGS_OUTPUT_READY         BIT(1)
  84 #define HASH_FLAGS_CPU                  BIT(2)
  85 #define HASH_FLAGS_DMA_ACTIVE           BIT(3)
  86 #define HASH_FLAGS_HMAC_INIT            BIT(4)
  87 #define HASH_FLAGS_HMAC_FINAL           BIT(5)
  88 #define HASH_FLAGS_HMAC_KEY             BIT(6)
  89 #define HASH_FLAGS_SHA3_MODE            BIT(7)
  90 #define HASH_FLAGS_FINAL                BIT(15)
  91 #define HASH_FLAGS_FINUP                BIT(16)
  92 #define HASH_FLAGS_ALGO_MASK            GENMASK(20, 17)
  93 #define HASH_FLAGS_ALGO_SHIFT           17
  94 #define HASH_FLAGS_ERRORS               BIT(21)
  95 #define HASH_FLAGS_EMPTY                BIT(22)
  96 #define HASH_FLAGS_HMAC                 BIT(23)
  97 #define HASH_FLAGS_SGS_COPIED           BIT(24)
  98
  99 #define HASH_OP_UPDATE                  1
 100 #define HASH_OP_FINAL                   2
 101
 102 #define HASH_BURST_LEVEL                4
 103
 104 enum stm32_hash_data_format {
 105         HASH_DATA_32_BITS               = 0x0,
 106         HASH_DATA_16_BITS               = 0x1,
 107         HASH_DATA_8_BITS                = 0x2,
 108         HASH_DATA_1_BIT                 = 0x3
 109 };
 110
 111 #define HASH_BUFLEN                     (SHA3_224_BLOCK_SIZE + 4)
 112 #define HASH_MAX_KEY_SIZE               (SHA512_BLOCK_SIZE * 8)
 113
 114 enum stm32_hash_algo {
 115         HASH_SHA1                       = 0,
 116         HASH_MD5                        = 1,
 117         HASH_SHA224                     = 2,
 118         HASH_SHA256                     = 3,
 119         HASH_SHA3_224                   = 4,
 120         HASH_SHA3_256                   = 5,
 121         HASH_SHA3_384                   = 6,
 122         HASH_SHA3_512                   = 7,
 123         HASH_SHA384                     = 12,
 124         HASH_SHA512                     = 15,
 125 };
 126
 127 enum ux500_hash_algo {
 128         HASH_SHA256_UX500               = 0,
 129         HASH_SHA1_UX500                 = 1,
 130 };
 131
 132 #define HASH_AUTOSUSPEND_DELAY          50
 133
 134 struct stm32_hash_ctx {
 135         struct stm32_hash_dev   *hdev;
 136         struct crypto_shash     *xtfm;
 137         unsigned long           flags;
 138
 139         u8                      key[HASH_MAX_KEY_SIZE];
 140         int                     keylen;
 141 };
 142
 143 struct stm32_hash_state {
 144         u32                     flags;
 145
 146         u16                     bufcnt;
 147         u16                     blocklen;
 148
 149         u8 buffer[HASH_BUFLEN] __aligned(sizeof(u32));
 150
 151         /* hash state */
 152         u32                     hw_context[3 + HASH_CSR_NB_MAX];
 153 };
 154
 155 struct stm32_hash_request_ctx {
 156         struct stm32_hash_dev   *hdev;
 157         unsigned long           op;
 158
 159         u8 digest[SHA512_DIGEST_SIZE] __aligned(sizeof(u32));
 160         size_t                  digcnt;
 161
 162         struct scatterlist      *sg;
 163         struct scatterlist      sgl[2]; /* scatterlist used to realize alignment */
 164         unsigned int            offset;
 165         unsigned int            total;
 166         struct scatterlist      sg_key;
 167
 168         dma_addr_t              dma_addr;
 169         size_t                  dma_ct;
 170         int                     nents;
 171
 172         u8                      data_type;
 173
 174         struct stm32_hash_state state;
 175 };
 176
 177 struct stm32_hash_algs_info {
 178         struct ahash_engine_alg *algs_list;
 179         size_t                  size;
 180 };
 181
 182 struct stm32_hash_pdata {
 183         const int                               alg_shift;
 184         const struct stm32_hash_algs_info       *algs_info;
 185         size_t                                  algs_info_size;
 186         bool                                    has_sr;
 187         bool                                    has_mdmat;
 188         bool                                    context_secured;
 189         bool                                    broken_emptymsg;
 190         bool                                    ux500;
 191 };
 192
 193 struct stm32_hash_dev {
 194         struct list_head        list;
 195         struct device           *dev;
 196         struct clk              *clk;
 197         struct reset_control    *rst;
 198         void __iomem            *io_base;
 199         phys_addr_t             phys_base;
 200         u8                      xmit_buf[HASH_BUFLEN] __aligned(sizeof(u32));
 201         u32                     dma_mode;
 202         bool                    polled;
 203
 204         struct ahash_request    *req;
 205         struct crypto_engine    *engine;
 206
 207         unsigned long           flags;
 208
 209         struct dma_chan         *dma_lch;
 210         struct completion       dma_completion;
 211
 212         const struct stm32_hash_pdata   *pdata;
 213 };
 214
 215 struct stm32_hash_drv {
 216         struct list_head        dev_list;
 217         spinlock_t              lock; /* List protection access */
 218 };
 219
 220 static struct stm32_hash_drv stm32_hash = {
 221         .dev_list = LIST_HEAD_INIT(stm32_hash.dev_list),
 222         .lock = __SPIN_LOCK_UNLOCKED(stm32_hash.lock),
 223 };
 224
 225 static void stm32_hash_dma_callback(void *param);
 226 static int stm32_hash_prepare_request(struct ahash_request *req);
 227 static void stm32_hash_unprepare_request(struct ahash_request *req);
 228
 229 static inline u32 stm32_hash_read(struct stm32_hash_dev *hdev, u32 offset)
 230 {
 231         return readl_relaxed(hdev->io_base + offset);
 232 }
 233
 234 static inline void stm32_hash_write(struct stm32_hash_dev *hdev,
 235                                     u32 offset, u32 value)
 236 {
 237         writel_relaxed(value, hdev->io_base + offset);
 238 }
 239
 240 /**
 241  * stm32_hash_wait_busy - wait until hash processor is available. It return an
 242  * error if the hash core is processing a block of data for more than 10 ms.
 243  * @hdev: the stm32_hash_dev device.
 244  */
 245 static inline int stm32_hash_wait_busy(struct stm32_hash_dev *hdev)
 246 {
 247         u32 status;
 248
 249         /* The Ux500 lacks the special status register, we poll the DCAL bit instead */
 250         if (!hdev->pdata->has_sr)
 251                 return readl_relaxed_poll_timeout(hdev->io_base + HASH_STR, status,
 252                                                   !(status & HASH_STR_DCAL), 10, 10000);
 253
 254         return readl_relaxed_poll_timeout(hdev->io_base + HASH_SR, status,
 255                                    !(status & HASH_SR_BUSY), 10, 10000);
 256 }
 257
 258 /**
 259  * stm32_hash_set_nblw - set the number of valid bytes in the last word.
 260  * @hdev: the stm32_hash_dev device.
 261  * @length: the length of the final word.
 262  */
 263 static void stm32_hash_set_nblw(struct stm32_hash_dev *hdev, int length)
 264 {
 265         u32 reg;
 266
 267         reg = stm32_hash_read(hdev, HASH_STR);
 268         reg &= ~(HASH_STR_NBLW_MASK);
 269         reg |= (8U * ((length) % 4U));
 270         stm32_hash_write(hdev, HASH_STR, reg);
 271 }
 272
 273 static int stm32_hash_write_key(struct stm32_hash_dev *hdev)
 274 {
 275         struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req);
 276         struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 277         u32 reg;
 278         int keylen = ctx->keylen;
 279         void *key = ctx->key;
 280
 281         if (keylen) {
 282                 stm32_hash_set_nblw(hdev, keylen);
 283
 284                 while (keylen > 0) {
 285                         stm32_hash_write(hdev, HASH_DIN, *(u32 *)key);
 286                         keylen -= 4;
 287                         key += 4;
 288                 }
 289
 290                 reg = stm32_hash_read(hdev, HASH_STR);
 291                 reg |= HASH_STR_DCAL;
 292                 stm32_hash_write(hdev, HASH_STR, reg);
 293
 294                 return -EINPROGRESS;
 295         }
 296
 297         return 0;
 298 }
 299
 300 /**
 301  * stm32_hash_write_ctrl - Initialize the hash processor, only if
 302  * HASH_FLAGS_INIT is set.
 303  * @hdev: the stm32_hash_dev device
 304  */
 305 static void stm32_hash_write_ctrl(struct stm32_hash_dev *hdev)
 306 {
 307         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
 308         struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req);
 309         struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 310         struct stm32_hash_state *state = &rctx->state;
 311         u32 alg = (state->flags & HASH_FLAGS_ALGO_MASK) >> HASH_FLAGS_ALGO_SHIFT;
 312
 313         u32 reg = HASH_CR_INIT;
 314
 315         if (!(hdev->flags & HASH_FLAGS_INIT)) {
 316                 if (hdev->pdata->ux500) {
 317                         reg |= ((alg & BIT(0)) << HASH_CR_ALGO_POS);
 318                 } else {
 319                         if (hdev->pdata->alg_shift == HASH_CR_ALGO_POS)
 320                                 reg |= ((alg & BIT(1)) << 17) |
 321                                        ((alg & BIT(0)) << HASH_CR_ALGO_POS);
 322                         else
 323                                 reg |= alg << hdev->pdata->alg_shift;
 324                 }
 325
 326                 reg |= (rctx->data_type << HASH_CR_DATATYPE_POS);
 327
 328                 if (state->flags & HASH_FLAGS_HMAC) {
 329                         hdev->flags |= HASH_FLAGS_HMAC;
 330                         reg |= HASH_CR_MODE;
 331                         if (ctx->keylen > crypto_ahash_blocksize(tfm))
 332                                 reg |= HASH_CR_LKEY;
 333                 }
 334
 335                 if (!hdev->polled)
 336                         stm32_hash_write(hdev, HASH_IMR, HASH_DCIE);
 337
 338                 stm32_hash_write(hdev, HASH_CR, reg);
 339
 340                 hdev->flags |= HASH_FLAGS_INIT;
 341
 342                 /*
 343                  * After first block + 1 words are fill up,
 344                  * we only need to fill 1 block to start partial computation
 345                  */
 346                 rctx->state.blocklen -= sizeof(u32);
 347
 348                 dev_dbg(hdev->dev, "Write Control %x\n", reg);
 349         }
 350 }
 351
 352 static void stm32_hash_append_sg(struct stm32_hash_request_ctx *rctx)
 353 {
 354         struct stm32_hash_state *state = &rctx->state;
 355         size_t count;
 356
 357         while ((state->bufcnt < state->blocklen) && rctx->total) {
 358                 count = min(rctx->sg->length - rctx->offset, rctx->total);
 359                 count = min_t(size_t, count, state->blocklen - state->bufcnt);
 360
 361                 if (count <= 0) {
 362                         if ((rctx->sg->length == 0) && !sg_is_last(rctx->sg)) {
 363                                 rctx->sg = sg_next(rctx->sg);
 364                                 continue;
 365                         } else {
 366                                 break;
 367                         }
 368                 }
 369
 370                 scatterwalk_map_and_copy(state->buffer + state->bufcnt,
 371                                          rctx->sg, rctx->offset, count, 0);
 372
 373                 state->bufcnt += count;
 374                 rctx->offset += count;
 375                 rctx->total -= count;
 376
 377                 if (rctx->offset == rctx->sg->length) {
 378                         rctx->sg = sg_next(rctx->sg);
 379                         if (rctx->sg)
 380                                 rctx->offset = 0;
 381                         else
 382                                 rctx->total = 0;
 383                 }
 384         }
 385 }
 386
 387 static int stm32_hash_xmit_cpu(struct stm32_hash_dev *hdev,
 388                                const u8 *buf, size_t length, int final)
 389 {
 390         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
 391         struct stm32_hash_state *state = &rctx->state;
 392         unsigned int count, len32;
 393         const u32 *buffer = (const u32 *)buf;
 394         u32 reg;
 395
 396         if (final) {
 397                 hdev->flags |= HASH_FLAGS_FINAL;
 398
 399                 /* Do not process empty messages if hw is buggy. */
 400                 if (!(hdev->flags & HASH_FLAGS_INIT) && !length &&
 401                     hdev->pdata->broken_emptymsg) {
 402                         state->flags |= HASH_FLAGS_EMPTY;
 403                         return 0;
 404                 }
 405         }
 406
 407         len32 = DIV_ROUND_UP(length, sizeof(u32));
 408
 409         dev_dbg(hdev->dev, "%s: length: %zd, final: %x len32 %i\n",
 410                 __func__, length, final, len32);
 411
 412         hdev->flags |= HASH_FLAGS_CPU;
 413
 414         stm32_hash_write_ctrl(hdev);
 415
 416         if (stm32_hash_wait_busy(hdev))
 417                 return -ETIMEDOUT;
 418
 419         if ((hdev->flags & HASH_FLAGS_HMAC) &&
 420             (!(hdev->flags & HASH_FLAGS_HMAC_KEY))) {
 421                 hdev->flags |= HASH_FLAGS_HMAC_KEY;
 422                 stm32_hash_write_key(hdev);
 423                 if (stm32_hash_wait_busy(hdev))
 424                         return -ETIMEDOUT;
 425         }
 426
 427         for (count = 0; count < len32; count++)
 428                 stm32_hash_write(hdev, HASH_DIN, buffer[count]);
 429
 430         if (final) {
 431                 if (stm32_hash_wait_busy(hdev))
 432                         return -ETIMEDOUT;
 433
 434                 stm32_hash_set_nblw(hdev, length);
 435                 reg = stm32_hash_read(hdev, HASH_STR);
 436                 reg |= HASH_STR_DCAL;
 437                 stm32_hash_write(hdev, HASH_STR, reg);
 438                 if (hdev->flags & HASH_FLAGS_HMAC) {
 439                         if (stm32_hash_wait_busy(hdev))
 440                                 return -ETIMEDOUT;
 441                         stm32_hash_write_key(hdev);
 442                 }
 443                 return -EINPROGRESS;
 444         }
 445
 446         return 0;
 447 }
 448
 449 static int hash_swap_reg(struct stm32_hash_request_ctx *rctx)
 450 {
 451         struct stm32_hash_state *state = &rctx->state;
 452
 453         switch ((state->flags & HASH_FLAGS_ALGO_MASK) >>
 454                 HASH_FLAGS_ALGO_SHIFT) {
 455         case HASH_MD5:
 456         case HASH_SHA1:
 457         case HASH_SHA224:
 458         case HASH_SHA256:
 459                 if (state->flags & HASH_FLAGS_HMAC)
 460                         return HASH_CSR_NB_SHA256_HMAC;
 461                 else
 462                         return HASH_CSR_NB_SHA256;
 463                 break;
 464
 465         case HASH_SHA384:
 466         case HASH_SHA512:
 467                 if (state->flags & HASH_FLAGS_HMAC)
 468                         return HASH_CSR_NB_SHA512_HMAC;
 469                 else
 470                         return HASH_CSR_NB_SHA512;
 471                 break;
 472
 473         case HASH_SHA3_224:
 474         case HASH_SHA3_256:
 475         case HASH_SHA3_384:
 476         case HASH_SHA3_512:
 477                 if (state->flags & HASH_FLAGS_HMAC)
 478                         return HASH_CSR_NB_SHA3_HMAC;
 479                 else
 480                         return HASH_CSR_NB_SHA3;
 481                 break;
 482
 483         default:
 484                 return -EINVAL;
 485         }
 486 }
 487
 488 static int stm32_hash_update_cpu(struct stm32_hash_dev *hdev)
 489 {
 490         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
 491         struct stm32_hash_state *state = &rctx->state;
 492         int bufcnt, err = 0, final;
 493
 494         dev_dbg(hdev->dev, "%s flags %x\n", __func__, state->flags);
 495
 496         final = state->flags & HASH_FLAGS_FINAL;
 497
 498         while ((rctx->total >= state->blocklen) ||
 499                (state->bufcnt + rctx->total >= state->blocklen)) {
 500                 stm32_hash_append_sg(rctx);
 501                 bufcnt = state->bufcnt;
 502                 state->bufcnt = 0;
 503                 err = stm32_hash_xmit_cpu(hdev, state->buffer, bufcnt, 0);
 504                 if (err)
 505                         return err;
 506         }
 507
 508         stm32_hash_append_sg(rctx);
 509
 510         if (final) {
 511                 bufcnt = state->bufcnt;
 512                 state->bufcnt = 0;
 513                 return stm32_hash_xmit_cpu(hdev, state->buffer, bufcnt, 1);
 514         }
 515
 516         return err;
 517 }
 518
 519 static int stm32_hash_xmit_dma(struct stm32_hash_dev *hdev,
 520                                struct scatterlist *sg, int length, int mdmat)
 521 {
 522         struct dma_async_tx_descriptor *in_desc;
 523         dma_cookie_t cookie;
 524         u32 reg;
 525         int err;
 526
 527         dev_dbg(hdev->dev, "%s mdmat: %x length: %d\n", __func__, mdmat, length);
 528
 529         /* do not use dma if there is no data to send */
 530         if (length <= 0)
 531                 return 0;
 532
 533         in_desc = dmaengine_prep_slave_sg(hdev->dma_lch, sg, 1,
 534                                           DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT |
 535                                           DMA_CTRL_ACK);
 536         if (!in_desc) {
 537                 dev_err(hdev->dev, "dmaengine_prep_slave error\n");
 538                 return -ENOMEM;
 539         }
 540
 541         reinit_completion(&hdev->dma_completion);
 542         in_desc->callback = stm32_hash_dma_callback;
 543         in_desc->callback_param = hdev;
 544
 545         hdev->flags |= HASH_FLAGS_DMA_ACTIVE;
 546
 547         reg = stm32_hash_read(hdev, HASH_CR);
 548
 549         if (hdev->pdata->has_mdmat) {
 550                 if (mdmat)
 551                         reg |= HASH_CR_MDMAT;
 552                 else
 553                         reg &= ~HASH_CR_MDMAT;
 554         }
 555         reg |= HASH_CR_DMAE;
 556
 557         stm32_hash_write(hdev, HASH_CR, reg);
 558
 559
 560         cookie = dmaengine_submit(in_desc);
 561         err = dma_submit_error(cookie);
 562         if (err)
 563                 return -ENOMEM;
 564
 565         dma_async_issue_pending(hdev->dma_lch);
 566
 567         if (!wait_for_completion_timeout(&hdev->dma_completion,
 568                                          msecs_to_jiffies(100)))
 569                 err = -ETIMEDOUT;
 570
 571         if (dma_async_is_tx_complete(hdev->dma_lch, cookie,
 572                                      NULL, NULL) != DMA_COMPLETE)
 573                 err = -ETIMEDOUT;
 574
 575         if (err) {
 576                 dev_err(hdev->dev, "DMA Error %i\n", err);
 577                 dmaengine_terminate_all(hdev->dma_lch);
 578                 return err;
 579         }
 580
 581         return -EINPROGRESS;
 582 }
 583
 584 static void stm32_hash_dma_callback(void *param)
 585 {
 586         struct stm32_hash_dev *hdev = param;
 587
 588         complete(&hdev->dma_completion);
 589 }
 590
 591 static int stm32_hash_hmac_dma_send(struct stm32_hash_dev *hdev)
 592 {
 593         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
 594         struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req);
 595         struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 596         int err;
 597
 598         if (ctx->keylen < rctx->state.blocklen || hdev->dma_mode > 0) {
 599                 err = stm32_hash_write_key(hdev);
 600                 if (stm32_hash_wait_busy(hdev))
 601                         return -ETIMEDOUT;
 602         } else {
 603                 if (!(hdev->flags & HASH_FLAGS_HMAC_KEY))
 604                         sg_init_one(&rctx->sg_key, ctx->key,
 605                                     ALIGN(ctx->keylen, sizeof(u32)));
 606
 607                 rctx->dma_ct = dma_map_sg(hdev->dev, &rctx->sg_key, 1,
 608                                           DMA_TO_DEVICE);
 609                 if (rctx->dma_ct == 0) {
 610                         dev_err(hdev->dev, "dma_map_sg error\n");
 611                         return -ENOMEM;
 612                 }
 613
 614                 err = stm32_hash_xmit_dma(hdev, &rctx->sg_key, ctx->keylen, 0);
 615
 616                 dma_unmap_sg(hdev->dev, &rctx->sg_key, 1, DMA_TO_DEVICE);
 617         }
 618
 619         return err;
 620 }
 621
 622 static int stm32_hash_dma_init(struct stm32_hash_dev *hdev)
 623 {
 624         struct dma_slave_config dma_conf;
 625         struct dma_chan *chan;
 626         int err;
 627
 628         memset(&dma_conf, 0, sizeof(dma_conf));
 629
 630         dma_conf.direction = DMA_MEM_TO_DEV;
 631         dma_conf.dst_addr = hdev->phys_base + HASH_DIN;
 632         dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 633         dma_conf.src_maxburst = HASH_BURST_LEVEL;
 634         dma_conf.dst_maxburst = HASH_BURST_LEVEL;
 635         dma_conf.device_fc = false;
 636
 637         chan = dma_request_chan(hdev->dev, "in");
 638         if (IS_ERR(chan))
 639                 return PTR_ERR(chan);
 640
 641         hdev->dma_lch = chan;
 642
 643         err = dmaengine_slave_config(hdev->dma_lch, &dma_conf);
 644         if (err) {
 645                 dma_release_channel(hdev->dma_lch);
 646                 hdev->dma_lch = NULL;
 647                 dev_err(hdev->dev, "Couldn't configure DMA slave.\n");
 648                 return err;
 649         }
 650
 651         init_completion(&hdev->dma_completion);
 652
 653         return 0;
 654 }
 655
 656 static int stm32_hash_dma_send(struct stm32_hash_dev *hdev)
 657 {
 658         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
 659         u32 *buffer = (void *)rctx->state.buffer;
 660         struct scatterlist sg[1], *tsg;
 661         int err = 0, reg, ncp = 0;
 662         unsigned int i, len = 0, bufcnt = 0;
 663         bool final = hdev->flags & HASH_FLAGS_FINAL;
 664         bool is_last = false;
 665         u32 last_word;
 666
 667         dev_dbg(hdev->dev, "%s total: %d bufcnt: %d final: %d\n",
 668                 __func__, rctx->total, rctx->state.bufcnt, final);
 669
 670         if (rctx->nents < 0)
 671                 return -EINVAL;
 672
 673         stm32_hash_write_ctrl(hdev);
 674
 675         if (hdev->flags & HASH_FLAGS_HMAC && (!(hdev->flags & HASH_FLAGS_HMAC_KEY))) {
 676                 hdev->flags |= HASH_FLAGS_HMAC_KEY;
 677                 err = stm32_hash_hmac_dma_send(hdev);
 678                 if (err != -EINPROGRESS)
 679                         return err;
 680         }
 681
 682         for_each_sg(rctx->sg, tsg, rctx->nents, i) {
 683                 sg[0] = *tsg;
 684                 len = sg->length;
 685
 686                 if (sg_is_last(sg) || (bufcnt + sg[0].length) >= rctx->total) {
 687                         if (!final) {
 688                                 /* Always manually put the last word of a non-final transfer. */
 689                                 len -= sizeof(u32);
 690                                 sg_pcopy_to_buffer(rctx->sg, rctx->nents, &last_word, 4, len);
 691                                 sg->length -= sizeof(u32);
 692                         } else {
 693                                 /*
 694                                  * In Multiple DMA mode, DMA must be aborted before the final
 695                                  * transfer.
 696                                  */
 697                                 sg->length = rctx->total - bufcnt;
 698                                 if (hdev->dma_mode > 0) {
 699                                         len = (ALIGN(sg->length, 16) - 16);
 700
 701                                         ncp = sg_pcopy_to_buffer(rctx->sg, rctx->nents,
 702                                                                  rctx->state.buffer,
 703                                                                  sg->length - len,
 704                                                                  rctx->total - sg->length + len);
 705
 706                                         if (!len)
 707                                                 break;
 708
 709                                         sg->length = len;
 710                                 } else {
 711                                         is_last = true;
 712                                         if (!(IS_ALIGNED(sg->length, sizeof(u32)))) {
 713                                                 len = sg->length;
 714                                                 sg->length = ALIGN(sg->length,
 715                                                                    sizeof(u32));
 716                                         }
 717                                 }
 718                         }
 719                 }
 720
 721                 rctx->dma_ct = dma_map_sg(hdev->dev, sg, 1,
 722                                           DMA_TO_DEVICE);
 723                 if (rctx->dma_ct == 0) {
 724                         dev_err(hdev->dev, "dma_map_sg error\n");
 725                         return -ENOMEM;
 726                 }
 727
 728                 err = stm32_hash_xmit_dma(hdev, sg, len, !is_last);
 729
 730                 /* The last word of a non final transfer is sent manually. */
 731                 if (!final) {
 732                         stm32_hash_write(hdev, HASH_DIN, last_word);
 733                         len += sizeof(u32);
 734                 }
 735
 736                 rctx->total -= len;
 737
 738                 bufcnt += sg[0].length;
 739                 dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);
 740
 741                 if (err == -ENOMEM || err == -ETIMEDOUT)
 742                         return err;
 743                 if (is_last)
 744                         break;
 745         }
 746
 747         /*
 748          * When the second last block transfer of 4 words is performed by the DMA,
 749          * the software must set the DMA Abort bit (DMAA) to 1 before completing the
 750          * last transfer of 4 words or less.
 751          */
 752         if (final) {
 753                 if (hdev->dma_mode > 0) {
 754                         if (stm32_hash_wait_busy(hdev))
 755                                 return -ETIMEDOUT;
 756                         reg = stm32_hash_read(hdev, HASH_CR);
 757                         reg &= ~HASH_CR_DMAE;
 758                         reg |= HASH_CR_DMAA;
 759                         stm32_hash_write(hdev, HASH_CR, reg);
 760
 761                         if (ncp) {
 762                                 memset(buffer + ncp, 0, 4 - DIV_ROUND_UP(ncp, sizeof(u32)));
 763                                 writesl(hdev->io_base + HASH_DIN, buffer,
 764                                         DIV_ROUND_UP(ncp, sizeof(u32)));
 765                         }
 766
 767                         stm32_hash_set_nblw(hdev, ncp);
 768                         reg = stm32_hash_read(hdev, HASH_STR);
 769                         reg |= HASH_STR_DCAL;
 770                         stm32_hash_write(hdev, HASH_STR, reg);
 771                         err = -EINPROGRESS;
 772                 }
 773
 774                 /*
 775                  * The hash processor needs the key to be loaded a second time in order
 776                  * to process the HMAC.
 777                  */
 778                 if (hdev->flags & HASH_FLAGS_HMAC) {
 779                         if (stm32_hash_wait_busy(hdev))
 780                                 return -ETIMEDOUT;
 781                         err = stm32_hash_hmac_dma_send(hdev);
 782                 }
 783
 784                 return err;
 785         }
 786
 787         if (err != -EINPROGRESS)
 788                 return err;
 789
 790         return 0;
 791 }
 792
 793 static struct stm32_hash_dev *stm32_hash_find_dev(struct stm32_hash_ctx *ctx)
 794 {
 795         struct stm32_hash_dev *hdev = NULL, *tmp;
 796
 797         spin_lock_bh(&stm32_hash.lock);
 798         if (!ctx->hdev) {
 799                 list_for_each_entry(tmp, &stm32_hash.dev_list, list) {
 800                         hdev = tmp;
 801                         break;
 802                 }
 803                 ctx->hdev = hdev;
 804         } else {
 805                 hdev = ctx->hdev;
 806         }
 807
 808         spin_unlock_bh(&stm32_hash.lock);
 809
 810         return hdev;
 811 }
 812
 813 static int stm32_hash_init(struct ahash_request *req)
 814 {
 815         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 816         struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 817         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
 818         struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
 819         struct stm32_hash_state *state = &rctx->state;
 820         bool sha3_mode = ctx->flags & HASH_FLAGS_SHA3_MODE;
 821
 822         rctx->hdev = hdev;
 823         state->flags = 0;
 824
 825         if (!(hdev->dma_lch &&  hdev->pdata->has_mdmat))
 826                 state->flags |= HASH_FLAGS_CPU;
 827
 828         if (sha3_mode)
 829                 state->flags |= HASH_FLAGS_SHA3_MODE;
 830
 831         rctx->digcnt = crypto_ahash_digestsize(tfm);
 832         switch (rctx->digcnt) {
 833         case MD5_DIGEST_SIZE:
 834                 state->flags |= HASH_MD5 << HASH_FLAGS_ALGO_SHIFT;
 835                 break;
 836         case SHA1_DIGEST_SIZE:
 837                 if (hdev->pdata->ux500)
 838                         state->flags |= HASH_SHA1_UX500 << HASH_FLAGS_ALGO_SHIFT;
 839                 else
 840                         state->flags |= HASH_SHA1 << HASH_FLAGS_ALGO_SHIFT;
 841                 break;
 842         case SHA224_DIGEST_SIZE:
 843                 if (sha3_mode)
 844                         state->flags |= HASH_SHA3_224 << HASH_FLAGS_ALGO_SHIFT;
 845                 else
 846                         state->flags |= HASH_SHA224 << HASH_FLAGS_ALGO_SHIFT;
 847                 break;
 848         case SHA256_DIGEST_SIZE:
 849                 if (sha3_mode) {
 850                         state->flags |= HASH_SHA3_256 << HASH_FLAGS_ALGO_SHIFT;
 851                 } else {
 852                         if (hdev->pdata->ux500)
 853                                 state->flags |= HASH_SHA256_UX500 << HASH_FLAGS_ALGO_SHIFT;
 854                         else
 855                                 state->flags |= HASH_SHA256 << HASH_FLAGS_ALGO_SHIFT;
 856                 }
 857                 break;
 858         case SHA384_DIGEST_SIZE:
 859                 if (sha3_mode)
 860                         state->flags |= HASH_SHA3_384 << HASH_FLAGS_ALGO_SHIFT;
 861                 else
 862                         state->flags |= HASH_SHA384 << HASH_FLAGS_ALGO_SHIFT;
 863                 break;
 864         case SHA512_DIGEST_SIZE:
 865                 if (sha3_mode)
 866                         state->flags |= HASH_SHA3_512 << HASH_FLAGS_ALGO_SHIFT;
 867                 else
 868                         state->flags |= HASH_SHA512 << HASH_FLAGS_ALGO_SHIFT;
 869                 break;
 870         default:
 871                 return -EINVAL;
 872         }
 873
 874         rctx->state.bufcnt = 0;
 875         rctx->state.blocklen = crypto_ahash_blocksize(tfm) + sizeof(u32);
 876         if (rctx->state.blocklen > HASH_BUFLEN) {
 877                 dev_err(hdev->dev, "Error, block too large");
 878                 return -EINVAL;
 879         }
 880         rctx->nents = 0;
 881         rctx->total = 0;
 882         rctx->offset = 0;
 883         rctx->data_type = HASH_DATA_8_BITS;
 884
 885         if (ctx->flags & HASH_FLAGS_HMAC)
 886                 state->flags |= HASH_FLAGS_HMAC;
 887
 888         dev_dbg(hdev->dev, "%s Flags %x\n", __func__, state->flags);
 889
 890         return 0;
 891 }
 892
 893 static int stm32_hash_update_req(struct stm32_hash_dev *hdev)
 894 {
 895         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
 896         struct stm32_hash_state *state = &rctx->state;
 897
 898         dev_dbg(hdev->dev, "update_req: total: %u, digcnt: %zd, final: 0",
 899                 rctx->total, rctx->digcnt);
 900
 901         if (!(state->flags & HASH_FLAGS_CPU))
 902                 return stm32_hash_dma_send(hdev);
 903
 904         return stm32_hash_update_cpu(hdev);
 905 }
 906
 907 static int stm32_hash_final_req(struct stm32_hash_dev *hdev)
 908 {
 909         struct ahash_request *req = hdev->req;
 910         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
 911         struct stm32_hash_state *state = &rctx->state;
 912         int buflen = state->bufcnt;
 913
 914         if (!(state->flags & HASH_FLAGS_CPU)) {
 915                 hdev->flags |= HASH_FLAGS_FINAL;
 916                 return stm32_hash_dma_send(hdev);
 917         }
 918
 919         if (state->flags & HASH_FLAGS_FINUP)
 920                 return stm32_hash_update_req(hdev);
 921
 922         state->bufcnt = 0;
 923
 924         return stm32_hash_xmit_cpu(hdev, state->buffer, buflen, 1);
 925 }
 926
 927 static void stm32_hash_emptymsg_fallback(struct ahash_request *req)
 928 {
 929         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 930         struct stm32_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 931         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
 932         struct stm32_hash_dev *hdev = rctx->hdev;
 933         int ret;
 934
 935         dev_dbg(hdev->dev, "use fallback message size 0 key size %d\n",
 936                 ctx->keylen);
 937
 938         if (!ctx->xtfm) {
 939                 dev_err(hdev->dev, "no fallback engine\n");
 940                 return;
 941         }
 942
 943         if (ctx->keylen) {
 944                 ret = crypto_shash_setkey(ctx->xtfm, ctx->key, ctx->keylen);
 945                 if (ret) {
 946                         dev_err(hdev->dev, "failed to set key ret=%d\n", ret);
 947                         return;
 948                 }
 949         }
 950
 951         ret = crypto_shash_tfm_digest(ctx->xtfm, NULL, 0, rctx->digest);
 952         if (ret)
 953                 dev_err(hdev->dev, "shash digest error\n");
 954 }
 955
 956 static void stm32_hash_copy_hash(struct ahash_request *req)
 957 {
 958         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 959         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
 960         struct stm32_hash_state *state = &rctx->state;
 961         struct stm32_hash_dev *hdev = rctx->hdev;
 962         __be32 *hash = (void *)rctx->digest;
 963         unsigned int i, hashsize;
 964
 965         if (hdev->pdata->broken_emptymsg && (state->flags & HASH_FLAGS_EMPTY))
 966                 return stm32_hash_emptymsg_fallback(req);
 967
 968         hashsize = crypto_ahash_digestsize(tfm);
 969
 970         for (i = 0; i < hashsize / sizeof(u32); i++) {
 971                 if (hdev->pdata->ux500)
 972                         hash[i] = cpu_to_be32(stm32_hash_read(hdev,
 973                                               HASH_UX500_HREG(i)));
 974                 else
 975                         hash[i] = cpu_to_be32(stm32_hash_read(hdev,
 976                                               HASH_HREG(i)));
 977         }
 978 }
 979
 980 static int stm32_hash_finish(struct ahash_request *req)
 981 {
 982         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
 983         u32 reg;
 984
 985         reg = stm32_hash_read(rctx->hdev, HASH_SR);
 986         reg &= ~HASH_SR_OUTPUT_READY;
 987         stm32_hash_write(rctx->hdev, HASH_SR, reg);
 988
 989         if (!req->result)
 990                 return -EINVAL;
 991
 992         memcpy(req->result, rctx->digest, rctx->digcnt);
 993
 994         return 0;
 995 }
 996
 997 static void stm32_hash_finish_req(struct ahash_request *req, int err)
 998 {
 999         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1000         struct stm32_hash_state *state = &rctx->state;
1001         struct stm32_hash_dev *hdev = rctx->hdev;
1002
1003         if (hdev->flags & HASH_FLAGS_DMA_ACTIVE)
1004                 state->flags |= HASH_FLAGS_DMA_ACTIVE;
1005         else
1006                 state->flags &= ~HASH_FLAGS_DMA_ACTIVE;
1007
1008         if (!err && (HASH_FLAGS_FINAL & hdev->flags)) {
1009                 stm32_hash_copy_hash(req);
1010                 err = stm32_hash_finish(req);
1011         }
1012
1013         /* Finalized request mist be unprepared here */
1014         stm32_hash_unprepare_request(req);
1015
1016         crypto_finalize_hash_request(hdev->engine, req, err);
1017 }
1018
1019 static int stm32_hash_handle_queue(struct stm32_hash_dev *hdev,
1020                                    struct ahash_request *req)
1021 {
1022         return crypto_transfer_hash_request_to_engine(hdev->engine, req);
1023 }
1024
1025 static int stm32_hash_one_request(struct crypto_engine *engine, void *areq)
1026 {
1027         struct ahash_request *req = container_of(areq, struct ahash_request,
1028                                                  base);
1029         struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
1030         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1031         struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
1032         struct stm32_hash_state *state = &rctx->state;
1033         int swap_reg;
1034         int err = 0;
1035
1036         if (!hdev)
1037                 return -ENODEV;
1038
1039         dev_dbg(hdev->dev, "processing new req, op: %lu, nbytes %d\n",
1040                 rctx->op, req->nbytes);
1041
1042         pm_runtime_get_sync(hdev->dev);
1043
1044         err = stm32_hash_prepare_request(req);
1045         if (err)
1046                 return err;
1047
1048         hdev->req = req;
1049         hdev->flags = 0;
1050         swap_reg = hash_swap_reg(rctx);
1051
1052         if (state->flags & HASH_FLAGS_INIT) {
1053                 u32 *preg = rctx->state.hw_context;
1054                 u32 reg;
1055                 int i;
1056
1057                 if (!hdev->pdata->ux500)
1058                         stm32_hash_write(hdev, HASH_IMR, *preg++);
1059                 stm32_hash_write(hdev, HASH_STR, *preg++);
1060                 stm32_hash_write(hdev, HASH_CR, *preg);
1061                 reg = *preg++ | HASH_CR_INIT;
1062                 stm32_hash_write(hdev, HASH_CR, reg);
1063
1064                 for (i = 0; i < swap_reg; i++)
1065                         stm32_hash_write(hdev, HASH_CSR(i), *preg++);
1066
1067                 hdev->flags |= HASH_FLAGS_INIT;
1068
1069                 if (state->flags & HASH_FLAGS_HMAC)
1070                         hdev->flags |= HASH_FLAGS_HMAC |
1071                                        HASH_FLAGS_HMAC_KEY;
1072
1073                 if (state->flags & HASH_FLAGS_CPU)
1074                         hdev->flags |= HASH_FLAGS_CPU;
1075
1076                 if (state->flags & HASH_FLAGS_DMA_ACTIVE)
1077                         hdev->flags |= HASH_FLAGS_DMA_ACTIVE;
1078         }
1079
1080         if (rctx->op == HASH_OP_UPDATE)
1081                 err = stm32_hash_update_req(hdev);
1082         else if (rctx->op == HASH_OP_FINAL)
1083                 err = stm32_hash_final_req(hdev);
1084
1085         /* If we have an IRQ, wait for that, else poll for completion */
1086         if (err == -EINPROGRESS && hdev->polled) {
1087                 if (stm32_hash_wait_busy(hdev))
1088                         err = -ETIMEDOUT;
1089                 else {
1090                         hdev->flags |= HASH_FLAGS_OUTPUT_READY;
1091                         err = 0;
1092                 }
1093         }
1094
1095         if (err != -EINPROGRESS)
1096         /* done task will not finish it, so do it here */
1097                 stm32_hash_finish_req(req, err);
1098
1099         return 0;
1100 }
1101
1102 static int stm32_hash_copy_sgs(struct stm32_hash_request_ctx *rctx,
1103                                struct scatterlist *sg, int bs,
1104                                unsigned int new_len)
1105 {
1106         struct stm32_hash_state *state = &rctx->state;
1107         int pages;
1108         void *buf;
1109
1110         pages = get_order(new_len);
1111
1112         buf = (void *)__get_free_pages(GFP_ATOMIC, pages);
1113         if (!buf) {
1114                 pr_err("Couldn't allocate pages for unaligned cases.\n");
1115                 return -ENOMEM;
1116         }
1117
1118         if (state->bufcnt)
1119                 memcpy(buf, rctx->hdev->xmit_buf, state->bufcnt);
1120
1121         scatterwalk_map_and_copy(buf + state->bufcnt, sg, rctx->offset,
1122                                  min(new_len, rctx->total) - state->bufcnt, 0);
1123         sg_init_table(rctx->sgl, 1);
1124         sg_set_buf(rctx->sgl, buf, new_len);
1125         rctx->sg = rctx->sgl;
1126         state->flags |= HASH_FLAGS_SGS_COPIED;
1127         rctx->nents = 1;
1128         rctx->offset += new_len - state->bufcnt;
1129         state->bufcnt = 0;
1130         rctx->total = new_len;
1131
1132         return 0;
1133 }
1134
1135 static int stm32_hash_align_sgs(struct scatterlist *sg,
1136                                 int nbytes, int bs, bool init, bool final,
1137                                 struct stm32_hash_request_ctx *rctx)
1138 {
1139         struct stm32_hash_state *state = &rctx->state;
1140         struct stm32_hash_dev *hdev = rctx->hdev;
1141         struct scatterlist *sg_tmp = sg;
1142         int offset = rctx->offset;
1143         int new_len;
1144         int n = 0;
1145         int bufcnt = state->bufcnt;
1146         bool secure_ctx = hdev->pdata->context_secured;
1147         bool aligned = true;
1148
1149         if (!sg || !sg->length || !nbytes) {
1150                 if (bufcnt) {
1151                         bufcnt = DIV_ROUND_UP(bufcnt, bs) * bs;
1152                         sg_init_table(rctx->sgl, 1);
1153                         sg_set_buf(rctx->sgl, rctx->hdev->xmit_buf, bufcnt);
1154                         rctx->sg = rctx->sgl;
1155                         rctx->nents = 1;
1156                 }
1157
1158                 return 0;
1159         }
1160
1161         new_len = nbytes;
1162
1163         if (offset)
1164                 aligned = false;
1165
1166         if (final) {
1167                 new_len = DIV_ROUND_UP(new_len, bs) * bs;
1168         } else {
1169                 new_len = (new_len - 1) / bs * bs; // return n block - 1 block
1170
1171                 /*
1172                  * Context save in some version of HASH IP can only be done when the
1173                  * FIFO is ready to get a new block. This implies to send n block plus a
1174                  * 32 bit word in the first DMA send.
1175                  */
1176                 if (init && secure_ctx) {
1177                         new_len += sizeof(u32);
1178                         if (unlikely(new_len > nbytes))
1179                                 new_len -= bs;
1180                 }
1181         }
1182
1183         if (!new_len)
1184                 return 0;
1185
1186         if (nbytes != new_len)
1187                 aligned = false;
1188
1189         while (nbytes > 0 && sg_tmp) {
1190                 n++;
1191
1192                 if (bufcnt) {
1193                         if (!IS_ALIGNED(bufcnt, bs)) {
1194                                 aligned = false;
1195                                 break;
1196                         }
1197                         nbytes -= bufcnt;
1198                         bufcnt = 0;
1199                         if (!nbytes)
1200                                 aligned = false;
1201
1202                         continue;
1203                 }
1204
1205                 if (offset < sg_tmp->length) {
1206                         if (!IS_ALIGNED(offset + sg_tmp->offset, 4)) {
1207                                 aligned = false;
1208                                 break;
1209                         }
1210
1211                         if (!IS_ALIGNED(sg_tmp->length - offset, bs)) {
1212                                 aligned = false;
1213                                 break;
1214                         }
1215                 }
1216
1217                 if (offset) {
1218                         offset -= sg_tmp->length;
1219                         if (offset < 0) {
1220                                 nbytes += offset;
1221                                 offset = 0;
1222                         }
1223                 } else {
1224                         nbytes -= sg_tmp->length;
1225                 }
1226
1227                 sg_tmp = sg_next(sg_tmp);
1228
1229                 if (nbytes < 0) {
1230                         aligned = false;
1231                         break;
1232                 }
1233         }
1234
1235         if (!aligned)
1236                 return stm32_hash_copy_sgs(rctx, sg, bs, new_len);
1237
1238         rctx->total = new_len;
1239         rctx->offset += new_len;
1240         rctx->nents = n;
1241         if (state->bufcnt) {
1242                 sg_init_table(rctx->sgl, 2);
1243                 sg_set_buf(rctx->sgl, rctx->hdev->xmit_buf, state->bufcnt);
1244                 sg_chain(rctx->sgl, 2, sg);
1245                 rctx->sg = rctx->sgl;
1246         } else {
1247                 rctx->sg = sg;
1248         }
1249
1250         return 0;
1251 }
1252
1253 static int stm32_hash_prepare_request(struct ahash_request *req)
1254 {
1255         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1256         struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
1257         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1258         struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
1259         struct stm32_hash_state *state = &rctx->state;
1260         unsigned int nbytes;
1261         int ret, hash_later, bs;
1262         bool update = rctx->op & HASH_OP_UPDATE;
1263         bool init = !(state->flags & HASH_FLAGS_INIT);
1264         bool finup = state->flags & HASH_FLAGS_FINUP;
1265         bool final = state->flags & HASH_FLAGS_FINAL;
1266
1267         if (!hdev->dma_lch || state->flags & HASH_FLAGS_CPU)
1268                 return 0;
1269
1270         bs = crypto_ahash_blocksize(tfm);
1271
1272         nbytes = state->bufcnt;
1273
1274         /*
1275          * In case of update request nbytes must correspond to the content of the
1276          * buffer + the offset minus the content of the request already in the
1277          * buffer.
1278          */
1279         if (update || finup)
1280                 nbytes += req->nbytes - rctx->offset;
1281
1282         dev_dbg(hdev->dev,
1283                 "%s: nbytes=%d, bs=%d, total=%d, offset=%d, bufcnt=%d\n",
1284                 __func__, nbytes, bs, rctx->total, rctx->offset, state->bufcnt);
1285
1286         if (!nbytes)
1287                 return 0;
1288
1289         rctx->total = nbytes;
1290
1291         if (update && req->nbytes && (!IS_ALIGNED(state->bufcnt, bs))) {
1292                 int len = bs - state->bufcnt % bs;
1293
1294                 if (len > req->nbytes)
1295                         len = req->nbytes;
1296                 scatterwalk_map_and_copy(state->buffer + state->bufcnt, req->src,
1297                                          0, len, 0);
1298                 state->bufcnt += len;
1299                 rctx->offset = len;
1300         }
1301
1302         /* copy buffer in a temporary one that is used for sg alignment */
1303         if (state->bufcnt)
1304                 memcpy(hdev->xmit_buf, state->buffer, state->bufcnt);
1305
1306         ret = stm32_hash_align_sgs(req->src, nbytes, bs, init, final, rctx);
1307         if (ret)
1308                 return ret;
1309
1310         hash_later = nbytes - rctx->total;
1311         if (hash_later < 0)
1312                 hash_later = 0;
1313
1314         if (hash_later && hash_later <= state->blocklen) {
1315                 scatterwalk_map_and_copy(state->buffer,
1316                                          req->src,
1317                                          req->nbytes - hash_later,
1318                                          hash_later, 0);
1319
1320                 state->bufcnt = hash_later;
1321         } else {
1322                 state->bufcnt = 0;
1323         }
1324
1325         if (hash_later > state->blocklen) {
1326                 /* FIXME: add support of this case */
1327                 pr_err("Buffer contains more than one block.\n");
1328                 return -ENOMEM;
1329         }
1330
1331         rctx->total = min(nbytes, rctx->total);
1332
1333         return 0;
1334 }
1335
1336 static void stm32_hash_unprepare_request(struct ahash_request *req)
1337 {
1338         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1339         struct stm32_hash_state *state = &rctx->state;
1340         struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
1341         struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
1342         u32 *preg = state->hw_context;
1343         int swap_reg, i;
1344
1345         if (hdev->dma_lch)
1346                 dmaengine_terminate_sync(hdev->dma_lch);
1347
1348         if (state->flags & HASH_FLAGS_SGS_COPIED)
1349                 free_pages((unsigned long)sg_virt(rctx->sg), get_order(rctx->sg->length));
1350
1351         rctx->sg = NULL;
1352         rctx->offset = 0;
1353
1354         state->flags &= ~(HASH_FLAGS_SGS_COPIED);
1355
1356         if (!(hdev->flags & HASH_FLAGS_INIT))
1357                 goto pm_runtime;
1358
1359         state->flags |= HASH_FLAGS_INIT;
1360
1361         if (stm32_hash_wait_busy(hdev)) {
1362                 dev_warn(hdev->dev, "Wait busy failed.");
1363                 return;
1364         }
1365
1366         swap_reg = hash_swap_reg(rctx);
1367
1368         if (!hdev->pdata->ux500)
1369                 *preg++ = stm32_hash_read(hdev, HASH_IMR);
1370         *preg++ = stm32_hash_read(hdev, HASH_STR);
1371         *preg++ = stm32_hash_read(hdev, HASH_CR);
1372         for (i = 0; i < swap_reg; i++)
1373                 *preg++ = stm32_hash_read(hdev, HASH_CSR(i));
1374
1375 pm_runtime:
1376         pm_runtime_mark_last_busy(hdev->dev);
1377         pm_runtime_put_autosuspend(hdev->dev);
1378 }
1379
1380 static int stm32_hash_enqueue(struct ahash_request *req, unsigned int op)
1381 {
1382         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1383         struct stm32_hash_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
1384         struct stm32_hash_dev *hdev = ctx->hdev;
1385
1386         rctx->op = op;
1387
1388         return stm32_hash_handle_queue(hdev, req);
1389 }
1390
1391 static int stm32_hash_update(struct ahash_request *req)
1392 {
1393         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1394         struct stm32_hash_state *state = &rctx->state;
1395
1396         if (!req->nbytes)
1397                 return 0;
1398
1399
1400         if (state->flags & HASH_FLAGS_CPU) {
1401                 rctx->total = req->nbytes;
1402                 rctx->sg = req->src;
1403                 rctx->offset = 0;
1404
1405                 if ((state->bufcnt + rctx->total < state->blocklen)) {
1406                         stm32_hash_append_sg(rctx);
1407                         return 0;
1408                 }
1409         } else { /* DMA mode */
1410                 if (state->bufcnt + req->nbytes <= state->blocklen) {
1411                         scatterwalk_map_and_copy(state->buffer + state->bufcnt, req->src,
1412                                                  0, req->nbytes, 0);
1413                         state->bufcnt += req->nbytes;
1414                         return 0;
1415                 }
1416         }
1417
1418         return stm32_hash_enqueue(req, HASH_OP_UPDATE);
1419 }
1420
1421 static int stm32_hash_final(struct ahash_request *req)
1422 {
1423         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1424         struct stm32_hash_state *state = &rctx->state;
1425
1426         state->flags |= HASH_FLAGS_FINAL;
1427
1428         return stm32_hash_enqueue(req, HASH_OP_FINAL);
1429 }
1430
1431 static int stm32_hash_finup(struct ahash_request *req)
1432 {
1433         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1434         struct stm32_hash_state *state = &rctx->state;
1435
1436         if (!req->nbytes)
1437                 goto out;
1438
1439         state->flags |= HASH_FLAGS_FINUP;
1440
1441         if ((state->flags & HASH_FLAGS_CPU)) {
1442                 rctx->total = req->nbytes;
1443                 rctx->sg = req->src;
1444                 rctx->offset = 0;
1445         }
1446
1447 out:
1448         return stm32_hash_final(req);
1449 }
1450
1451 static int stm32_hash_digest(struct ahash_request *req)
1452 {
1453         return stm32_hash_init(req) ?: stm32_hash_finup(req);
1454 }
1455
1456 static int stm32_hash_export(struct ahash_request *req, void *out)
1457 {
1458         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1459
1460         memcpy(out, &rctx->state, sizeof(rctx->state));
1461
1462         return 0;
1463 }
1464
1465 static int stm32_hash_import(struct ahash_request *req, const void *in)
1466 {
1467         struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1468
1469         stm32_hash_init(req);
1470         memcpy(&rctx->state, in, sizeof(rctx->state));
1471
1472         return 0;
1473 }
1474
1475 static int stm32_hash_setkey(struct crypto_ahash *tfm,
1476                              const u8 *key, unsigned int keylen)
1477 {
1478         struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
1479
1480         if (keylen <= HASH_MAX_KEY_SIZE) {
1481                 memcpy(ctx->key, key, keylen);
1482                 ctx->keylen = keylen;
1483         } else {
1484                 return -ENOMEM;
1485         }
1486
1487         return 0;
1488 }
1489
1490 static int stm32_hash_init_fallback(struct crypto_tfm *tfm)
1491 {
1492         struct stm32_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1493         struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
1494         const char *name = crypto_tfm_alg_name(tfm);
1495         struct crypto_shash *xtfm;
1496
1497         /* The fallback is only needed on Ux500 */
1498         if (!hdev->pdata->ux500)
1499                 return 0;
1500
1501         xtfm = crypto_alloc_shash(name, 0, CRYPTO_ALG_NEED_FALLBACK);
1502         if (IS_ERR(xtfm)) {
1503                 dev_err(hdev->dev, "failed to allocate %s fallback\n",
1504                         name);
1505                 return PTR_ERR(xtfm);
1506         }
1507         dev_info(hdev->dev, "allocated %s fallback\n", name);
1508         ctx->xtfm = xtfm;
1509
1510         return 0;
1511 }
1512
1513 static int stm32_hash_cra_init_algs(struct crypto_tfm *tfm, u32 algs_flags)
1514 {
1515         struct stm32_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1516
1517         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1518                                  sizeof(struct stm32_hash_request_ctx));
1519
1520         ctx->keylen = 0;
1521
1522         if (algs_flags)
1523                 ctx->flags |= algs_flags;
1524
1525         return stm32_hash_init_fallback(tfm);
1526 }
1527
1528 static int stm32_hash_cra_init(struct crypto_tfm *tfm)
1529 {
1530         return stm32_hash_cra_init_algs(tfm, 0);
1531 }
1532
1533 static int stm32_hash_cra_hmac_init(struct crypto_tfm *tfm)
1534 {
1535         return stm32_hash_cra_init_algs(tfm, HASH_FLAGS_HMAC);
1536 }
1537
1538 static int stm32_hash_cra_sha3_init(struct crypto_tfm *tfm)
1539 {
1540         return stm32_hash_cra_init_algs(tfm, HASH_FLAGS_SHA3_MODE);
1541 }
1542
1543 static int stm32_hash_cra_sha3_hmac_init(struct crypto_tfm *tfm)
1544 {
1545         return stm32_hash_cra_init_algs(tfm, HASH_FLAGS_SHA3_MODE |
1546                                         HASH_FLAGS_HMAC);
1547 }
1548
1549 static void stm32_hash_cra_exit(struct crypto_tfm *tfm)
1550 {
1551         struct stm32_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1552
1553         if (ctx->xtfm)
1554                 crypto_free_shash(ctx->xtfm);
1555 }
1556
1557 static irqreturn_t stm32_hash_irq_thread(int irq, void *dev_id)
1558 {
1559         struct stm32_hash_dev *hdev = dev_id;
1560
1561         if (HASH_FLAGS_OUTPUT_READY & hdev->flags) {
1562                 hdev->flags &= ~HASH_FLAGS_OUTPUT_READY;
1563                 goto finish;
1564         }
1565
1566         return IRQ_HANDLED;
1567
1568 finish:
1569         /* Finish current request */
1570         stm32_hash_finish_req(hdev->req, 0);
1571
1572         return IRQ_HANDLED;
1573 }
1574
1575 static irqreturn_t stm32_hash_irq_handler(int irq, void *dev_id)
1576 {
1577         struct stm32_hash_dev *hdev = dev_id;
1578         u32 reg;
1579
1580         reg = stm32_hash_read(hdev, HASH_SR);
1581         if (reg & HASH_SR_OUTPUT_READY) {
1582                 hdev->flags |= HASH_FLAGS_OUTPUT_READY;
1583                 /* Disable IT*/
1584                 stm32_hash_write(hdev, HASH_IMR, 0);
1585                 return IRQ_WAKE_THREAD;
1586         }
1587
1588         return IRQ_NONE;
1589 }
1590
1591 static struct ahash_engine_alg algs_md5[] = {
1592         {
1593                 .base.init = stm32_hash_init,
1594                 .base.update = stm32_hash_update,
1595                 .base.final = stm32_hash_final,
1596                 .base.finup = stm32_hash_finup,
1597                 .base.digest = stm32_hash_digest,
1598                 .base.export = stm32_hash_export,
1599                 .base.import = stm32_hash_import,
1600                 .base.halg = {
1601                         .digestsize = MD5_DIGEST_SIZE,
1602                         .statesize = sizeof(struct stm32_hash_state),
1603                         .base = {
1604                                 .cra_name = "md5",
1605                                 .cra_driver_name = "stm32-md5",
1606                                 .cra_priority = 200,
1607                                 .cra_flags = CRYPTO_ALG_ASYNC |
1608                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
1609                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1610                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1611                                 .cra_init = stm32_hash_cra_init,
1612                                 .cra_exit = stm32_hash_cra_exit,
1613                                 .cra_module = THIS_MODULE,
1614                         }
1615                 },
1616                 .op = {
1617                         .do_one_request = stm32_hash_one_request,
1618                 },
1619         },
1620         {
1621                 .base.init = stm32_hash_init,
1622                 .base.update = stm32_hash_update,
1623                 .base.final = stm32_hash_final,
1624                 .base.finup = stm32_hash_finup,
1625                 .base.digest = stm32_hash_digest,
1626                 .base.export = stm32_hash_export,
1627                 .base.import = stm32_hash_import,
1628                 .base.setkey = stm32_hash_setkey,
1629                 .base.halg = {
1630                         .digestsize = MD5_DIGEST_SIZE,
1631                         .statesize = sizeof(struct stm32_hash_state),
1632                         .base = {
1633                                 .cra_name = "hmac(md5)",
1634                                 .cra_driver_name = "stm32-hmac-md5",
1635                                 .cra_priority = 200,
1636                                 .cra_flags = CRYPTO_ALG_ASYNC |
1637                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
1638                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1639                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1640                                 .cra_init = stm32_hash_cra_hmac_init,
1641                                 .cra_exit = stm32_hash_cra_exit,
1642                                 .cra_module = THIS_MODULE,
1643                         }
1644                 },
1645                 .op = {
1646                         .do_one_request = stm32_hash_one_request,
1647                 },
1648         }
1649 };
1650
1651 static struct ahash_engine_alg algs_sha1[] = {
1652         {
1653                 .base.init = stm32_hash_init,
1654                 .base.update = stm32_hash_update,
1655                 .base.final = stm32_hash_final,
1656                 .base.finup = stm32_hash_finup,
1657                 .base.digest = stm32_hash_digest,
1658                 .base.export = stm32_hash_export,
1659                 .base.import = stm32_hash_import,
1660                 .base.halg = {
1661                         .digestsize = SHA1_DIGEST_SIZE,
1662                         .statesize = sizeof(struct stm32_hash_state),
1663                         .base = {
1664                                 .cra_name = "sha1",
1665                                 .cra_driver_name = "stm32-sha1",
1666                                 .cra_priority = 200,
1667                                 .cra_flags = CRYPTO_ALG_ASYNC |
1668                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
1669                                 .cra_blocksize = SHA1_BLOCK_SIZE,
1670                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1671                                 .cra_init = stm32_hash_cra_init,
1672                                 .cra_exit = stm32_hash_cra_exit,
1673                                 .cra_module = THIS_MODULE,
1674                         }
1675                 },
1676                 .op = {
1677                         .do_one_request = stm32_hash_one_request,
1678                 },
1679         },
1680         {
1681                 .base.init = stm32_hash_init,
1682                 .base.update = stm32_hash_update,
1683                 .base.final = stm32_hash_final,
1684                 .base.finup = stm32_hash_finup,
1685                 .base.digest = stm32_hash_digest,
1686                 .base.export = stm32_hash_export,
1687                 .base.import = stm32_hash_import,
1688                 .base.setkey = stm32_hash_setkey,
1689                 .base.halg = {
1690                         .digestsize = SHA1_DIGEST_SIZE,
1691                         .statesize = sizeof(struct stm32_hash_state),
1692                         .base = {
1693                                 .cra_name = "hmac(sha1)",
1694                                 .cra_driver_name = "stm32-hmac-sha1",
1695                                 .cra_priority = 200,
1696                                 .cra_flags = CRYPTO_ALG_ASYNC |
1697                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
1698                                 .cra_blocksize = SHA1_BLOCK_SIZE,
1699                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1700                                 .cra_init = stm32_hash_cra_hmac_init,
1701                                 .cra_exit = stm32_hash_cra_exit,
1702                                 .cra_module = THIS_MODULE,
1703                         }
1704                 },
1705                 .op = {
1706                         .do_one_request = stm32_hash_one_request,
1707                 },
1708         },
1709 };
1710
1711 static struct ahash_engine_alg algs_sha224[] = {
1712         {
1713                 .base.init = stm32_hash_init,
1714                 .base.update = stm32_hash_update,
1715                 .base.final = stm32_hash_final,
1716                 .base.finup = stm32_hash_finup,
1717                 .base.digest = stm32_hash_digest,
1718                 .base.export = stm32_hash_export,
1719                 .base.import = stm32_hash_import,
1720                 .base.halg = {
1721                         .digestsize = SHA224_DIGEST_SIZE,
1722                         .statesize = sizeof(struct stm32_hash_state),
1723                         .base = {
1724                                 .cra_name = "sha224",
1725                                 .cra_driver_name = "stm32-sha224",
1726                                 .cra_priority = 200,
1727                                 .cra_flags = CRYPTO_ALG_ASYNC |
1728                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
1729                                 .cra_blocksize = SHA224_BLOCK_SIZE,
1730                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1731                                 .cra_init = stm32_hash_cra_init,
1732                                 .cra_exit = stm32_hash_cra_exit,
1733                                 .cra_module = THIS_MODULE,
1734                         }
1735                 },
1736                 .op = {
1737                         .do_one_request = stm32_hash_one_request,
1738                 },
1739         },
1740         {
1741                 .base.init = stm32_hash_init,
1742                 .base.update = stm32_hash_update,
1743                 .base.final = stm32_hash_final,
1744                 .base.finup = stm32_hash_finup,
1745                 .base.digest = stm32_hash_digest,
1746                 .base.setkey = stm32_hash_setkey,
1747                 .base.export = stm32_hash_export,
1748                 .base.import = stm32_hash_import,
1749                 .base.halg = {
1750                         .digestsize = SHA224_DIGEST_SIZE,
1751                         .statesize = sizeof(struct stm32_hash_state),
1752                         .base = {
1753                                 .cra_name = "hmac(sha224)",
1754                                 .cra_driver_name = "stm32-hmac-sha224",
1755                                 .cra_priority = 200,
1756                                 .cra_flags = CRYPTO_ALG_ASYNC |
1757                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
1758                                 .cra_blocksize = SHA224_BLOCK_SIZE,
1759                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1760                                 .cra_init = stm32_hash_cra_hmac_init,
1761                                 .cra_exit = stm32_hash_cra_exit,
1762                                 .cra_module = THIS_MODULE,
1763                         }
1764                 },
1765                 .op = {
1766                         .do_one_request = stm32_hash_one_request,
1767                 },
1768         },
1769 };
1770
1771 static struct ahash_engine_alg algs_sha256[] = {
1772         {
1773                 .base.init = stm32_hash_init,
1774                 .base.update = stm32_hash_update,
1775                 .base.final = stm32_hash_final,
1776                 .base.finup = stm32_hash_finup,
1777                 .base.digest = stm32_hash_digest,
1778                 .base.export = stm32_hash_export,
1779                 .base.import = stm32_hash_import,
1780                 .base.halg = {
1781                         .digestsize = SHA256_DIGEST_SIZE,
1782                         .statesize = sizeof(struct stm32_hash_state),
1783                         .base = {
1784                                 .cra_name = "sha256",
1785                                 .cra_driver_name = "stm32-sha256",
1786                                 .cra_priority = 200,
1787                                 .cra_flags = CRYPTO_ALG_ASYNC |
1788                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
1789                                 .cra_blocksize = SHA256_BLOCK_SIZE,
1790                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1791                                 .cra_init = stm32_hash_cra_init,
1792                                 .cra_exit = stm32_hash_cra_exit,
1793                                 .cra_module = THIS_MODULE,
1794                         }
1795                 },
1796                 .op = {
1797                         .do_one_request = stm32_hash_one_request,
1798                 },
1799         },
1800         {
1801                 .base.init = stm32_hash_init,
1802                 .base.update = stm32_hash_update,
1803                 .base.final = stm32_hash_final,
1804                 .base.finup = stm32_hash_finup,
1805                 .base.digest = stm32_hash_digest,
1806                 .base.export = stm32_hash_export,
1807                 .base.import = stm32_hash_import,
1808                 .base.setkey = stm32_hash_setkey,
1809                 .base.halg = {
1810                         .digestsize = SHA256_DIGEST_SIZE,
1811                         .statesize = sizeof(struct stm32_hash_state),
1812                         .base = {
1813                                 .cra_name = "hmac(sha256)",
1814                                 .cra_driver_name = "stm32-hmac-sha256",
1815                                 .cra_priority = 200,
1816                                 .cra_flags = CRYPTO_ALG_ASYNC |
1817                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
1818                                 .cra_blocksize = SHA256_BLOCK_SIZE,
1819                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1820                                 .cra_init = stm32_hash_cra_hmac_init,
1821                                 .cra_exit = stm32_hash_cra_exit,
1822                                 .cra_module = THIS_MODULE,
1823                         }
1824                 },
1825                 .op = {
1826                         .do_one_request = stm32_hash_one_request,
1827                 },
1828         },
1829 };
1830
1831 static struct ahash_engine_alg algs_sha384_sha512[] = {
1832         {
1833                 .base.init = stm32_hash_init,
1834                 .base.update = stm32_hash_update,
1835                 .base.final = stm32_hash_final,
1836                 .base.finup = stm32_hash_finup,
1837                 .base.digest = stm32_hash_digest,
1838                 .base.export = stm32_hash_export,
1839                 .base.import = stm32_hash_import,
1840                 .base.halg = {
1841                         .digestsize = SHA384_DIGEST_SIZE,
1842                         .statesize = sizeof(struct stm32_hash_state),
1843                         .base = {
1844                                 .cra_name = "sha384",
1845                                 .cra_driver_name = "stm32-sha384",
1846                                 .cra_priority = 200,
1847                                 .cra_flags = CRYPTO_ALG_ASYNC |
1848                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
1849                                 .cra_blocksize = SHA384_BLOCK_SIZE,
1850                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1851                                 .cra_init = stm32_hash_cra_init,
1852                                 .cra_exit = stm32_hash_cra_exit,
1853                                 .cra_module = THIS_MODULE,
1854                         }
1855                 },
1856                 .op = {
1857                         .do_one_request = stm32_hash_one_request,
1858                 },
1859         },
1860         {
1861                 .base.init = stm32_hash_init,
1862                 .base.update = stm32_hash_update,
1863                 .base.final = stm32_hash_final,
1864                 .base.finup = stm32_hash_finup,
1865                 .base.digest = stm32_hash_digest,
1866                 .base.setkey = stm32_hash_setkey,
1867                 .base.export = stm32_hash_export,
1868                 .base.import = stm32_hash_import,
1869                 .base.halg = {
1870                         .digestsize = SHA384_DIGEST_SIZE,
1871                         .statesize = sizeof(struct stm32_hash_state),
1872                         .base = {
1873                                 .cra_name = "hmac(sha384)",
1874                                 .cra_driver_name = "stm32-hmac-sha384",
1875                                 .cra_priority = 200,
1876                                 .cra_flags = CRYPTO_ALG_ASYNC |
1877                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
1878                                 .cra_blocksize = SHA384_BLOCK_SIZE,
1879                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1880                                 .cra_init = stm32_hash_cra_hmac_init,
1881                                 .cra_exit = stm32_hash_cra_exit,
1882                                 .cra_module = THIS_MODULE,
1883                         }
1884                 },
1885                 .op = {
1886                         .do_one_request = stm32_hash_one_request,
1887                 },
1888         },
1889         {
1890                 .base.init = stm32_hash_init,
1891                 .base.update = stm32_hash_update,
1892                 .base.final = stm32_hash_final,
1893                 .base.finup = stm32_hash_finup,
1894                 .base.digest = stm32_hash_digest,
1895                 .base.export = stm32_hash_export,
1896                 .base.import = stm32_hash_import,
1897                 .base.halg = {
1898                         .digestsize = SHA512_DIGEST_SIZE,
1899                         .statesize = sizeof(struct stm32_hash_state),
1900                         .base = {
1901                                 .cra_name = "sha512",
1902                                 .cra_driver_name = "stm32-sha512",
1903                                 .cra_priority = 200,
1904                                 .cra_flags = CRYPTO_ALG_ASYNC |
1905                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
1906                                 .cra_blocksize = SHA512_BLOCK_SIZE,
1907                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1908                                 .cra_init = stm32_hash_cra_init,
1909                                 .cra_exit = stm32_hash_cra_exit,
1910                                 .cra_module = THIS_MODULE,
1911                         }
1912                 },
1913                 .op = {
1914                         .do_one_request = stm32_hash_one_request,
1915                 },
1916         },
1917         {
1918                 .base.init = stm32_hash_init,
1919                 .base.update = stm32_hash_update,
1920                 .base.final = stm32_hash_final,
1921                 .base.finup = stm32_hash_finup,
1922                 .base.digest = stm32_hash_digest,
1923                 .base.export = stm32_hash_export,
1924                 .base.import = stm32_hash_import,
1925                 .base.setkey = stm32_hash_setkey,
1926                 .base.halg = {
1927                         .digestsize = SHA512_DIGEST_SIZE,
1928                         .statesize = sizeof(struct stm32_hash_state),
1929                         .base = {
1930                                 .cra_name = "hmac(sha512)",
1931                                 .cra_driver_name = "stm32-hmac-sha512",
1932                                 .cra_priority = 200,
1933                                 .cra_flags = CRYPTO_ALG_ASYNC |
1934                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
1935                                 .cra_blocksize = SHA512_BLOCK_SIZE,
1936                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1937                                 .cra_init = stm32_hash_cra_hmac_init,
1938                                 .cra_exit = stm32_hash_cra_exit,
1939                                 .cra_module = THIS_MODULE,
1940                         }
1941                 },
1942                 .op = {
1943                         .do_one_request = stm32_hash_one_request,
1944                 },
1945         },
1946 };
1947
1948 static struct ahash_engine_alg algs_sha3[] = {
1949         {
1950                 .base.init = stm32_hash_init,
1951                 .base.update = stm32_hash_update,
1952                 .base.final = stm32_hash_final,
1953                 .base.finup = stm32_hash_finup,
1954                 .base.digest = stm32_hash_digest,
1955                 .base.export = stm32_hash_export,
1956                 .base.import = stm32_hash_import,
1957                 .base.halg = {
1958                         .digestsize = SHA3_224_DIGEST_SIZE,
1959                         .statesize = sizeof(struct stm32_hash_state),
1960                         .base = {
1961                                 .cra_name = "sha3-224",
1962                                 .cra_driver_name = "stm32-sha3-224",
1963                                 .cra_priority = 200,
1964                                 .cra_flags = CRYPTO_ALG_ASYNC |
1965                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
1966                                 .cra_blocksize = SHA3_224_BLOCK_SIZE,
1967                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1968                                 .cra_init = stm32_hash_cra_sha3_init,
1969                                 .cra_exit = stm32_hash_cra_exit,
1970                                 .cra_module = THIS_MODULE,
1971                         }
1972                 },
1973                 .op = {
1974                         .do_one_request = stm32_hash_one_request,
1975                 },
1976         },
1977         {
1978                 .base.init = stm32_hash_init,
1979                 .base.update = stm32_hash_update,
1980                 .base.final = stm32_hash_final,
1981                 .base.finup = stm32_hash_finup,
1982                 .base.digest = stm32_hash_digest,
1983                 .base.export = stm32_hash_export,
1984                 .base.import = stm32_hash_import,
1985                 .base.setkey = stm32_hash_setkey,
1986                 .base.halg = {
1987                         .digestsize = SHA3_224_DIGEST_SIZE,
1988                         .statesize = sizeof(struct stm32_hash_state),
1989                         .base = {
1990                                 .cra_name = "hmac(sha3-224)",
1991                                 .cra_driver_name = "stm32-hmac-sha3-224",
1992                                 .cra_priority = 200,
1993                                 .cra_flags = CRYPTO_ALG_ASYNC |
1994                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
1995                                 .cra_blocksize = SHA3_224_BLOCK_SIZE,
1996                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
1997                                 .cra_init = stm32_hash_cra_sha3_hmac_init,
1998                                 .cra_exit = stm32_hash_cra_exit,
1999                                 .cra_module = THIS_MODULE,
2000                         }
2001                 },
2002                 .op = {
2003                         .do_one_request = stm32_hash_one_request,
2004                 },
2005         },
2006         {
2007                 .base.init = stm32_hash_init,
2008                 .base.update = stm32_hash_update,
2009                 .base.final = stm32_hash_final,
2010                 .base.finup = stm32_hash_finup,
2011                 .base.digest = stm32_hash_digest,
2012                 .base.export = stm32_hash_export,
2013                 .base.import = stm32_hash_import,
2014                 .base.halg = {
2015                         .digestsize = SHA3_256_DIGEST_SIZE,
2016                         .statesize = sizeof(struct stm32_hash_state),
2017                         .base = {
2018                                 .cra_name = "sha3-256",
2019                                 .cra_driver_name = "stm32-sha3-256",
2020                                 .cra_priority = 200,
2021                                 .cra_flags = CRYPTO_ALG_ASYNC |
2022                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
2023                                 .cra_blocksize = SHA3_256_BLOCK_SIZE,
2024                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
2025                                 .cra_init = stm32_hash_cra_sha3_init,
2026                                 .cra_exit = stm32_hash_cra_exit,
2027                                 .cra_module = THIS_MODULE,
2028                         }
2029                 },
2030                 .op = {
2031                         .do_one_request = stm32_hash_one_request,
2032                 },
2033         },
2034         {
2035                 .base.init = stm32_hash_init,
2036                 .base.update = stm32_hash_update,
2037                 .base.final = stm32_hash_final,
2038                 .base.finup = stm32_hash_finup,
2039                 .base.digest = stm32_hash_digest,
2040                 .base.export = stm32_hash_export,
2041                 .base.import = stm32_hash_import,
2042                 .base.setkey = stm32_hash_setkey,
2043                 .base.halg = {
2044                         .digestsize = SHA3_256_DIGEST_SIZE,
2045                         .statesize = sizeof(struct stm32_hash_state),
2046                         .base = {
2047                                 .cra_name = "hmac(sha3-256)",
2048                                 .cra_driver_name = "stm32-hmac-sha3-256",
2049                                 .cra_priority = 200,
2050                                 .cra_flags = CRYPTO_ALG_ASYNC |
2051                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
2052                                 .cra_blocksize = SHA3_256_BLOCK_SIZE,
2053                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
2054                                 .cra_init = stm32_hash_cra_sha3_hmac_init,
2055                                 .cra_exit = stm32_hash_cra_exit,
2056                                 .cra_module = THIS_MODULE,
2057                         }
2058                 },
2059                 .op = {
2060                         .do_one_request = stm32_hash_one_request,
2061                 },
2062         },
2063         {
2064                 .base.init = stm32_hash_init,
2065                 .base.update = stm32_hash_update,
2066                 .base.final = stm32_hash_final,
2067                 .base.finup = stm32_hash_finup,
2068                 .base.digest = stm32_hash_digest,
2069                 .base.export = stm32_hash_export,
2070                 .base.import = stm32_hash_import,
2071                 .base.halg = {
2072                         .digestsize = SHA3_384_DIGEST_SIZE,
2073                         .statesize = sizeof(struct stm32_hash_state),
2074                         .base = {
2075                                 .cra_name = "sha3-384",
2076                                 .cra_driver_name = "stm32-sha3-384",
2077                                 .cra_priority = 200,
2078                                 .cra_flags = CRYPTO_ALG_ASYNC |
2079                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
2080                                 .cra_blocksize = SHA3_384_BLOCK_SIZE,
2081                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
2082                                 .cra_init = stm32_hash_cra_sha3_init,
2083                                 .cra_exit = stm32_hash_cra_exit,
2084                                 .cra_module = THIS_MODULE,
2085                         }
2086                 },
2087                 .op = {
2088                         .do_one_request = stm32_hash_one_request,
2089                 },
2090         },
2091         {
2092                 .base.init = stm32_hash_init,
2093                 .base.update = stm32_hash_update,
2094                 .base.final = stm32_hash_final,
2095                 .base.finup = stm32_hash_finup,
2096                 .base.digest = stm32_hash_digest,
2097                 .base.export = stm32_hash_export,
2098                 .base.import = stm32_hash_import,
2099                 .base.setkey = stm32_hash_setkey,
2100                 .base.halg = {
2101                         .digestsize = SHA3_384_DIGEST_SIZE,
2102                         .statesize = sizeof(struct stm32_hash_state),
2103                         .base = {
2104                                 .cra_name = "hmac(sha3-384)",
2105                                 .cra_driver_name = "stm32-hmac-sha3-384",
2106                                 .cra_priority = 200,
2107                                 .cra_flags = CRYPTO_ALG_ASYNC |
2108                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
2109                                 .cra_blocksize = SHA3_384_BLOCK_SIZE,
2110                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
2111                                 .cra_init = stm32_hash_cra_sha3_hmac_init,
2112                                 .cra_exit = stm32_hash_cra_exit,
2113                                 .cra_module = THIS_MODULE,
2114                         }
2115                 },
2116                 .op = {
2117                         .do_one_request = stm32_hash_one_request,
2118                 },
2119         },
2120         {
2121                 .base.init = stm32_hash_init,
2122                 .base.update = stm32_hash_update,
2123                 .base.final = stm32_hash_final,
2124                 .base.finup = stm32_hash_finup,
2125                 .base.digest = stm32_hash_digest,
2126                 .base.export = stm32_hash_export,
2127                 .base.import = stm32_hash_import,
2128                 .base.halg = {
2129                         .digestsize = SHA3_512_DIGEST_SIZE,
2130                         .statesize = sizeof(struct stm32_hash_state),
2131                         .base = {
2132                                 .cra_name = "sha3-512",
2133                                 .cra_driver_name = "stm32-sha3-512",
2134                                 .cra_priority = 200,
2135                                 .cra_flags = CRYPTO_ALG_ASYNC |
2136                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
2137                                 .cra_blocksize = SHA3_512_BLOCK_SIZE,
2138                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
2139                                 .cra_init = stm32_hash_cra_sha3_init,
2140                                 .cra_exit = stm32_hash_cra_exit,
2141                                 .cra_module = THIS_MODULE,
2142                         }
2143                 },
2144                 .op = {
2145                         .do_one_request = stm32_hash_one_request,
2146                 },
2147         },
2148         {
2149                 .base.init = stm32_hash_init,
2150                 .base.update = stm32_hash_update,
2151                 .base.final = stm32_hash_final,
2152                 .base.finup = stm32_hash_finup,
2153                 .base.digest = stm32_hash_digest,
2154                 .base.export = stm32_hash_export,
2155                 .base.import = stm32_hash_import,
2156                 .base.setkey = stm32_hash_setkey,
2157                 .base.halg = {
2158                         .digestsize = SHA3_512_DIGEST_SIZE,
2159                         .statesize = sizeof(struct stm32_hash_state),
2160                         .base = {
2161                                 .cra_name = "hmac(sha3-512)",
2162                                 .cra_driver_name = "stm32-hmac-sha3-512",
2163                                 .cra_priority = 200,
2164                                 .cra_flags = CRYPTO_ALG_ASYNC |
2165                                         CRYPTO_ALG_KERN_DRIVER_ONLY,
2166                                 .cra_blocksize = SHA3_512_BLOCK_SIZE,
2167                                 .cra_ctxsize = sizeof(struct stm32_hash_ctx),
2168                                 .cra_init = stm32_hash_cra_sha3_hmac_init,
2169                                 .cra_exit = stm32_hash_cra_exit,
2170                                 .cra_module = THIS_MODULE,
2171                         }
2172                 },
2173                 .op = {
2174                         .do_one_request = stm32_hash_one_request,
2175                 },
2176         }
2177 };
2178
2179 static int stm32_hash_register_algs(struct stm32_hash_dev *hdev)
2180 {
2181         unsigned int i, j;
2182         int err;
2183
2184         for (i = 0; i < hdev->pdata->algs_info_size; i++) {
2185                 for (j = 0; j < hdev->pdata->algs_info[i].size; j++) {
2186                         err = crypto_engine_register_ahash(
2187                                 &hdev->pdata->algs_info[i].algs_list[j]);
2188                         if (err)
2189                                 goto err_algs;
2190                 }
2191         }
2192
2193         return 0;
2194 err_algs:
2195         dev_err(hdev->dev, "Algo %d : %d failed\n", i, j);
2196         for (; i--; ) {
2197                 for (; j--;)
2198                         crypto_engine_unregister_ahash(
2199                                 &hdev->pdata->algs_info[i].algs_list[j]);
2200         }
2201
2202         return err;
2203 }
2204
2205 static int stm32_hash_unregister_algs(struct stm32_hash_dev *hdev)
2206 {
2207         unsigned int i, j;
2208
2209         for (i = 0; i < hdev->pdata->algs_info_size; i++) {
2210                 for (j = 0; j < hdev->pdata->algs_info[i].size; j++)
2211                         crypto_engine_unregister_ahash(
2212                                 &hdev->pdata->algs_info[i].algs_list[j]);
2213         }
2214
2215         return 0;
2216 }
2217
2218 static struct stm32_hash_algs_info stm32_hash_algs_info_ux500[] = {
2219         {
2220                 .algs_list      = algs_sha1,
2221                 .size           = ARRAY_SIZE(algs_sha1),
2222         },
2223         {
2224                 .algs_list      = algs_sha256,
2225                 .size           = ARRAY_SIZE(algs_sha256),
2226         },
2227 };
2228
2229 static const struct stm32_hash_pdata stm32_hash_pdata_ux500 = {
2230         .alg_shift      = 7,
2231         .algs_info      = stm32_hash_algs_info_ux500,
2232         .algs_info_size = ARRAY_SIZE(stm32_hash_algs_info_ux500),
2233         .broken_emptymsg = true,
2234         .ux500          = true,
2235 };
2236
2237 static struct stm32_hash_algs_info stm32_hash_algs_info_stm32f4[] = {
2238         {
2239                 .algs_list      = algs_md5,
2240                 .size           = ARRAY_SIZE(algs_md5),
2241         },
2242         {
2243                 .algs_list      = algs_sha1,
2244                 .size           = ARRAY_SIZE(algs_sha1),
2245         },
2246 };
2247
2248 static const struct stm32_hash_pdata stm32_hash_pdata_stm32f4 = {
2249         .alg_shift      = 7,
2250         .algs_info      = stm32_hash_algs_info_stm32f4,
2251         .algs_info_size = ARRAY_SIZE(stm32_hash_algs_info_stm32f4),
2252         .has_sr         = true,
2253         .has_mdmat      = true,
2254 };
2255
2256 static struct stm32_hash_algs_info stm32_hash_algs_info_stm32f7[] = {
2257         {
2258                 .algs_list      = algs_md5,
2259                 .size           = ARRAY_SIZE(algs_md5),
2260         },
2261         {
2262                 .algs_list      = algs_sha1,
2263                 .size           = ARRAY_SIZE(algs_sha1),
2264         },
2265         {
2266                 .algs_list      = algs_sha224,
2267                 .size           = ARRAY_SIZE(algs_sha224),
2268         },
2269         {
2270                 .algs_list      = algs_sha256,
2271                 .size           = ARRAY_SIZE(algs_sha256),
2272         },
2273 };
2274
2275 static const struct stm32_hash_pdata stm32_hash_pdata_stm32f7 = {
2276         .alg_shift      = 7,
2277         .algs_info      = stm32_hash_algs_info_stm32f7,
2278         .algs_info_size = ARRAY_SIZE(stm32_hash_algs_info_stm32f7),
2279         .has_sr         = true,
2280         .has_mdmat      = true,
2281 };
2282
2283 static struct stm32_hash_algs_info stm32_hash_algs_info_stm32mp13[] = {
2284         {
2285                 .algs_list      = algs_sha1,
2286                 .size           = ARRAY_SIZE(algs_sha1),
2287         },
2288         {
2289                 .algs_list      = algs_sha224,
2290                 .size           = ARRAY_SIZE(algs_sha224),
2291         },
2292         {
2293                 .algs_list      = algs_sha256,
2294                 .size           = ARRAY_SIZE(algs_sha256),
2295         },
2296         {
2297                 .algs_list      = algs_sha384_sha512,
2298                 .size           = ARRAY_SIZE(algs_sha384_sha512),
2299         },
2300         {
2301                 .algs_list      = algs_sha3,
2302                 .size           = ARRAY_SIZE(algs_sha3),
2303         },
2304 };
2305
2306 static const struct stm32_hash_pdata stm32_hash_pdata_stm32mp13 = {
2307         .alg_shift      = 17,
2308         .algs_info      = stm32_hash_algs_info_stm32mp13,
2309         .algs_info_size = ARRAY_SIZE(stm32_hash_algs_info_stm32mp13),
2310         .has_sr         = true,
2311         .has_mdmat      = true,
2312         .context_secured = true,
2313 };
2314
2315 static const struct of_device_id stm32_hash_of_match[] = {
2316         { .compatible = "stericsson,ux500-hash", .data = &stm32_hash_pdata_ux500 },
2317         { .compatible = "st,stm32f456-hash", .data = &stm32_hash_pdata_stm32f4 },
2318         { .compatible = "st,stm32f756-hash", .data = &stm32_hash_pdata_stm32f7 },
2319         { .compatible = "st,stm32mp13-hash", .data = &stm32_hash_pdata_stm32mp13 },
2320         {},
2321 };
2322
2323 MODULE_DEVICE_TABLE(of, stm32_hash_of_match);
2324
2325 static int stm32_hash_get_of_match(struct stm32_hash_dev *hdev,
2326                                    struct device *dev)
2327 {
2328         hdev->pdata = of_device_get_match_data(dev);
2329         if (!hdev->pdata) {
2330                 dev_err(dev, "no compatible OF match\n");
2331                 return -EINVAL;
2332         }
2333
2334         return 0;
2335 }
2336
2337 static int stm32_hash_probe(struct platform_device *pdev)
2338 {
2339         struct stm32_hash_dev *hdev;
2340         struct device *dev = &pdev->dev;
2341         struct resource *res;
2342         int ret, irq;
2343
2344         hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL);
2345         if (!hdev)
2346                 return -ENOMEM;
2347
2348         hdev->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2349         if (IS_ERR(hdev->io_base))
2350                 return PTR_ERR(hdev->io_base);
2351
2352         hdev->phys_base = res->start;
2353
2354         ret = stm32_hash_get_of_match(hdev, dev);
2355         if (ret)
2356                 return ret;
2357
2358         irq = platform_get_irq_optional(pdev, 0);
2359         if (irq < 0 && irq != -ENXIO)
2360                 return irq;
2361
2362         if (irq > 0) {
2363                 ret = devm_request_threaded_irq(dev, irq,
2364                                                 stm32_hash_irq_handler,
2365                                                 stm32_hash_irq_thread,
2366                                                 IRQF_ONESHOT,
2367                                                 dev_name(dev), hdev);
2368                 if (ret) {
2369                         dev_err(dev, "Cannot grab IRQ\n");
2370                         return ret;
2371                 }
2372         } else {
2373                 dev_info(dev, "No IRQ, use polling mode\n");
2374                 hdev->polled = true;
2375         }
2376
2377         hdev->clk = devm_clk_get(&pdev->dev, NULL);
2378         if (IS_ERR(hdev->clk))
2379                 return dev_err_probe(dev, PTR_ERR(hdev->clk),
2380                                      "failed to get clock for hash\n");
2381
2382         ret = clk_prepare_enable(hdev->clk);
2383         if (ret) {
2384                 dev_err(dev, "failed to enable hash clock (%d)\n", ret);
2385                 return ret;
2386         }
2387
2388         pm_runtime_set_autosuspend_delay(dev, HASH_AUTOSUSPEND_DELAY);
2389         pm_runtime_use_autosuspend(dev);
2390
2391         pm_runtime_get_noresume(dev);
2392         pm_runtime_set_active(dev);
2393         pm_runtime_enable(dev);
2394
2395         hdev->rst = devm_reset_control_get(&pdev->dev, NULL);
2396         if (IS_ERR(hdev->rst)) {
2397                 if (PTR_ERR(hdev->rst) == -EPROBE_DEFER) {
2398                         ret = -EPROBE_DEFER;
2399                         goto err_reset;
2400                 }
2401         } else {
2402                 reset_control_assert(hdev->rst);
2403                 udelay(2);
2404                 reset_control_deassert(hdev->rst);
2405         }
2406
2407         hdev->dev = dev;
2408
2409         platform_set_drvdata(pdev, hdev);
2410
2411         ret = stm32_hash_dma_init(hdev);
2412         switch (ret) {
2413         case 0:
2414                 break;
2415         case -ENOENT:
2416         case -ENODEV:
2417                 dev_info(dev, "DMA mode not available\n");
2418                 break;
2419         default:
2420                 dev_err(dev, "DMA init error %d\n", ret);
2421                 goto err_dma;
2422         }
2423
2424         spin_lock(&stm32_hash.lock);
2425         list_add_tail(&hdev->list, &stm32_hash.dev_list);
2426         spin_unlock(&stm32_hash.lock);
2427
2428         /* Initialize crypto engine */
2429         hdev->engine = crypto_engine_alloc_init(dev, 1);
2430         if (!hdev->engine) {
2431                 ret = -ENOMEM;
2432                 goto err_engine;
2433         }
2434
2435         ret = crypto_engine_start(hdev->engine);
2436         if (ret)
2437                 goto err_engine_start;
2438
2439         if (hdev->pdata->ux500)
2440                 /* FIXME: implement DMA mode for Ux500 */
2441                 hdev->dma_mode = 0;
2442         else
2443                 hdev->dma_mode = stm32_hash_read(hdev, HASH_HWCFGR) & HASH_HWCFG_DMA_MASK;
2444
2445         /* Register algos */
2446         ret = stm32_hash_register_algs(hdev);
2447         if (ret)
2448                 goto err_algs;
2449
2450         dev_info(dev, "Init HASH done HW ver %x DMA mode %u\n",
2451                  stm32_hash_read(hdev, HASH_VER), hdev->dma_mode);
2452
2453         pm_runtime_put_sync(dev);
2454
2455         return 0;
2456
2457 err_algs:
2458 err_engine_start:
2459         crypto_engine_exit(hdev->engine);
2460 err_engine:
2461         spin_lock(&stm32_hash.lock);
2462         list_del(&hdev->list);
2463         spin_unlock(&stm32_hash.lock);
2464 err_dma:
2465         if (hdev->dma_lch)
2466                 dma_release_channel(hdev->dma_lch);
2467 err_reset:
2468         pm_runtime_disable(dev);
2469         pm_runtime_put_noidle(dev);
2470
2471         clk_disable_unprepare(hdev->clk);
2472
2473         return ret;
2474 }
2475
2476 static void stm32_hash_remove(struct platform_device *pdev)
2477 {
2478         struct stm32_hash_dev *hdev = platform_get_drvdata(pdev);
2479         int ret;
2480
2481         ret = pm_runtime_get_sync(hdev->dev);
2482
2483         stm32_hash_unregister_algs(hdev);
2484
2485         crypto_engine_exit(hdev->engine);
2486
2487         spin_lock(&stm32_hash.lock);
2488         list_del(&hdev->list);
2489         spin_unlock(&stm32_hash.lock);
2490
2491         if (hdev->dma_lch)
2492                 dma_release_channel(hdev->dma_lch);
2493
2494         pm_runtime_disable(hdev->dev);
2495         pm_runtime_put_noidle(hdev->dev);
2496
2497         if (ret >= 0)
2498                 clk_disable_unprepare(hdev->clk);
2499 }
2500
2501 #ifdef CONFIG_PM
2502 static int stm32_hash_runtime_suspend(struct device *dev)
2503 {
2504         struct stm32_hash_dev *hdev = dev_get_drvdata(dev);
2505
2506         clk_disable_unprepare(hdev->clk);
2507
2508         return 0;
2509 }
2510
2511 static int stm32_hash_runtime_resume(struct device *dev)
2512 {
2513         struct stm32_hash_dev *hdev = dev_get_drvdata(dev);
2514         int ret;
2515
2516         ret = clk_prepare_enable(hdev->clk);
2517         if (ret) {
2518                 dev_err(hdev->dev, "Failed to prepare_enable clock\n");
2519                 return ret;
2520         }
2521
2522         return 0;
2523 }
2524 #endif
2525
2526 static const struct dev_pm_ops stm32_hash_pm_ops = {
2527         SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
2528                                 pm_runtime_force_resume)
2529         SET_RUNTIME_PM_OPS(stm32_hash_runtime_suspend,
2530                            stm32_hash_runtime_resume, NULL)
2531 };
2532
2533 static struct platform_driver stm32_hash_driver = {
2534         .probe          = stm32_hash_probe,
2535         .remove         = stm32_hash_remove,
2536         .driver         = {
2537                 .name   = "stm32-hash",
2538                 .pm = &stm32_hash_pm_ops,
2539                 .of_match_table = stm32_hash_of_match,
2540         }
2541 };
2542
2543 module_platform_driver(stm32_hash_driver);
2544
2545 MODULE_DESCRIPTION("STM32 SHA1/SHA2/SHA3 & MD5 (HMAC) hw accelerator driver");
2546 MODULE_AUTHOR("Lionel Debieve <[email protected]>");
2547 MODULE_LICENSE("GPL v2");