drivers/crypto/rockchip/rk3288_crypto_ahash.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Crypto acceleration support for Rockchip RK3288
   4  *
   5  * Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd
   6  *
   7  * Author: Zain Wang <[email protected]>
   8  *
   9  * Some ideas are from marvell/cesa.c and s5p-sss.c driver.
  10  */
  11 #include <linux/device.h>
  12 #include <asm/unaligned.h>
  13 #include <linux/iopoll.h>
  14 #include "rk3288_crypto.h"
  15
  16 /*
  17  * IC can not process zero message hash,
  18  * so we put the fixed hash out when met zero message.
  19  */
  20
  21 static bool rk_ahash_need_fallback(struct ahash_request *req)
  22 {
  23         struct scatterlist *sg;
  24
  25         sg = req->src;
  26         while (sg) {
  27                 if (!IS_ALIGNED(sg->offset, sizeof(u32))) {
  28                         return true;
  29                 }
  30                 if (sg->length % 4) {
  31                         return true;
  32                 }
  33                 sg = sg_next(sg);
  34         }
  35         return false;
  36 }
  37
  38 static int rk_ahash_digest_fb(struct ahash_request *areq)
  39 {
  40         struct rk_ahash_rctx *rctx = ahash_request_ctx(areq);
  41         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
  42         struct rk_ahash_ctx *tfmctx = crypto_ahash_ctx(tfm);
  43         struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
  44         struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.hash);
  45
  46         algt->stat_fb++;
  47
  48         ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
  49         rctx->fallback_req.base.flags = areq->base.flags &
  50                                         CRYPTO_TFM_REQ_MAY_SLEEP;
  51
  52         rctx->fallback_req.nbytes = areq->nbytes;
  53         rctx->fallback_req.src = areq->src;
  54         rctx->fallback_req.result = areq->result;
  55
  56         return crypto_ahash_digest(&rctx->fallback_req);
  57 }
  58
  59 static int zero_message_process(struct ahash_request *req)
  60 {
  61         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  62         int rk_digest_size = crypto_ahash_digestsize(tfm);
  63
  64         switch (rk_digest_size) {
  65         case SHA1_DIGEST_SIZE:
  66                 memcpy(req->result, sha1_zero_message_hash, rk_digest_size);
  67                 break;
  68         case SHA256_DIGEST_SIZE:
  69                 memcpy(req->result, sha256_zero_message_hash, rk_digest_size);
  70                 break;
  71         case MD5_DIGEST_SIZE:
  72                 memcpy(req->result, md5_zero_message_hash, rk_digest_size);
  73                 break;
  74         default:
  75                 return -EINVAL;
  76         }
  77
  78         return 0;
  79 }
  80
  81 static void rk_ahash_reg_init(struct ahash_request *req,
  82                               struct rk_crypto_info *dev)
  83 {
  84         struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
  85         int reg_status;
  86
  87         reg_status = CRYPTO_READ(dev, RK_CRYPTO_CTRL) |
  88                      RK_CRYPTO_HASH_FLUSH | _SBF(0xffff, 16);
  89         CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, reg_status);
  90
  91         reg_status = CRYPTO_READ(dev, RK_CRYPTO_CTRL);
  92         reg_status &= (~RK_CRYPTO_HASH_FLUSH);
  93         reg_status |= _SBF(0xffff, 16);
  94         CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, reg_status);
  95
  96         memset_io(dev->reg + RK_CRYPTO_HASH_DOUT_0, 0, 32);
  97
  98         CRYPTO_WRITE(dev, RK_CRYPTO_INTENA, RK_CRYPTO_HRDMA_ERR_ENA |
  99                                             RK_CRYPTO_HRDMA_DONE_ENA);
 100
 101         CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, RK_CRYPTO_HRDMA_ERR_INT |
 102                                             RK_CRYPTO_HRDMA_DONE_INT);
 103
 104         CRYPTO_WRITE(dev, RK_CRYPTO_HASH_CTRL, rctx->mode |
 105                                                RK_CRYPTO_HASH_SWAP_DO);
 106
 107         CRYPTO_WRITE(dev, RK_CRYPTO_CONF, RK_CRYPTO_BYTESWAP_HRFIFO |
 108                                           RK_CRYPTO_BYTESWAP_BRFIFO |
 109                                           RK_CRYPTO_BYTESWAP_BTFIFO);
 110
 111         CRYPTO_WRITE(dev, RK_CRYPTO_HASH_MSG_LEN, req->nbytes);
 112 }
 113
 114 static int rk_ahash_init(struct ahash_request *req)
 115 {
 116         struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 117         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 118         struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
 119
 120         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 121         rctx->fallback_req.base.flags = req->base.flags &
 122                                         CRYPTO_TFM_REQ_MAY_SLEEP;
 123
 124         return crypto_ahash_init(&rctx->fallback_req);
 125 }
 126
 127 static int rk_ahash_update(struct ahash_request *req)
 128 {
 129         struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 130         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 131         struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
 132
 133         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 134         rctx->fallback_req.base.flags = req->base.flags &
 135                                         CRYPTO_TFM_REQ_MAY_SLEEP;
 136         rctx->fallback_req.nbytes = req->nbytes;
 137         rctx->fallback_req.src = req->src;
 138
 139         return crypto_ahash_update(&rctx->fallback_req);
 140 }
 141
 142 static int rk_ahash_final(struct ahash_request *req)
 143 {
 144         struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 145         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 146         struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
 147
 148         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 149         rctx->fallback_req.base.flags = req->base.flags &
 150                                         CRYPTO_TFM_REQ_MAY_SLEEP;
 151         rctx->fallback_req.result = req->result;
 152
 153         return crypto_ahash_final(&rctx->fallback_req);
 154 }
 155
 156 static int rk_ahash_finup(struct ahash_request *req)
 157 {
 158         struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 159         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 160         struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
 161
 162         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 163         rctx->fallback_req.base.flags = req->base.flags &
 164                                         CRYPTO_TFM_REQ_MAY_SLEEP;
 165
 166         rctx->fallback_req.nbytes = req->nbytes;
 167         rctx->fallback_req.src = req->src;
 168         rctx->fallback_req.result = req->result;
 169
 170         return crypto_ahash_finup(&rctx->fallback_req);
 171 }
 172
 173 static int rk_ahash_import(struct ahash_request *req, const void *in)
 174 {
 175         struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 176         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 177         struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
 178
 179         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 180         rctx->fallback_req.base.flags = req->base.flags &
 181                                         CRYPTO_TFM_REQ_MAY_SLEEP;
 182
 183         return crypto_ahash_import(&rctx->fallback_req, in);
 184 }
 185
 186 static int rk_ahash_export(struct ahash_request *req, void *out)
 187 {
 188         struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 189         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 190         struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
 191
 192         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 193         rctx->fallback_req.base.flags = req->base.flags &
 194                                         CRYPTO_TFM_REQ_MAY_SLEEP;
 195
 196         return crypto_ahash_export(&rctx->fallback_req, out);
 197 }
 198
 199 static int rk_ahash_digest(struct ahash_request *req)
 200 {
 201         struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 202         struct rk_crypto_info *dev;
 203         struct crypto_engine *engine;
 204
 205         if (rk_ahash_need_fallback(req))
 206                 return rk_ahash_digest_fb(req);
 207
 208         if (!req->nbytes)
 209                 return zero_message_process(req);
 210
 211         dev = get_rk_crypto();
 212
 213         rctx->dev = dev;
 214         engine = dev->engine;
 215
 216         return crypto_transfer_hash_request_to_engine(engine, req);
 217 }
 218
 219 static void crypto_ahash_dma_start(struct rk_crypto_info *dev, struct scatterlist *sg)
 220 {
 221         CRYPTO_WRITE(dev, RK_CRYPTO_HRDMAS, sg_dma_address(sg));
 222         CRYPTO_WRITE(dev, RK_CRYPTO_HRDMAL, sg_dma_len(sg) / 4);
 223         CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, RK_CRYPTO_HASH_START |
 224                                           (RK_CRYPTO_HASH_START << 16));
 225 }
 226
 227 static int rk_hash_prepare(struct crypto_engine *engine, void *breq)
 228 {
 229         struct ahash_request *areq = container_of(breq, struct ahash_request, base);
 230         struct rk_ahash_rctx *rctx = ahash_request_ctx(areq);
 231         struct rk_crypto_info *rkc = rctx->dev;
 232         int ret;
 233
 234         ret = dma_map_sg(rkc->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE);
 235         if (ret <= 0)
 236                 return -EINVAL;
 237
 238         rctx->nrsg = ret;
 239
 240         return 0;
 241 }
 242
 243 static int rk_hash_unprepare(struct crypto_engine *engine, void *breq)
 244 {
 245         struct ahash_request *areq = container_of(breq, struct ahash_request, base);
 246         struct rk_ahash_rctx *rctx = ahash_request_ctx(areq);
 247         struct rk_crypto_info *rkc = rctx->dev;
 248
 249         dma_unmap_sg(rkc->dev, areq->src, rctx->nrsg, DMA_TO_DEVICE);
 250         return 0;
 251 }
 252
 253 static int rk_hash_run(struct crypto_engine *engine, void *breq)
 254 {
 255         struct ahash_request *areq = container_of(breq, struct ahash_request, base);
 256         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
 257         struct rk_ahash_rctx *rctx = ahash_request_ctx(areq);
 258         struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
 259         struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.hash);
 260         struct scatterlist *sg = areq->src;
 261         struct rk_crypto_info *rkc = rctx->dev;
 262         int err = 0;
 263         int i;
 264         u32 v;
 265
 266         err = pm_runtime_resume_and_get(rkc->dev);
 267         if (err)
 268                 return err;
 269
 270         rctx->mode = 0;
 271
 272         algt->stat_req++;
 273         rkc->nreq++;
 274
 275         switch (crypto_ahash_digestsize(tfm)) {
 276         case SHA1_DIGEST_SIZE:
 277                 rctx->mode = RK_CRYPTO_HASH_SHA1;
 278                 break;
 279         case SHA256_DIGEST_SIZE:
 280                 rctx->mode = RK_CRYPTO_HASH_SHA256;
 281                 break;
 282         case MD5_DIGEST_SIZE:
 283                 rctx->mode = RK_CRYPTO_HASH_MD5;
 284                 break;
 285         default:
 286                 err =  -EINVAL;
 287                 goto theend;
 288         }
 289
 290         rk_ahash_reg_init(areq, rkc);
 291
 292         while (sg) {
 293                 reinit_completion(&rkc->complete);
 294                 rkc->status = 0;
 295                 crypto_ahash_dma_start(rkc, sg);
 296                 wait_for_completion_interruptible_timeout(&rkc->complete,
 297                                                           msecs_to_jiffies(2000));
 298                 if (!rkc->status) {
 299                         dev_err(rkc->dev, "DMA timeout\n");
 300                         err = -EFAULT;
 301                         goto theend;
 302                 }
 303                 sg = sg_next(sg);
 304         }
 305
 306         /*
 307          * it will take some time to process date after last dma
 308          * transmission.
 309          *
 310          * waiting time is relative with the last date len,
 311          * so cannot set a fixed time here.
 312          * 10us makes system not call here frequently wasting
 313          * efficiency, and make it response quickly when dma
 314          * complete.
 315          */
 316         readl_poll_timeout(rkc->reg + RK_CRYPTO_HASH_STS, v, v == 0, 10, 1000);
 317
 318         for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++) {
 319                 v = readl(rkc->reg + RK_CRYPTO_HASH_DOUT_0 + i * 4);
 320                 put_unaligned_le32(v, areq->result + i * 4);
 321         }
 322
 323 theend:
 324         pm_runtime_put_autosuspend(rkc->dev);
 325
 326         local_bh_disable();
 327         crypto_finalize_hash_request(engine, breq, err);
 328         local_bh_enable();
 329
 330         return 0;
 331 }
 332
 333 static int rk_cra_hash_init(struct crypto_tfm *tfm)
 334 {
 335         struct rk_ahash_ctx *tctx = crypto_tfm_ctx(tfm);
 336         const char *alg_name = crypto_tfm_alg_name(tfm);
 337         struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg);
 338         struct rk_crypto_tmp *algt = container_of(alg, struct rk_crypto_tmp, alg.hash);
 339
 340         /* for fallback */
 341         tctx->fallback_tfm = crypto_alloc_ahash(alg_name, 0,
 342                                                 CRYPTO_ALG_NEED_FALLBACK);
 343         if (IS_ERR(tctx->fallback_tfm)) {
 344                 dev_err(algt->dev->dev, "Could not load fallback driver.\n");
 345                 return PTR_ERR(tctx->fallback_tfm);
 346         }
 347
 348         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 349                                  sizeof(struct rk_ahash_rctx) +
 350                                  crypto_ahash_reqsize(tctx->fallback_tfm));
 351
 352         tctx->enginectx.op.do_one_request = rk_hash_run;
 353         tctx->enginectx.op.prepare_request = rk_hash_prepare;
 354         tctx->enginectx.op.unprepare_request = rk_hash_unprepare;
 355
 356         return 0;
 357 }
 358
 359 static void rk_cra_hash_exit(struct crypto_tfm *tfm)
 360 {
 361         struct rk_ahash_ctx *tctx = crypto_tfm_ctx(tfm);
 362
 363         crypto_free_ahash(tctx->fallback_tfm);
 364 }
 365
 366 struct rk_crypto_tmp rk_ahash_sha1 = {
 367         .type = CRYPTO_ALG_TYPE_AHASH,
 368         .alg.hash = {
 369                 .init = rk_ahash_init,
 370                 .update = rk_ahash_update,
 371                 .final = rk_ahash_final,
 372                 .finup = rk_ahash_finup,
 373                 .export = rk_ahash_export,
 374                 .import = rk_ahash_import,
 375                 .digest = rk_ahash_digest,
 376                 .halg = {
 377                          .digestsize = SHA1_DIGEST_SIZE,
 378                          .statesize = sizeof(struct sha1_state),
 379                          .base = {
 380                                   .cra_name = "sha1",
 381                                   .cra_driver_name = "rk-sha1",
 382                                   .cra_priority = 300,
 383                                   .cra_flags = CRYPTO_ALG_ASYNC |
 384                                                CRYPTO_ALG_NEED_FALLBACK,
 385                                   .cra_blocksize = SHA1_BLOCK_SIZE,
 386                                   .cra_ctxsize = sizeof(struct rk_ahash_ctx),
 387                                   .cra_alignmask = 3,
 388                                   .cra_init = rk_cra_hash_init,
 389                                   .cra_exit = rk_cra_hash_exit,
 390                                   .cra_module = THIS_MODULE,
 391                         }
 392                 }
 393         }
 394 };
 395
 396 struct rk_crypto_tmp rk_ahash_sha256 = {
 397         .type = CRYPTO_ALG_TYPE_AHASH,
 398         .alg.hash = {
 399                 .init = rk_ahash_init,
 400                 .update = rk_ahash_update,
 401                 .final = rk_ahash_final,
 402                 .finup = rk_ahash_finup,
 403                 .export = rk_ahash_export,
 404                 .import = rk_ahash_import,
 405                 .digest = rk_ahash_digest,
 406                 .halg = {
 407                          .digestsize = SHA256_DIGEST_SIZE,
 408                          .statesize = sizeof(struct sha256_state),
 409                          .base = {
 410                                   .cra_name = "sha256",
 411                                   .cra_driver_name = "rk-sha256",
 412                                   .cra_priority = 300,
 413                                   .cra_flags = CRYPTO_ALG_ASYNC |
 414                                                CRYPTO_ALG_NEED_FALLBACK,
 415                                   .cra_blocksize = SHA256_BLOCK_SIZE,
 416                                   .cra_ctxsize = sizeof(struct rk_ahash_ctx),
 417                                   .cra_alignmask = 3,
 418                                   .cra_init = rk_cra_hash_init,
 419                                   .cra_exit = rk_cra_hash_exit,
 420                                   .cra_module = THIS_MODULE,
 421                         }
 422                 }
 423         }
 424 };
 425
 426 struct rk_crypto_tmp rk_ahash_md5 = {
 427         .type = CRYPTO_ALG_TYPE_AHASH,
 428         .alg.hash = {
 429                 .init = rk_ahash_init,
 430                 .update = rk_ahash_update,
 431                 .final = rk_ahash_final,
 432                 .finup = rk_ahash_finup,
 433                 .export = rk_ahash_export,
 434                 .import = rk_ahash_import,
 435                 .digest = rk_ahash_digest,
 436                 .halg = {
 437                          .digestsize = MD5_DIGEST_SIZE,
 438                          .statesize = sizeof(struct md5_state),
 439                          .base = {
 440                                   .cra_name = "md5",
 441                                   .cra_driver_name = "rk-md5",
 442                                   .cra_priority = 300,
 443                                   .cra_flags = CRYPTO_ALG_ASYNC |
 444                                                CRYPTO_ALG_NEED_FALLBACK,
 445                                   .cra_blocksize = SHA1_BLOCK_SIZE,
 446                                   .cra_ctxsize = sizeof(struct rk_ahash_ctx),
 447                                   .cra_alignmask = 3,
 448                                   .cra_init = rk_cra_hash_init,
 449                                   .cra_exit = rk_cra_hash_exit,
 450                                   .cra_module = THIS_MODULE,
 451                         }
 452                 }
 453         }
 454 };