drivers/crypto/aspeed/aspeed-hace-crypto.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Copyright (c) 2021 Aspeed Technology Inc.
   4  */
   5
   6 #include "aspeed-hace.h"
   7 #include <crypto/des.h>
   8 #include <crypto/engine.h>
   9 #include <crypto/internal/des.h>
  10 #include <crypto/internal/skcipher.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/err.h>
  13 #include <linux/io.h>
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/scatterlist.h>
  17 #include <linux/string.h>
  18
  19 #ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO_DEBUG
  20 #define CIPHER_DBG(h, fmt, ...) \
  21         dev_info((h)->dev, "%s() " fmt, __func__, ##__VA_ARGS__)
  22 #else
  23 #define CIPHER_DBG(h, fmt, ...) \
  24         dev_dbg((h)->dev, "%s() " fmt, __func__, ##__VA_ARGS__)
  25 #endif
  26
  27 static int aspeed_crypto_do_fallback(struct skcipher_request *areq)
  28 {
  29         struct aspeed_cipher_reqctx *rctx = skcipher_request_ctx(areq);
  30         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
  31         struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(tfm);
  32         int err;
  33
  34         skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
  35         skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,
  36                                       areq->base.complete, areq->base.data);
  37         skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,
  38                                    areq->cryptlen, areq->iv);
  39
  40         if (rctx->enc_cmd & HACE_CMD_ENCRYPT)
  41                 err = crypto_skcipher_encrypt(&rctx->fallback_req);
  42         else
  43                 err = crypto_skcipher_decrypt(&rctx->fallback_req);
  44
  45         return err;
  46 }
  47
  48 static bool aspeed_crypto_need_fallback(struct skcipher_request *areq)
  49 {
  50         struct aspeed_cipher_reqctx *rctx = skcipher_request_ctx(areq);
  51
  52         if (areq->cryptlen == 0)
  53                 return true;
  54
  55         if ((rctx->enc_cmd & HACE_CMD_DES_SELECT) &&
  56             !IS_ALIGNED(areq->cryptlen, DES_BLOCK_SIZE))
  57                 return true;
  58
  59         if ((!(rctx->enc_cmd & HACE_CMD_DES_SELECT)) &&
  60             !IS_ALIGNED(areq->cryptlen, AES_BLOCK_SIZE))
  61                 return true;
  62
  63         return false;
  64 }
  65
  66 static int aspeed_hace_crypto_handle_queue(struct aspeed_hace_dev *hace_dev,
  67                                            struct skcipher_request *req)
  68 {
  69         if (hace_dev->version == AST2500_VERSION &&
  70             aspeed_crypto_need_fallback(req)) {
  71                 CIPHER_DBG(hace_dev, "SW fallback\n");
  72                 return aspeed_crypto_do_fallback(req);
  73         }
  74
  75         return crypto_transfer_skcipher_request_to_engine(
  76                         hace_dev->crypt_engine_crypto, req);
  77 }
  78
  79 static int aspeed_crypto_do_request(struct crypto_engine *engine, void *areq)
  80 {
  81         struct skcipher_request *req = skcipher_request_cast(areq);
  82         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
  83         struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);
  84         struct aspeed_hace_dev *hace_dev = ctx->hace_dev;
  85         struct aspeed_engine_crypto *crypto_engine;
  86         int rc;
  87
  88         crypto_engine = &hace_dev->crypto_engine;
  89         crypto_engine->req = req;
  90         crypto_engine->flags |= CRYPTO_FLAGS_BUSY;
  91
  92         rc = ctx->start(hace_dev);
  93
  94         if (rc != -EINPROGRESS)
  95                 return -EIO;
  96
  97         return 0;
  98 }
  99
 100 static int aspeed_sk_complete(struct aspeed_hace_dev *hace_dev, int err)
 101 {
 102         struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine;
 103         struct aspeed_cipher_reqctx *rctx;
 104         struct skcipher_request *req;
 105
 106         CIPHER_DBG(hace_dev, "\n");
 107
 108         req = crypto_engine->req;
 109         rctx = skcipher_request_ctx(req);
 110
 111         if (rctx->enc_cmd & HACE_CMD_IV_REQUIRE) {
 112                 if (rctx->enc_cmd & HACE_CMD_DES_SELECT)
 113                         memcpy(req->iv, crypto_engine->cipher_ctx +
 114                                DES_KEY_SIZE, DES_KEY_SIZE);
 115                 else
 116                         memcpy(req->iv, crypto_engine->cipher_ctx,
 117                                AES_BLOCK_SIZE);
 118         }
 119
 120         crypto_engine->flags &= ~CRYPTO_FLAGS_BUSY;
 121
 122         crypto_finalize_skcipher_request(hace_dev->crypt_engine_crypto, req,
 123                                          err);
 124
 125         return err;
 126 }
 127
 128 static int aspeed_sk_transfer_sg(struct aspeed_hace_dev *hace_dev)
 129 {
 130         struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine;
 131         struct device *dev = hace_dev->dev;
 132         struct aspeed_cipher_reqctx *rctx;
 133         struct skcipher_request *req;
 134
 135         CIPHER_DBG(hace_dev, "\n");
 136
 137         req = crypto_engine->req;
 138         rctx = skcipher_request_ctx(req);
 139
 140         if (req->src == req->dst) {
 141                 dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_BIDIRECTIONAL);
 142         } else {
 143                 dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE);
 144                 dma_unmap_sg(dev, req->dst, rctx->dst_nents, DMA_FROM_DEVICE);
 145         }
 146
 147         return aspeed_sk_complete(hace_dev, 0);
 148 }
 149
 150 static int aspeed_sk_transfer(struct aspeed_hace_dev *hace_dev)
 151 {
 152         struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine;
 153         struct aspeed_cipher_reqctx *rctx;
 154         struct skcipher_request *req;
 155         struct scatterlist *out_sg;
 156         int nbytes = 0;
 157         int rc = 0;
 158
 159         req = crypto_engine->req;
 160         rctx = skcipher_request_ctx(req);
 161         out_sg = req->dst;
 162
 163         /* Copy output buffer to dst scatter-gather lists */
 164         nbytes = sg_copy_from_buffer(out_sg, rctx->dst_nents,
 165                                      crypto_engine->cipher_addr, req->cryptlen);
 166         if (!nbytes) {
 167                 dev_warn(hace_dev->dev, "invalid sg copy, %s:0x%x, %s:0x%x\n",
 168                          "nbytes", nbytes, "cryptlen", req->cryptlen);
 169                 rc = -EINVAL;
 170         }
 171
 172         CIPHER_DBG(hace_dev, "%s:%d, %s:%d, %s:%d, %s:%p\n",
 173                    "nbytes", nbytes, "req->cryptlen", req->cryptlen,
 174                    "nb_out_sg", rctx->dst_nents,
 175                    "cipher addr", crypto_engine->cipher_addr);
 176
 177         return aspeed_sk_complete(hace_dev, rc);
 178 }
 179
 180 static int aspeed_sk_start(struct aspeed_hace_dev *hace_dev)
 181 {
 182         struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine;
 183         struct aspeed_cipher_reqctx *rctx;
 184         struct skcipher_request *req;
 185         struct scatterlist *in_sg;
 186         int nbytes;
 187
 188         req = crypto_engine->req;
 189         rctx = skcipher_request_ctx(req);
 190         in_sg = req->src;
 191
 192         nbytes = sg_copy_to_buffer(in_sg, rctx->src_nents,
 193                                    crypto_engine->cipher_addr, req->cryptlen);
 194
 195         CIPHER_DBG(hace_dev, "%s:%d, %s:%d, %s:%d, %s:%p\n",
 196                    "nbytes", nbytes, "req->cryptlen", req->cryptlen,
 197                    "nb_in_sg", rctx->src_nents,
 198                    "cipher addr", crypto_engine->cipher_addr);
 199
 200         if (!nbytes) {
 201                 dev_warn(hace_dev->dev, "invalid sg copy, %s:0x%x, %s:0x%x\n",
 202                          "nbytes", nbytes, "cryptlen", req->cryptlen);
 203                 return -EINVAL;
 204         }
 205
 206         crypto_engine->resume = aspeed_sk_transfer;
 207
 208         /* Trigger engines */
 209         ast_hace_write(hace_dev, crypto_engine->cipher_dma_addr,
 210                        ASPEED_HACE_SRC);
 211         ast_hace_write(hace_dev, crypto_engine->cipher_dma_addr,
 212                        ASPEED_HACE_DEST);
 213         ast_hace_write(hace_dev, req->cryptlen, ASPEED_HACE_DATA_LEN);
 214         ast_hace_write(hace_dev, rctx->enc_cmd, ASPEED_HACE_CMD);
 215
 216         return -EINPROGRESS;
 217 }
 218
 219 static int aspeed_sk_start_sg(struct aspeed_hace_dev *hace_dev)
 220 {
 221         struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine;
 222         struct aspeed_sg_list *src_list, *dst_list;
 223         dma_addr_t src_dma_addr, dst_dma_addr;
 224         struct aspeed_cipher_reqctx *rctx;
 225         struct skcipher_request *req;
 226         struct scatterlist *s;
 227         int src_sg_len;
 228         int dst_sg_len;
 229         int total, i;
 230         int rc;
 231
 232         CIPHER_DBG(hace_dev, "\n");
 233
 234         req = crypto_engine->req;
 235         rctx = skcipher_request_ctx(req);
 236
 237         rctx->enc_cmd |= HACE_CMD_DES_SG_CTRL | HACE_CMD_SRC_SG_CTRL |
 238                          HACE_CMD_AES_KEY_HW_EXP | HACE_CMD_MBUS_REQ_SYNC_EN;
 239
 240         /* BIDIRECTIONAL */
 241         if (req->dst == req->src) {
 242                 src_sg_len = dma_map_sg(hace_dev->dev, req->src,
 243                                         rctx->src_nents, DMA_BIDIRECTIONAL);
 244                 dst_sg_len = src_sg_len;
 245                 if (!src_sg_len) {
 246                         dev_warn(hace_dev->dev, "dma_map_sg() src error\n");
 247                         return -EINVAL;
 248                 }
 249
 250         } else {
 251                 src_sg_len = dma_map_sg(hace_dev->dev, req->src,
 252                                         rctx->src_nents, DMA_TO_DEVICE);
 253                 if (!src_sg_len) {
 254                         dev_warn(hace_dev->dev, "dma_map_sg() src error\n");
 255                         return -EINVAL;
 256                 }
 257
 258                 dst_sg_len = dma_map_sg(hace_dev->dev, req->dst,
 259                                         rctx->dst_nents, DMA_FROM_DEVICE);
 260                 if (!dst_sg_len) {
 261                         dev_warn(hace_dev->dev, "dma_map_sg() dst error\n");
 262                         rc = -EINVAL;
 263                         goto free_req_src;
 264                 }
 265         }
 266
 267         src_list = (struct aspeed_sg_list *)crypto_engine->cipher_addr;
 268         src_dma_addr = crypto_engine->cipher_dma_addr;
 269         total = req->cryptlen;
 270
 271         for_each_sg(req->src, s, src_sg_len, i) {
 272                 u32 phy_addr = sg_dma_address(s);
 273                 u32 len = sg_dma_len(s);
 274
 275                 if (total > len)
 276                         total -= len;
 277                 else {
 278                         /* last sg list */
 279                         len = total;
 280                         len |= BIT(31);
 281                         total = 0;
 282                 }
 283
 284                 src_list[i].phy_addr = cpu_to_le32(phy_addr);
 285                 src_list[i].len = cpu_to_le32(len);
 286         }
 287
 288         if (total != 0) {
 289                 rc = -EINVAL;
 290                 goto free_req;
 291         }
 292
 293         if (req->dst == req->src) {
 294                 dst_list = src_list;
 295                 dst_dma_addr = src_dma_addr;
 296
 297         } else {
 298                 dst_list = (struct aspeed_sg_list *)crypto_engine->dst_sg_addr;
 299                 dst_dma_addr = crypto_engine->dst_sg_dma_addr;
 300                 total = req->cryptlen;
 301
 302                 for_each_sg(req->dst, s, dst_sg_len, i) {
 303                         u32 phy_addr = sg_dma_address(s);
 304                         u32 len = sg_dma_len(s);
 305
 306                         if (total > len)
 307                                 total -= len;
 308                         else {
 309                                 /* last sg list */
 310                                 len = total;
 311                                 len |= BIT(31);
 312                                 total = 0;
 313                         }
 314
 315                         dst_list[i].phy_addr = cpu_to_le32(phy_addr);
 316                         dst_list[i].len = cpu_to_le32(len);
 317
 318                 }
 319
 320                 dst_list[dst_sg_len].phy_addr = 0;
 321                 dst_list[dst_sg_len].len = 0;
 322         }
 323
 324         if (total != 0) {
 325                 rc = -EINVAL;
 326                 goto free_req;
 327         }
 328
 329         crypto_engine->resume = aspeed_sk_transfer_sg;
 330
 331         /* Memory barrier to ensure all data setup before engine starts */
 332         mb();
 333
 334         /* Trigger engines */
 335         ast_hace_write(hace_dev, src_dma_addr, ASPEED_HACE_SRC);
 336         ast_hace_write(hace_dev, dst_dma_addr, ASPEED_HACE_DEST);
 337         ast_hace_write(hace_dev, req->cryptlen, ASPEED_HACE_DATA_LEN);
 338         ast_hace_write(hace_dev, rctx->enc_cmd, ASPEED_HACE_CMD);
 339
 340         return -EINPROGRESS;
 341
 342 free_req:
 343         if (req->dst == req->src) {
 344                 dma_unmap_sg(hace_dev->dev, req->src, rctx->src_nents,
 345                              DMA_BIDIRECTIONAL);
 346
 347         } else {
 348                 dma_unmap_sg(hace_dev->dev, req->dst, rctx->dst_nents,
 349                              DMA_TO_DEVICE);
 350                 dma_unmap_sg(hace_dev->dev, req->src, rctx->src_nents,
 351                              DMA_TO_DEVICE);
 352         }
 353
 354         return rc;
 355
 356 free_req_src:
 357         dma_unmap_sg(hace_dev->dev, req->src, rctx->src_nents, DMA_TO_DEVICE);
 358
 359         return rc;
 360 }
 361
 362 static int aspeed_hace_skcipher_trigger(struct aspeed_hace_dev *hace_dev)
 363 {
 364         struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine;
 365         struct aspeed_cipher_reqctx *rctx;
 366         struct crypto_skcipher *cipher;
 367         struct aspeed_cipher_ctx *ctx;
 368         struct skcipher_request *req;
 369
 370         CIPHER_DBG(hace_dev, "\n");
 371
 372         req = crypto_engine->req;
 373         rctx = skcipher_request_ctx(req);
 374         cipher = crypto_skcipher_reqtfm(req);
 375         ctx = crypto_skcipher_ctx(cipher);
 376
 377         /* enable interrupt */
 378         rctx->enc_cmd |= HACE_CMD_ISR_EN;
 379
 380         rctx->dst_nents = sg_nents(req->dst);
 381         rctx->src_nents = sg_nents(req->src);
 382
 383         ast_hace_write(hace_dev, crypto_engine->cipher_ctx_dma,
 384                        ASPEED_HACE_CONTEXT);
 385
 386         if (rctx->enc_cmd & HACE_CMD_IV_REQUIRE) {
 387                 if (rctx->enc_cmd & HACE_CMD_DES_SELECT)
 388                         memcpy(crypto_engine->cipher_ctx + DES_BLOCK_SIZE,
 389                                req->iv, DES_BLOCK_SIZE);
 390                 else
 391                         memcpy(crypto_engine->cipher_ctx, req->iv,
 392                                AES_BLOCK_SIZE);
 393         }
 394
 395         if (hace_dev->version == AST2600_VERSION) {
 396                 memcpy(crypto_engine->cipher_ctx + 16, ctx->key, ctx->key_len);
 397
 398                 return aspeed_sk_start_sg(hace_dev);
 399         }
 400
 401         memcpy(crypto_engine->cipher_ctx + 16, ctx->key, AES_MAX_KEYLENGTH);
 402
 403         return aspeed_sk_start(hace_dev);
 404 }
 405
 406 static int aspeed_des_crypt(struct skcipher_request *req, u32 cmd)
 407 {
 408         struct aspeed_cipher_reqctx *rctx = skcipher_request_ctx(req);
 409         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
 410         struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);
 411         struct aspeed_hace_dev *hace_dev = ctx->hace_dev;
 412         u32 crypto_alg = cmd & HACE_CMD_OP_MODE_MASK;
 413
 414         CIPHER_DBG(hace_dev, "\n");
 415
 416         if (crypto_alg == HACE_CMD_CBC || crypto_alg == HACE_CMD_ECB) {
 417                 if (!IS_ALIGNED(req->cryptlen, DES_BLOCK_SIZE))
 418                         return -EINVAL;
 419         }
 420
 421         rctx->enc_cmd = cmd | HACE_CMD_DES_SELECT | HACE_CMD_RI_WO_DATA_ENABLE |
 422                         HACE_CMD_DES | HACE_CMD_CONTEXT_LOAD_ENABLE |
 423                         HACE_CMD_CONTEXT_SAVE_ENABLE;
 424
 425         return aspeed_hace_crypto_handle_queue(hace_dev, req);
 426 }
 427
 428 static int aspeed_des_setkey(struct crypto_skcipher *cipher, const u8 *key,
 429                              unsigned int keylen)
 430 {
 431         struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);
 432         struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
 433         struct aspeed_hace_dev *hace_dev = ctx->hace_dev;
 434         int rc;
 435
 436         CIPHER_DBG(hace_dev, "keylen: %d bits\n", keylen);
 437
 438         if (keylen != DES_KEY_SIZE && keylen != DES3_EDE_KEY_SIZE) {
 439                 dev_warn(hace_dev->dev, "invalid keylen: %d bits\n", keylen);
 440                 return -EINVAL;
 441         }
 442
 443         if (keylen == DES_KEY_SIZE) {
 444                 rc = crypto_des_verify_key(tfm, key);
 445                 if (rc)
 446                         return rc;
 447
 448         } else if (keylen == DES3_EDE_KEY_SIZE) {
 449                 rc = crypto_des3_ede_verify_key(tfm, key);
 450                 if (rc)
 451                         return rc;
 452         }
 453
 454         memcpy(ctx->key, key, keylen);
 455         ctx->key_len = keylen;
 456
 457         crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK);
 458         crypto_skcipher_set_flags(ctx->fallback_tfm, cipher->base.crt_flags &
 459                                   CRYPTO_TFM_REQ_MASK);
 460
 461         return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen);
 462 }
 463
 464 static int aspeed_tdes_ctr_decrypt(struct skcipher_request *req)
 465 {
 466         return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CTR |
 467                                 HACE_CMD_TRIPLE_DES);
 468 }
 469
 470 static int aspeed_tdes_ctr_encrypt(struct skcipher_request *req)
 471 {
 472         return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CTR |
 473                                 HACE_CMD_TRIPLE_DES);
 474 }
 475
 476 static int aspeed_tdes_cbc_decrypt(struct skcipher_request *req)
 477 {
 478         return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CBC |
 479                                 HACE_CMD_TRIPLE_DES);
 480 }
 481
 482 static int aspeed_tdes_cbc_encrypt(struct skcipher_request *req)
 483 {
 484         return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CBC |
 485                                 HACE_CMD_TRIPLE_DES);
 486 }
 487
 488 static int aspeed_tdes_ecb_decrypt(struct skcipher_request *req)
 489 {
 490         return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_ECB |
 491                                 HACE_CMD_TRIPLE_DES);
 492 }
 493
 494 static int aspeed_tdes_ecb_encrypt(struct skcipher_request *req)
 495 {
 496         return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_ECB |
 497                                 HACE_CMD_TRIPLE_DES);
 498 }
 499
 500 static int aspeed_des_ctr_decrypt(struct skcipher_request *req)
 501 {
 502         return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CTR |
 503                                 HACE_CMD_SINGLE_DES);
 504 }
 505
 506 static int aspeed_des_ctr_encrypt(struct skcipher_request *req)
 507 {
 508         return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CTR |
 509                                 HACE_CMD_SINGLE_DES);
 510 }
 511
 512 static int aspeed_des_cbc_decrypt(struct skcipher_request *req)
 513 {
 514         return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CBC |
 515                                 HACE_CMD_SINGLE_DES);
 516 }
 517
 518 static int aspeed_des_cbc_encrypt(struct skcipher_request *req)
 519 {
 520         return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CBC |
 521                                 HACE_CMD_SINGLE_DES);
 522 }
 523
 524 static int aspeed_des_ecb_decrypt(struct skcipher_request *req)
 525 {
 526         return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_ECB |
 527                                 HACE_CMD_SINGLE_DES);
 528 }
 529
 530 static int aspeed_des_ecb_encrypt(struct skcipher_request *req)
 531 {
 532         return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_ECB |
 533                                 HACE_CMD_SINGLE_DES);
 534 }
 535
 536 static int aspeed_aes_crypt(struct skcipher_request *req, u32 cmd)
 537 {
 538         struct aspeed_cipher_reqctx *rctx = skcipher_request_ctx(req);
 539         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
 540         struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);
 541         struct aspeed_hace_dev *hace_dev = ctx->hace_dev;
 542         u32 crypto_alg = cmd & HACE_CMD_OP_MODE_MASK;
 543
 544         if (crypto_alg == HACE_CMD_CBC || crypto_alg == HACE_CMD_ECB) {
 545                 if (!IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE))
 546                         return -EINVAL;
 547         }
 548
 549         CIPHER_DBG(hace_dev, "%s\n",
 550                    (cmd & HACE_CMD_ENCRYPT) ? "encrypt" : "decrypt");
 551
 552         cmd |= HACE_CMD_AES_SELECT | HACE_CMD_RI_WO_DATA_ENABLE |
 553                HACE_CMD_CONTEXT_LOAD_ENABLE | HACE_CMD_CONTEXT_SAVE_ENABLE;
 554
 555         switch (ctx->key_len) {
 556         case AES_KEYSIZE_128:
 557                 cmd |= HACE_CMD_AES128;
 558                 break;
 559         case AES_KEYSIZE_192:
 560                 cmd |= HACE_CMD_AES192;
 561                 break;
 562         case AES_KEYSIZE_256:
 563                 cmd |= HACE_CMD_AES256;
 564                 break;
 565         default:
 566                 return -EINVAL;
 567         }
 568
 569         rctx->enc_cmd = cmd;
 570
 571         return aspeed_hace_crypto_handle_queue(hace_dev, req);
 572 }
 573
 574 static int aspeed_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
 575                              unsigned int keylen)
 576 {
 577         struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);
 578         struct aspeed_hace_dev *hace_dev = ctx->hace_dev;
 579         struct crypto_aes_ctx gen_aes_key;
 580
 581         CIPHER_DBG(hace_dev, "keylen: %d bits\n", (keylen * 8));
 582
 583         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 584             keylen != AES_KEYSIZE_256)
 585                 return -EINVAL;
 586
 587         if (ctx->hace_dev->version == AST2500_VERSION) {
 588                 aes_expandkey(&gen_aes_key, key, keylen);
 589                 memcpy(ctx->key, gen_aes_key.key_enc, AES_MAX_KEYLENGTH);
 590
 591         } else {
 592                 memcpy(ctx->key, key, keylen);
 593         }
 594
 595         ctx->key_len = keylen;
 596
 597         crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK);
 598         crypto_skcipher_set_flags(ctx->fallback_tfm, cipher->base.crt_flags &
 599                                   CRYPTO_TFM_REQ_MASK);
 600
 601         return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen);
 602 }
 603
 604 static int aspeed_aes_ctr_decrypt(struct skcipher_request *req)
 605 {
 606         return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CTR);
 607 }
 608
 609 static int aspeed_aes_ctr_encrypt(struct skcipher_request *req)
 610 {
 611         return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CTR);
 612 }
 613
 614 static int aspeed_aes_cbc_decrypt(struct skcipher_request *req)
 615 {
 616         return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CBC);
 617 }
 618
 619 static int aspeed_aes_cbc_encrypt(struct skcipher_request *req)
 620 {
 621         return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CBC);
 622 }
 623
 624 static int aspeed_aes_ecb_decrypt(struct skcipher_request *req)
 625 {
 626         return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_ECB);
 627 }
 628
 629 static int aspeed_aes_ecb_encrypt(struct skcipher_request *req)
 630 {
 631         return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_ECB);
 632 }
 633
 634 static int aspeed_crypto_cra_init(struct crypto_skcipher *tfm)
 635 {
 636         struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(tfm);
 637         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
 638         const char *name = crypto_tfm_alg_name(&tfm->base);
 639         struct aspeed_hace_alg *crypto_alg;
 640
 641
 642         crypto_alg = container_of(alg, struct aspeed_hace_alg, alg.skcipher.base);
 643         ctx->hace_dev = crypto_alg->hace_dev;
 644         ctx->start = aspeed_hace_skcipher_trigger;
 645
 646         CIPHER_DBG(ctx->hace_dev, "%s\n", name);
 647
 648         ctx->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_ASYNC |
 649                                                   CRYPTO_ALG_NEED_FALLBACK);
 650         if (IS_ERR(ctx->fallback_tfm)) {
 651                 dev_err(ctx->hace_dev->dev, "ERROR: Cannot allocate fallback for %s %ld\n",
 652                         name, PTR_ERR(ctx->fallback_tfm));
 653                 return PTR_ERR(ctx->fallback_tfm);
 654         }
 655
 656         crypto_skcipher_set_reqsize(tfm, sizeof(struct aspeed_cipher_reqctx) +
 657                          crypto_skcipher_reqsize(ctx->fallback_tfm));
 658
 659         return 0;
 660 }
 661
 662 static void aspeed_crypto_cra_exit(struct crypto_skcipher *tfm)
 663 {
 664         struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(tfm);
 665         struct aspeed_hace_dev *hace_dev = ctx->hace_dev;
 666
 667         CIPHER_DBG(hace_dev, "%s\n", crypto_tfm_alg_name(&tfm->base));
 668         crypto_free_skcipher(ctx->fallback_tfm);
 669 }
 670
 671 static struct aspeed_hace_alg aspeed_crypto_algs[] = {
 672         {
 673                 .alg.skcipher.base = {
 674                         .min_keysize    = AES_MIN_KEY_SIZE,
 675                         .max_keysize    = AES_MAX_KEY_SIZE,
 676                         .setkey         = aspeed_aes_setkey,
 677                         .encrypt        = aspeed_aes_ecb_encrypt,
 678                         .decrypt        = aspeed_aes_ecb_decrypt,
 679                         .init           = aspeed_crypto_cra_init,
 680                         .exit           = aspeed_crypto_cra_exit,
 681                         .base = {
 682                                 .cra_name               = "ecb(aes)",
 683                                 .cra_driver_name        = "aspeed-ecb-aes",
 684                                 .cra_priority           = 300,
 685                                 .cra_flags              = CRYPTO_ALG_KERN_DRIVER_ONLY |
 686                                                           CRYPTO_ALG_ASYNC |
 687                                                           CRYPTO_ALG_NEED_FALLBACK,
 688                                 .cra_blocksize          = AES_BLOCK_SIZE,
 689                                 .cra_ctxsize            = sizeof(struct aspeed_cipher_ctx),
 690                                 .cra_alignmask          = 0x0f,
 691                                 .cra_module             = THIS_MODULE,
 692                         }
 693                 },
 694                 .alg.skcipher.op = {
 695                         .do_one_request = aspeed_crypto_do_request,
 696                 },
 697         },
 698         {
 699                 .alg.skcipher.base = {
 700                         .ivsize         = AES_BLOCK_SIZE,
 701                         .min_keysize    = AES_MIN_KEY_SIZE,
 702                         .max_keysize    = AES_MAX_KEY_SIZE,
 703                         .setkey         = aspeed_aes_setkey,
 704                         .encrypt        = aspeed_aes_cbc_encrypt,
 705                         .decrypt        = aspeed_aes_cbc_decrypt,
 706                         .init           = aspeed_crypto_cra_init,
 707                         .exit           = aspeed_crypto_cra_exit,
 708                         .base = {
 709                                 .cra_name               = "cbc(aes)",
 710                                 .cra_driver_name        = "aspeed-cbc-aes",
 711                                 .cra_priority           = 300,
 712                                 .cra_flags              = CRYPTO_ALG_KERN_DRIVER_ONLY |
 713                                                           CRYPTO_ALG_ASYNC |
 714                                                           CRYPTO_ALG_NEED_FALLBACK,
 715                                 .cra_blocksize          = AES_BLOCK_SIZE,
 716                                 .cra_ctxsize            = sizeof(struct aspeed_cipher_ctx),
 717                                 .cra_alignmask          = 0x0f,
 718                                 .cra_module             = THIS_MODULE,
 719                         }
 720                 },
 721                 .alg.skcipher.op = {
 722                         .do_one_request = aspeed_crypto_do_request,
 723                 },
 724         },
 725         {
 726                 .alg.skcipher.base = {
 727                         .min_keysize    = DES_KEY_SIZE,
 728                         .max_keysize    = DES_KEY_SIZE,
 729                         .setkey         = aspeed_des_setkey,
 730                         .encrypt        = aspeed_des_ecb_encrypt,
 731                         .decrypt        = aspeed_des_ecb_decrypt,
 732                         .init           = aspeed_crypto_cra_init,
 733                         .exit           = aspeed_crypto_cra_exit,
 734                         .base = {
 735                                 .cra_name               = "ecb(des)",
 736                                 .cra_driver_name        = "aspeed-ecb-des",
 737                                 .cra_priority           = 300,
 738                                 .cra_flags              = CRYPTO_ALG_KERN_DRIVER_ONLY |
 739                                                           CRYPTO_ALG_ASYNC |
 740                                                           CRYPTO_ALG_NEED_FALLBACK,
 741                                 .cra_blocksize          = DES_BLOCK_SIZE,
 742                                 .cra_ctxsize            = sizeof(struct aspeed_cipher_ctx),
 743                                 .cra_alignmask          = 0x0f,
 744                                 .cra_module             = THIS_MODULE,
 745                         }
 746                 },
 747                 .alg.skcipher.op = {
 748                         .do_one_request = aspeed_crypto_do_request,
 749                 },
 750         },
 751         {
 752                 .alg.skcipher.base = {
 753                         .ivsize         = DES_BLOCK_SIZE,
 754                         .min_keysize    = DES_KEY_SIZE,
 755                         .max_keysize    = DES_KEY_SIZE,
 756                         .setkey         = aspeed_des_setkey,
 757                         .encrypt        = aspeed_des_cbc_encrypt,
 758                         .decrypt        = aspeed_des_cbc_decrypt,
 759                         .init           = aspeed_crypto_cra_init,
 760                         .exit           = aspeed_crypto_cra_exit,
 761                         .base = {
 762                                 .cra_name               = "cbc(des)",
 763                                 .cra_driver_name        = "aspeed-cbc-des",
 764                                 .cra_priority           = 300,
 765                                 .cra_flags              = CRYPTO_ALG_KERN_DRIVER_ONLY |
 766                                                           CRYPTO_ALG_ASYNC |
 767                                                           CRYPTO_ALG_NEED_FALLBACK,
 768                                 .cra_blocksize          = DES_BLOCK_SIZE,
 769                                 .cra_ctxsize            = sizeof(struct aspeed_cipher_ctx),
 770                                 .cra_alignmask          = 0x0f,
 771                                 .cra_module             = THIS_MODULE,
 772                         }
 773                 },
 774                 .alg.skcipher.op = {
 775                         .do_one_request = aspeed_crypto_do_request,
 776                 },
 777         },
 778         {
 779                 .alg.skcipher.base = {
 780                         .min_keysize    = DES3_EDE_KEY_SIZE,
 781                         .max_keysize    = DES3_EDE_KEY_SIZE,
 782                         .setkey         = aspeed_des_setkey,
 783                         .encrypt        = aspeed_tdes_ecb_encrypt,
 784                         .decrypt        = aspeed_tdes_ecb_decrypt,
 785                         .init           = aspeed_crypto_cra_init,
 786                         .exit           = aspeed_crypto_cra_exit,
 787                         .base = {
 788                                 .cra_name               = "ecb(des3_ede)",
 789                                 .cra_driver_name        = "aspeed-ecb-tdes",
 790                                 .cra_priority           = 300,
 791                                 .cra_flags              = CRYPTO_ALG_KERN_DRIVER_ONLY |
 792                                                           CRYPTO_ALG_ASYNC |
 793                                                           CRYPTO_ALG_NEED_FALLBACK,
 794                                 .cra_blocksize          = DES_BLOCK_SIZE,
 795                                 .cra_ctxsize            = sizeof(struct aspeed_cipher_ctx),
 796                                 .cra_alignmask          = 0x0f,
 797                                 .cra_module             = THIS_MODULE,
 798                         }
 799                 },
 800                 .alg.skcipher.op = {
 801                         .do_one_request = aspeed_crypto_do_request,
 802                 },
 803         },
 804         {
 805                 .alg.skcipher.base = {
 806                         .ivsize         = DES_BLOCK_SIZE,
 807                         .min_keysize    = DES3_EDE_KEY_SIZE,
 808                         .max_keysize    = DES3_EDE_KEY_SIZE,
 809                         .setkey         = aspeed_des_setkey,
 810                         .encrypt        = aspeed_tdes_cbc_encrypt,
 811                         .decrypt        = aspeed_tdes_cbc_decrypt,
 812                         .init           = aspeed_crypto_cra_init,
 813                         .exit           = aspeed_crypto_cra_exit,
 814                         .base = {
 815                                 .cra_name               = "cbc(des3_ede)",
 816                                 .cra_driver_name        = "aspeed-cbc-tdes",
 817                                 .cra_priority           = 300,
 818                                 .cra_flags              = CRYPTO_ALG_KERN_DRIVER_ONLY |
 819                                                           CRYPTO_ALG_ASYNC |
 820                                                           CRYPTO_ALG_NEED_FALLBACK,
 821                                 .cra_blocksize          = DES_BLOCK_SIZE,
 822                                 .cra_ctxsize            = sizeof(struct aspeed_cipher_ctx),
 823                                 .cra_alignmask          = 0x0f,
 824                                 .cra_module             = THIS_MODULE,
 825                         }
 826                 },
 827                 .alg.skcipher.op = {
 828                         .do_one_request = aspeed_crypto_do_request,
 829                 },
 830         },
 831 };
 832
 833 static struct aspeed_hace_alg aspeed_crypto_algs_g6[] = {
 834         {
 835                 .alg.skcipher.base = {
 836                         .ivsize         = AES_BLOCK_SIZE,
 837                         .min_keysize    = AES_MIN_KEY_SIZE,
 838                         .max_keysize    = AES_MAX_KEY_SIZE,
 839                         .setkey         = aspeed_aes_setkey,
 840                         .encrypt        = aspeed_aes_ctr_encrypt,
 841                         .decrypt        = aspeed_aes_ctr_decrypt,
 842                         .init           = aspeed_crypto_cra_init,
 843                         .exit           = aspeed_crypto_cra_exit,
 844                         .base = {
 845                                 .cra_name               = "ctr(aes)",
 846                                 .cra_driver_name        = "aspeed-ctr-aes",
 847                                 .cra_priority           = 300,
 848                                 .cra_flags              = CRYPTO_ALG_KERN_DRIVER_ONLY |
 849                                                           CRYPTO_ALG_ASYNC,
 850                                 .cra_blocksize          = 1,
 851                                 .cra_ctxsize            = sizeof(struct aspeed_cipher_ctx),
 852                                 .cra_alignmask          = 0x0f,
 853                                 .cra_module             = THIS_MODULE,
 854                         }
 855                 },
 856                 .alg.skcipher.op = {
 857                         .do_one_request = aspeed_crypto_do_request,
 858                 },
 859         },
 860         {
 861                 .alg.skcipher.base = {
 862                         .ivsize         = DES_BLOCK_SIZE,
 863                         .min_keysize    = DES_KEY_SIZE,
 864                         .max_keysize    = DES_KEY_SIZE,
 865                         .setkey         = aspeed_des_setkey,
 866                         .encrypt        = aspeed_des_ctr_encrypt,
 867                         .decrypt        = aspeed_des_ctr_decrypt,
 868                         .init           = aspeed_crypto_cra_init,
 869                         .exit           = aspeed_crypto_cra_exit,
 870                         .base = {
 871                                 .cra_name               = "ctr(des)",
 872                                 .cra_driver_name        = "aspeed-ctr-des",
 873                                 .cra_priority           = 300,
 874                                 .cra_flags              = CRYPTO_ALG_KERN_DRIVER_ONLY |
 875                                                           CRYPTO_ALG_ASYNC,
 876                                 .cra_blocksize          = 1,
 877                                 .cra_ctxsize            = sizeof(struct aspeed_cipher_ctx),
 878                                 .cra_alignmask          = 0x0f,
 879                                 .cra_module             = THIS_MODULE,
 880                         }
 881                 },
 882                 .alg.skcipher.op = {
 883                         .do_one_request = aspeed_crypto_do_request,
 884                 },
 885         },
 886         {
 887                 .alg.skcipher.base = {
 888                         .ivsize         = DES_BLOCK_SIZE,
 889                         .min_keysize    = DES3_EDE_KEY_SIZE,
 890                         .max_keysize    = DES3_EDE_KEY_SIZE,
 891                         .setkey         = aspeed_des_setkey,
 892                         .encrypt        = aspeed_tdes_ctr_encrypt,
 893                         .decrypt        = aspeed_tdes_ctr_decrypt,
 894                         .init           = aspeed_crypto_cra_init,
 895                         .exit           = aspeed_crypto_cra_exit,
 896                         .base = {
 897                                 .cra_name               = "ctr(des3_ede)",
 898                                 .cra_driver_name        = "aspeed-ctr-tdes",
 899                                 .cra_priority           = 300,
 900                                 .cra_flags              = CRYPTO_ALG_KERN_DRIVER_ONLY |
 901                                                           CRYPTO_ALG_ASYNC,
 902                                 .cra_blocksize          = 1,
 903                                 .cra_ctxsize            = sizeof(struct aspeed_cipher_ctx),
 904                                 .cra_alignmask          = 0x0f,
 905                                 .cra_module             = THIS_MODULE,
 906                         }
 907                 },
 908                 .alg.skcipher.op = {
 909                         .do_one_request = aspeed_crypto_do_request,
 910                 },
 911         },
 912
 913 };
 914
 915 void aspeed_unregister_hace_crypto_algs(struct aspeed_hace_dev *hace_dev)
 916 {
 917         int i;
 918
 919         for (i = 0; i < ARRAY_SIZE(aspeed_crypto_algs); i++)
 920                 crypto_engine_unregister_skcipher(&aspeed_crypto_algs[i].alg.skcipher);
 921
 922         if (hace_dev->version != AST2600_VERSION)
 923                 return;
 924
 925         for (i = 0; i < ARRAY_SIZE(aspeed_crypto_algs_g6); i++)
 926                 crypto_engine_unregister_skcipher(&aspeed_crypto_algs_g6[i].alg.skcipher);
 927 }
 928
 929 void aspeed_register_hace_crypto_algs(struct aspeed_hace_dev *hace_dev)
 930 {
 931         int rc, i;
 932
 933         CIPHER_DBG(hace_dev, "\n");
 934
 935         for (i = 0; i < ARRAY_SIZE(aspeed_crypto_algs); i++) {
 936                 aspeed_crypto_algs[i].hace_dev = hace_dev;
 937                 rc = crypto_engine_register_skcipher(&aspeed_crypto_algs[i].alg.skcipher);
 938                 if (rc) {
 939                         CIPHER_DBG(hace_dev, "Failed to register %s\n",
 940                                    aspeed_crypto_algs[i].alg.skcipher.base.base.cra_name);
 941                 }
 942         }
 943
 944         if (hace_dev->version != AST2600_VERSION)
 945                 return;
 946
 947         for (i = 0; i < ARRAY_SIZE(aspeed_crypto_algs_g6); i++) {
 948                 aspeed_crypto_algs_g6[i].hace_dev = hace_dev;
 949                 rc = crypto_engine_register_skcipher(&aspeed_crypto_algs_g6[i].alg.skcipher);
 950                 if (rc) {
 951                         CIPHER_DBG(hace_dev, "Failed to register %s\n",
 952                                    aspeed_crypto_algs_g6[i].alg.skcipher.base.base.cra_name);
 953                 }
 954         }
 955 }