drivers/crypto/intel/qat/qat_common/qat_algs.c

   1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
   2 /* Copyright(c) 2014 - 2020 Intel Corporation */
   3 #include <linux/module.h>
   4 #include <linux/slab.h>
   5 #include <linux/crypto.h>
   6 #include <crypto/internal/aead.h>
   7 #include <crypto/internal/cipher.h>
   8 #include <crypto/internal/skcipher.h>
   9 #include <crypto/aes.h>
  10 #include <crypto/sha1.h>
  11 #include <crypto/sha2.h>
  12 #include <crypto/hash.h>
  13 #include <crypto/hmac.h>
  14 #include <crypto/algapi.h>
  15 #include <crypto/authenc.h>
  16 #include <crypto/scatterwalk.h>
  17 #include <crypto/xts.h>
  18 #include <linux/dma-mapping.h>
  19 #include "adf_accel_devices.h"
  20 #include "qat_algs_send.h"
  21 #include "adf_common_drv.h"
  22 #include "qat_crypto.h"
  23 #include "icp_qat_hw.h"
  24 #include "icp_qat_fw.h"
  25 #include "icp_qat_fw_la.h"
  26 #include "qat_bl.h"
  27
  28 #define QAT_AES_HW_CONFIG_ENC(alg, mode) \
  29         ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
  30                                        ICP_QAT_HW_CIPHER_NO_CONVERT, \
  31                                        ICP_QAT_HW_CIPHER_ENCRYPT)
  32
  33 #define QAT_AES_HW_CONFIG_DEC(alg, mode) \
  34         ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
  35                                        ICP_QAT_HW_CIPHER_KEY_CONVERT, \
  36                                        ICP_QAT_HW_CIPHER_DECRYPT)
  37
  38 #define QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode) \
  39         ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
  40                                        ICP_QAT_HW_CIPHER_NO_CONVERT, \
  41                                        ICP_QAT_HW_CIPHER_DECRYPT)
  42
  43 #define HW_CAP_AES_V2(accel_dev) \
  44         (GET_HW_DATA(accel_dev)->accel_capabilities_mask & \
  45          ICP_ACCEL_CAPABILITIES_AES_V2)
  46
  47 static DEFINE_MUTEX(algs_lock);
  48 static unsigned int active_devs;
  49
  50 /* Common content descriptor */
  51 struct qat_alg_cd {
  52         union {
  53                 struct qat_enc { /* Encrypt content desc */
  54                         struct icp_qat_hw_cipher_algo_blk cipher;
  55                         struct icp_qat_hw_auth_algo_blk hash;
  56                 } qat_enc_cd;
  57                 struct qat_dec { /* Decrypt content desc */
  58                         struct icp_qat_hw_auth_algo_blk hash;
  59                         struct icp_qat_hw_cipher_algo_blk cipher;
  60                 } qat_dec_cd;
  61         };
  62 } __aligned(64);
  63
  64 struct qat_alg_aead_ctx {
  65         struct qat_alg_cd *enc_cd;
  66         struct qat_alg_cd *dec_cd;
  67         dma_addr_t enc_cd_paddr;
  68         dma_addr_t dec_cd_paddr;
  69         struct icp_qat_fw_la_bulk_req enc_fw_req;
  70         struct icp_qat_fw_la_bulk_req dec_fw_req;
  71         struct crypto_shash *hash_tfm;
  72         enum icp_qat_hw_auth_algo qat_hash_alg;
  73         struct qat_crypto_instance *inst;
  74         union {
  75                 struct sha1_state sha1;
  76                 struct sha256_state sha256;
  77                 struct sha512_state sha512;
  78         };
  79         char ipad[SHA512_BLOCK_SIZE]; /* sufficient for SHA-1/SHA-256 as well */
  80         char opad[SHA512_BLOCK_SIZE];
  81 };
  82
  83 struct qat_alg_skcipher_ctx {
  84         struct icp_qat_hw_cipher_algo_blk *enc_cd;
  85         struct icp_qat_hw_cipher_algo_blk *dec_cd;
  86         dma_addr_t enc_cd_paddr;
  87         dma_addr_t dec_cd_paddr;
  88         struct icp_qat_fw_la_bulk_req enc_fw_req;
  89         struct icp_qat_fw_la_bulk_req dec_fw_req;
  90         struct qat_crypto_instance *inst;
  91         struct crypto_skcipher *ftfm;
  92         struct crypto_cipher *tweak;
  93         bool fallback;
  94         int mode;
  95 };
  96
  97 static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg)
  98 {
  99         switch (qat_hash_alg) {
 100         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 101                 return ICP_QAT_HW_SHA1_STATE1_SZ;
 102         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 103                 return ICP_QAT_HW_SHA256_STATE1_SZ;
 104         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 105                 return ICP_QAT_HW_SHA512_STATE1_SZ;
 106         default:
 107                 return -EFAULT;
 108         }
 109 }
 110
 111 static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
 112                                   struct qat_alg_aead_ctx *ctx,
 113                                   const u8 *auth_key,
 114                                   unsigned int auth_keylen)
 115 {
 116         SHASH_DESC_ON_STACK(shash, ctx->hash_tfm);
 117         int block_size = crypto_shash_blocksize(ctx->hash_tfm);
 118         int digest_size = crypto_shash_digestsize(ctx->hash_tfm);
 119         __be32 *hash_state_out;
 120         __be64 *hash512_state_out;
 121         int i, offset;
 122
 123         memset(ctx->ipad, 0, block_size);
 124         memset(ctx->opad, 0, block_size);
 125         shash->tfm = ctx->hash_tfm;
 126
 127         if (auth_keylen > block_size) {
 128                 int ret = crypto_shash_digest(shash, auth_key,
 129                                               auth_keylen, ctx->ipad);
 130                 if (ret)
 131                         return ret;
 132
 133                 memcpy(ctx->opad, ctx->ipad, digest_size);
 134         } else {
 135                 memcpy(ctx->ipad, auth_key, auth_keylen);
 136                 memcpy(ctx->opad, auth_key, auth_keylen);
 137         }
 138
 139         for (i = 0; i < block_size; i++) {
 140                 char *ipad_ptr = ctx->ipad + i;
 141                 char *opad_ptr = ctx->opad + i;
 142                 *ipad_ptr ^= HMAC_IPAD_VALUE;
 143                 *opad_ptr ^= HMAC_OPAD_VALUE;
 144         }
 145
 146         if (crypto_shash_init(shash))
 147                 return -EFAULT;
 148
 149         if (crypto_shash_update(shash, ctx->ipad, block_size))
 150                 return -EFAULT;
 151
 152         hash_state_out = (__be32 *)hash->sha.state1;
 153         hash512_state_out = (__be64 *)hash_state_out;
 154
 155         switch (ctx->qat_hash_alg) {
 156         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 157                 if (crypto_shash_export(shash, &ctx->sha1))
 158                         return -EFAULT;
 159                 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
 160                         *hash_state_out = cpu_to_be32(ctx->sha1.state[i]);
 161                 break;
 162         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 163                 if (crypto_shash_export(shash, &ctx->sha256))
 164                         return -EFAULT;
 165                 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
 166                         *hash_state_out = cpu_to_be32(ctx->sha256.state[i]);
 167                 break;
 168         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 169                 if (crypto_shash_export(shash, &ctx->sha512))
 170                         return -EFAULT;
 171                 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
 172                         *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]);
 173                 break;
 174         default:
 175                 return -EFAULT;
 176         }
 177
 178         if (crypto_shash_init(shash))
 179                 return -EFAULT;
 180
 181         if (crypto_shash_update(shash, ctx->opad, block_size))
 182                 return -EFAULT;
 183
 184         offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8);
 185         if (offset < 0)
 186                 return -EFAULT;
 187
 188         hash_state_out = (__be32 *)(hash->sha.state1 + offset);
 189         hash512_state_out = (__be64 *)hash_state_out;
 190
 191         switch (ctx->qat_hash_alg) {
 192         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 193                 if (crypto_shash_export(shash, &ctx->sha1))
 194                         return -EFAULT;
 195                 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
 196                         *hash_state_out = cpu_to_be32(ctx->sha1.state[i]);
 197                 break;
 198         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 199                 if (crypto_shash_export(shash, &ctx->sha256))
 200                         return -EFAULT;
 201                 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
 202                         *hash_state_out = cpu_to_be32(ctx->sha256.state[i]);
 203                 break;
 204         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 205                 if (crypto_shash_export(shash, &ctx->sha512))
 206                         return -EFAULT;
 207                 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
 208                         *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]);
 209                 break;
 210         default:
 211                 return -EFAULT;
 212         }
 213         memzero_explicit(ctx->ipad, block_size);
 214         memzero_explicit(ctx->opad, block_size);
 215         return 0;
 216 }
 217
 218 static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)
 219 {
 220         header->hdr_flags =
 221                 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
 222         header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA;
 223         header->comn_req_flags =
 224                 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
 225                                             QAT_COMN_PTR_TYPE_SGL);
 226         ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,
 227                                   ICP_QAT_FW_LA_PARTIAL_NONE);
 228         ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
 229                                            ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
 230         ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
 231                                 ICP_QAT_FW_LA_NO_PROTO);
 232         ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
 233                                        ICP_QAT_FW_LA_NO_UPDATE_STATE);
 234 }
 235
 236 static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm,
 237                                          int alg,
 238                                          struct crypto_authenc_keys *keys,
 239                                          int mode)
 240 {
 241         struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);
 242         unsigned int digestsize = crypto_aead_authsize(aead_tfm);
 243         struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd;
 244         struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher;
 245         struct icp_qat_hw_auth_algo_blk *hash =
 246                 (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx +
 247                 sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen);
 248         struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req;
 249         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
 250         struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
 251         void *ptr = &req_tmpl->cd_ctrl;
 252         struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
 253         struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
 254
 255         /* CD setup */
 256         cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);
 257         memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
 258         hash->sha.inner_setup.auth_config.config =
 259                 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
 260                                              ctx->qat_hash_alg, digestsize);
 261         hash->sha.inner_setup.auth_counter.counter =
 262                 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
 263
 264         if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
 265                 return -EFAULT;
 266
 267         /* Request setup */
 268         qat_alg_init_common_hdr(header);
 269         header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 270         ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
 271                                            ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
 272         ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
 273                                    ICP_QAT_FW_LA_RET_AUTH_RES);
 274         ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
 275                                    ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
 276         cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
 277         cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
 278
 279         /* Cipher CD config setup */
 280         cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
 281         cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
 282         cipher_cd_ctrl->cipher_cfg_offset = 0;
 283         ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
 284         ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
 285         /* Auth CD config setup */
 286         hash_cd_ctrl->hash_cfg_offset = ((char *)hash - (char *)cipher) >> 3;
 287         hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
 288         hash_cd_ctrl->inner_res_sz = digestsize;
 289         hash_cd_ctrl->final_sz = digestsize;
 290
 291         switch (ctx->qat_hash_alg) {
 292         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 293                 hash_cd_ctrl->inner_state1_sz =
 294                         round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
 295                 hash_cd_ctrl->inner_state2_sz =
 296                         round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
 297                 break;
 298         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 299                 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
 300                 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
 301                 break;
 302         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 303                 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
 304                 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
 305                 break;
 306         default:
 307                 break;
 308         }
 309         hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
 310                         ((sizeof(struct icp_qat_hw_auth_setup) +
 311                          round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
 312         ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
 313         ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
 314         return 0;
 315 }
 316
 317 static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm,
 318                                          int alg,
 319                                          struct crypto_authenc_keys *keys,
 320                                          int mode)
 321 {
 322         struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);
 323         unsigned int digestsize = crypto_aead_authsize(aead_tfm);
 324         struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd;
 325         struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash;
 326         struct icp_qat_hw_cipher_algo_blk *cipher =
 327                 (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx +
 328                 sizeof(struct icp_qat_hw_auth_setup) +
 329                 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2);
 330         struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req;
 331         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
 332         struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
 333         void *ptr = &req_tmpl->cd_ctrl;
 334         struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
 335         struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
 336         struct icp_qat_fw_la_auth_req_params *auth_param =
 337                 (struct icp_qat_fw_la_auth_req_params *)
 338                 ((char *)&req_tmpl->serv_specif_rqpars +
 339                 sizeof(struct icp_qat_fw_la_cipher_req_params));
 340
 341         /* CD setup */
 342         cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg, mode);
 343         memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
 344         hash->sha.inner_setup.auth_config.config =
 345                 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
 346                                              ctx->qat_hash_alg,
 347                                              digestsize);
 348         hash->sha.inner_setup.auth_counter.counter =
 349                 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
 350
 351         if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
 352                 return -EFAULT;
 353
 354         /* Request setup */
 355         qat_alg_init_common_hdr(header);
 356         header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 357         ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
 358                                            ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
 359         ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
 360                                    ICP_QAT_FW_LA_NO_RET_AUTH_RES);
 361         ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
 362                                    ICP_QAT_FW_LA_CMP_AUTH_RES);
 363         cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
 364         cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
 365
 366         /* Cipher CD config setup */
 367         cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
 368         cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
 369         cipher_cd_ctrl->cipher_cfg_offset =
 370                 (sizeof(struct icp_qat_hw_auth_setup) +
 371                  roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3;
 372         ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
 373         ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
 374
 375         /* Auth CD config setup */
 376         hash_cd_ctrl->hash_cfg_offset = 0;
 377         hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
 378         hash_cd_ctrl->inner_res_sz = digestsize;
 379         hash_cd_ctrl->final_sz = digestsize;
 380
 381         switch (ctx->qat_hash_alg) {
 382         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 383                 hash_cd_ctrl->inner_state1_sz =
 384                         round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
 385                 hash_cd_ctrl->inner_state2_sz =
 386                         round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
 387                 break;
 388         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 389                 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
 390                 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
 391                 break;
 392         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 393                 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
 394                 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
 395                 break;
 396         default:
 397                 break;
 398         }
 399
 400         hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
 401                         ((sizeof(struct icp_qat_hw_auth_setup) +
 402                          round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
 403         auth_param->auth_res_sz = digestsize;
 404         ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
 405         ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
 406         return 0;
 407 }
 408
 409 static void qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx *ctx,
 410                                       struct icp_qat_fw_la_bulk_req *req,
 411                                       struct icp_qat_hw_cipher_algo_blk *cd,
 412                                       const u8 *key, unsigned int keylen)
 413 {
 414         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
 415         struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
 416         struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl;
 417         bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);
 418         int mode = ctx->mode;
 419
 420         qat_alg_init_common_hdr(header);
 421         header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
 422         cd_pars->u.s.content_desc_params_sz =
 423                                 sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3;
 424
 425         if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) {
 426                 ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags,
 427                                              ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE);
 428
 429                 /* Store both XTS keys in CD, only the first key is sent
 430                  * to the HW, the second key is used for tweak calculation
 431                  */
 432                 memcpy(cd->ucs_aes.key, key, keylen);
 433                 keylen = keylen / 2;
 434         } else if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_CTR_MODE) {
 435                 ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags,
 436                                              ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE);
 437                 memcpy(cd->ucs_aes.key, key, keylen);
 438                 keylen = round_up(keylen, 16);
 439         } else {
 440                 memcpy(cd->aes.key, key, keylen);
 441         }
 442
 443         /* Cipher CD config setup */
 444         cd_ctrl->cipher_key_sz = keylen >> 3;
 445         cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
 446         cd_ctrl->cipher_cfg_offset = 0;
 447         ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
 448         ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
 449 }
 450
 451 static void qat_alg_skcipher_init_enc(struct qat_alg_skcipher_ctx *ctx,
 452                                       int alg, const u8 *key,
 453                                       unsigned int keylen, int mode)
 454 {
 455         struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;
 456         struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;
 457         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
 458
 459         qat_alg_skcipher_init_com(ctx, req, enc_cd, key, keylen);
 460         cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
 461         enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);
 462 }
 463
 464 static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen,
 465                                     u8 *key_reverse)
 466 {
 467         struct crypto_aes_ctx aes_expanded;
 468         int nrounds;
 469         u8 *key;
 470
 471         aes_expandkey(&aes_expanded, key_forward, keylen);
 472         if (keylen == AES_KEYSIZE_128) {
 473                 nrounds = 10;
 474                 key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds);
 475                 memcpy(key_reverse, key, AES_BLOCK_SIZE);
 476         } else {
 477                 /* AES_KEYSIZE_256 */
 478                 nrounds = 14;
 479                 key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds);
 480                 memcpy(key_reverse, key, AES_BLOCK_SIZE);
 481                 memcpy(key_reverse + AES_BLOCK_SIZE, key - AES_BLOCK_SIZE,
 482                        AES_BLOCK_SIZE);
 483         }
 484 }
 485
 486 static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx,
 487                                       int alg, const u8 *key,
 488                                       unsigned int keylen, int mode)
 489 {
 490         struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;
 491         struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;
 492         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
 493         bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);
 494
 495         qat_alg_skcipher_init_com(ctx, req, dec_cd, key, keylen);
 496         cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
 497
 498         if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) {
 499                 /* Key reversing not supported, set no convert */
 500                 dec_cd->aes.cipher_config.val =
 501                                 QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode);
 502
 503                 /* In-place key reversal */
 504                 qat_alg_xts_reverse_key(dec_cd->ucs_aes.key, keylen / 2,
 505                                         dec_cd->ucs_aes.key);
 506         } else if (mode != ICP_QAT_HW_CIPHER_CTR_MODE) {
 507                 dec_cd->aes.cipher_config.val =
 508                                         QAT_AES_HW_CONFIG_DEC(alg, mode);
 509         } else {
 510                 dec_cd->aes.cipher_config.val =
 511                                         QAT_AES_HW_CONFIG_ENC(alg, mode);
 512         }
 513 }
 514
 515 static int qat_alg_validate_key(int key_len, int *alg, int mode)
 516 {
 517         if (mode != ICP_QAT_HW_CIPHER_XTS_MODE) {
 518                 switch (key_len) {
 519                 case AES_KEYSIZE_128:
 520                         *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
 521                         break;
 522                 case AES_KEYSIZE_192:
 523                         *alg = ICP_QAT_HW_CIPHER_ALGO_AES192;
 524                         break;
 525                 case AES_KEYSIZE_256:
 526                         *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
 527                         break;
 528                 default:
 529                         return -EINVAL;
 530                 }
 531         } else {
 532                 switch (key_len) {
 533                 case AES_KEYSIZE_128 << 1:
 534                         *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
 535                         break;
 536                 case AES_KEYSIZE_256 << 1:
 537                         *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
 538                         break;
 539                 default:
 540                         return -EINVAL;
 541                 }
 542         }
 543         return 0;
 544 }
 545
 546 static int qat_alg_aead_init_sessions(struct crypto_aead *tfm, const u8 *key,
 547                                       unsigned int keylen,  int mode)
 548 {
 549         struct crypto_authenc_keys keys;
 550         int alg;
 551
 552         if (crypto_authenc_extractkeys(&keys, key, keylen))
 553                 goto bad_key;
 554
 555         if (qat_alg_validate_key(keys.enckeylen, &alg, mode))
 556                 goto bad_key;
 557
 558         if (qat_alg_aead_init_enc_session(tfm, alg, &keys, mode))
 559                 goto error;
 560
 561         if (qat_alg_aead_init_dec_session(tfm, alg, &keys, mode))
 562                 goto error;
 563
 564         memzero_explicit(&keys, sizeof(keys));
 565         return 0;
 566 bad_key:
 567         memzero_explicit(&keys, sizeof(keys));
 568         return -EINVAL;
 569 error:
 570         memzero_explicit(&keys, sizeof(keys));
 571         return -EFAULT;
 572 }
 573
 574 static int qat_alg_skcipher_init_sessions(struct qat_alg_skcipher_ctx *ctx,
 575                                           const u8 *key,
 576                                           unsigned int keylen,
 577                                           int mode)
 578 {
 579         int alg;
 580
 581         if (qat_alg_validate_key(keylen, &alg, mode))
 582                 return -EINVAL;
 583
 584         qat_alg_skcipher_init_enc(ctx, alg, key, keylen, mode);
 585         qat_alg_skcipher_init_dec(ctx, alg, key, keylen, mode);
 586         return 0;
 587 }
 588
 589 static int qat_alg_aead_rekey(struct crypto_aead *tfm, const u8 *key,
 590                               unsigned int keylen)
 591 {
 592         struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
 593
 594         memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
 595         memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
 596         memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
 597         memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
 598
 599         return qat_alg_aead_init_sessions(tfm, key, keylen,
 600                                           ICP_QAT_HW_CIPHER_CBC_MODE);
 601 }
 602
 603 static int qat_alg_aead_newkey(struct crypto_aead *tfm, const u8 *key,
 604                                unsigned int keylen)
 605 {
 606         struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
 607         struct qat_crypto_instance *inst = NULL;
 608         int node = numa_node_id();
 609         struct device *dev;
 610         int ret;
 611
 612         inst = qat_crypto_get_instance_node(node);
 613         if (!inst)
 614                 return -EINVAL;
 615         dev = &GET_DEV(inst->accel_dev);
 616         ctx->inst = inst;
 617         ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
 618                                          &ctx->enc_cd_paddr,
 619                                          GFP_ATOMIC);
 620         if (!ctx->enc_cd) {
 621                 ret = -ENOMEM;
 622                 goto out_free_inst;
 623         }
 624         ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
 625                                          &ctx->dec_cd_paddr,
 626                                          GFP_ATOMIC);
 627         if (!ctx->dec_cd) {
 628                 ret = -ENOMEM;
 629                 goto out_free_enc;
 630         }
 631
 632         ret = qat_alg_aead_init_sessions(tfm, key, keylen,
 633                                          ICP_QAT_HW_CIPHER_CBC_MODE);
 634         if (ret)
 635                 goto out_free_all;
 636
 637         return 0;
 638
 639 out_free_all:
 640         memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
 641         dma_free_coherent(dev, sizeof(struct qat_alg_cd),
 642                           ctx->dec_cd, ctx->dec_cd_paddr);
 643         ctx->dec_cd = NULL;
 644 out_free_enc:
 645         memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
 646         dma_free_coherent(dev, sizeof(struct qat_alg_cd),
 647                           ctx->enc_cd, ctx->enc_cd_paddr);
 648         ctx->enc_cd = NULL;
 649 out_free_inst:
 650         ctx->inst = NULL;
 651         qat_crypto_put_instance(inst);
 652         return ret;
 653 }
 654
 655 static int qat_alg_aead_setkey(struct crypto_aead *tfm, const u8 *key,
 656                                unsigned int keylen)
 657 {
 658         struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
 659
 660         if (ctx->enc_cd)
 661                 return qat_alg_aead_rekey(tfm, key, keylen);
 662         else
 663                 return qat_alg_aead_newkey(tfm, key, keylen);
 664 }
 665
 666 static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
 667                                   struct qat_crypto_request *qat_req)
 668 {
 669         struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx;
 670         struct qat_crypto_instance *inst = ctx->inst;
 671         struct aead_request *areq = qat_req->aead_req;
 672         u8 stat_filed = qat_resp->comn_resp.comn_status;
 673         int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
 674
 675         qat_bl_free_bufl(inst->accel_dev, &qat_req->buf);
 676         if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
 677                 res = -EBADMSG;
 678         aead_request_complete(areq, res);
 679 }
 680
 681 static void qat_alg_update_iv_ctr_mode(struct qat_crypto_request *qat_req)
 682 {
 683         struct skcipher_request *sreq = qat_req->skcipher_req;
 684         u64 iv_lo_prev;
 685         u64 iv_lo;
 686         u64 iv_hi;
 687
 688         memcpy(qat_req->iv, sreq->iv, AES_BLOCK_SIZE);
 689
 690         iv_lo = be64_to_cpu(qat_req->iv_lo);
 691         iv_hi = be64_to_cpu(qat_req->iv_hi);
 692
 693         iv_lo_prev = iv_lo;
 694         iv_lo += DIV_ROUND_UP(sreq->cryptlen, AES_BLOCK_SIZE);
 695         if (iv_lo < iv_lo_prev)
 696                 iv_hi++;
 697
 698         qat_req->iv_lo = cpu_to_be64(iv_lo);
 699         qat_req->iv_hi = cpu_to_be64(iv_hi);
 700 }
 701
 702 static void qat_alg_update_iv_cbc_mode(struct qat_crypto_request *qat_req)
 703 {
 704         struct skcipher_request *sreq = qat_req->skcipher_req;
 705         int offset = sreq->cryptlen - AES_BLOCK_SIZE;
 706         struct scatterlist *sgl;
 707
 708         if (qat_req->encryption)
 709                 sgl = sreq->dst;
 710         else
 711                 sgl = sreq->src;
 712
 713         scatterwalk_map_and_copy(qat_req->iv, sgl, offset, AES_BLOCK_SIZE, 0);
 714 }
 715
 716 static void qat_alg_update_iv(struct qat_crypto_request *qat_req)
 717 {
 718         struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
 719         struct device *dev = &GET_DEV(ctx->inst->accel_dev);
 720
 721         switch (ctx->mode) {
 722         case ICP_QAT_HW_CIPHER_CTR_MODE:
 723                 qat_alg_update_iv_ctr_mode(qat_req);
 724                 break;
 725         case ICP_QAT_HW_CIPHER_CBC_MODE:
 726                 qat_alg_update_iv_cbc_mode(qat_req);
 727                 break;
 728         case ICP_QAT_HW_CIPHER_XTS_MODE:
 729                 break;
 730         default:
 731                 dev_warn(dev, "Unsupported IV update for cipher mode %d\n",
 732                          ctx->mode);
 733         }
 734 }
 735
 736 static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
 737                                       struct qat_crypto_request *qat_req)
 738 {
 739         struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
 740         struct qat_crypto_instance *inst = ctx->inst;
 741         struct skcipher_request *sreq = qat_req->skcipher_req;
 742         u8 stat_filed = qat_resp->comn_resp.comn_status;
 743         int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
 744
 745         qat_bl_free_bufl(inst->accel_dev, &qat_req->buf);
 746         if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
 747                 res = -EINVAL;
 748
 749         if (qat_req->encryption)
 750                 qat_alg_update_iv(qat_req);
 751
 752         memcpy(sreq->iv, qat_req->iv, AES_BLOCK_SIZE);
 753
 754         skcipher_request_complete(sreq, res);
 755 }
 756
 757 void qat_alg_callback(void *resp)
 758 {
 759         struct icp_qat_fw_la_resp *qat_resp = resp;
 760         struct qat_crypto_request *qat_req =
 761                                 (void *)(__force long)qat_resp->opaque_data;
 762         struct qat_instance_backlog *backlog = qat_req->alg_req.backlog;
 763
 764         qat_req->cb(qat_resp, qat_req);
 765
 766         qat_alg_send_backlog(backlog);
 767 }
 768
 769 static int qat_alg_send_sym_message(struct qat_crypto_request *qat_req,
 770                                     struct qat_crypto_instance *inst,
 771                                     struct crypto_async_request *base)
 772 {
 773         struct qat_alg_req *alg_req = &qat_req->alg_req;
 774
 775         alg_req->fw_req = (u32 *)&qat_req->req;
 776         alg_req->tx_ring = inst->sym_tx;
 777         alg_req->base = base;
 778         alg_req->backlog = &inst->backlog;
 779
 780         return qat_alg_send_message(alg_req);
 781 }
 782
 783 static int qat_alg_aead_dec(struct aead_request *areq)
 784 {
 785         struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
 786         struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
 787         struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 788         struct qat_crypto_request *qat_req = aead_request_ctx(areq);
 789         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 790         struct icp_qat_fw_la_auth_req_params *auth_param;
 791         struct icp_qat_fw_la_bulk_req *msg;
 792         int digst_size = crypto_aead_authsize(aead_tfm);
 793         gfp_t f = qat_algs_alloc_flags(&areq->base);
 794         int ret;
 795         u32 cipher_len;
 796
 797         cipher_len = areq->cryptlen - digst_size;
 798         if (cipher_len % AES_BLOCK_SIZE != 0)
 799                 return -EINVAL;
 800
 801         ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst,
 802                                  &qat_req->buf, NULL, f);
 803         if (unlikely(ret))
 804                 return ret;
 805
 806         msg = &qat_req->req;
 807         *msg = ctx->dec_fw_req;
 808         qat_req->aead_ctx = ctx;
 809         qat_req->aead_req = areq;
 810         qat_req->cb = qat_aead_alg_callback;
 811         qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
 812         qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
 813         qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
 814         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
 815         cipher_param->cipher_length = cipher_len;
 816         cipher_param->cipher_offset = areq->assoclen;
 817         memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE);
 818         auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param));
 819         auth_param->auth_off = 0;
 820         auth_param->auth_len = areq->assoclen + cipher_param->cipher_length;
 821
 822         ret = qat_alg_send_sym_message(qat_req, ctx->inst, &areq->base);
 823         if (ret == -ENOSPC)
 824                 qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf);
 825
 826         return ret;
 827 }
 828
 829 static int qat_alg_aead_enc(struct aead_request *areq)
 830 {
 831         struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
 832         struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
 833         struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 834         struct qat_crypto_request *qat_req = aead_request_ctx(areq);
 835         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 836         struct icp_qat_fw_la_auth_req_params *auth_param;
 837         gfp_t f = qat_algs_alloc_flags(&areq->base);
 838         struct icp_qat_fw_la_bulk_req *msg;
 839         u8 *iv = areq->iv;
 840         int ret;
 841
 842         if (areq->cryptlen % AES_BLOCK_SIZE != 0)
 843                 return -EINVAL;
 844
 845         ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst,
 846                                  &qat_req->buf, NULL, f);
 847         if (unlikely(ret))
 848                 return ret;
 849
 850         msg = &qat_req->req;
 851         *msg = ctx->enc_fw_req;
 852         qat_req->aead_ctx = ctx;
 853         qat_req->aead_req = areq;
 854         qat_req->cb = qat_aead_alg_callback;
 855         qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
 856         qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
 857         qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
 858         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
 859         auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param));
 860
 861         memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);
 862         cipher_param->cipher_length = areq->cryptlen;
 863         cipher_param->cipher_offset = areq->assoclen;
 864
 865         auth_param->auth_off = 0;
 866         auth_param->auth_len = areq->assoclen + areq->cryptlen;
 867
 868         ret = qat_alg_send_sym_message(qat_req, ctx->inst, &areq->base);
 869         if (ret == -ENOSPC)
 870                 qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf);
 871
 872         return ret;
 873 }
 874
 875 static int qat_alg_skcipher_rekey(struct qat_alg_skcipher_ctx *ctx,
 876                                   const u8 *key, unsigned int keylen,
 877                                   int mode)
 878 {
 879         memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
 880         memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
 881         memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
 882         memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
 883
 884         return qat_alg_skcipher_init_sessions(ctx, key, keylen, mode);
 885 }
 886
 887 static int qat_alg_skcipher_newkey(struct qat_alg_skcipher_ctx *ctx,
 888                                    const u8 *key, unsigned int keylen,
 889                                    int mode)
 890 {
 891         struct qat_crypto_instance *inst = NULL;
 892         struct device *dev;
 893         int node = numa_node_id();
 894         int ret;
 895
 896         inst = qat_crypto_get_instance_node(node);
 897         if (!inst)
 898                 return -EINVAL;
 899         dev = &GET_DEV(inst->accel_dev);
 900         ctx->inst = inst;
 901         ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
 902                                          &ctx->enc_cd_paddr,
 903                                          GFP_ATOMIC);
 904         if (!ctx->enc_cd) {
 905                 ret = -ENOMEM;
 906                 goto out_free_instance;
 907         }
 908         ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
 909                                          &ctx->dec_cd_paddr,
 910                                          GFP_ATOMIC);
 911         if (!ctx->dec_cd) {
 912                 ret = -ENOMEM;
 913                 goto out_free_enc;
 914         }
 915
 916         ret = qat_alg_skcipher_init_sessions(ctx, key, keylen, mode);
 917         if (ret)
 918                 goto out_free_all;
 919
 920         return 0;
 921
 922 out_free_all:
 923         memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
 924         dma_free_coherent(dev, sizeof(*ctx->dec_cd),
 925                           ctx->dec_cd, ctx->dec_cd_paddr);
 926         ctx->dec_cd = NULL;
 927 out_free_enc:
 928         memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
 929         dma_free_coherent(dev, sizeof(*ctx->enc_cd),
 930                           ctx->enc_cd, ctx->enc_cd_paddr);
 931         ctx->enc_cd = NULL;
 932 out_free_instance:
 933         ctx->inst = NULL;
 934         qat_crypto_put_instance(inst);
 935         return ret;
 936 }
 937
 938 static int qat_alg_skcipher_setkey(struct crypto_skcipher *tfm,
 939                                    const u8 *key, unsigned int keylen,
 940                                    int mode)
 941 {
 942         struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
 943
 944         ctx->mode = mode;
 945
 946         if (ctx->enc_cd)
 947                 return qat_alg_skcipher_rekey(ctx, key, keylen, mode);
 948         else
 949                 return qat_alg_skcipher_newkey(ctx, key, keylen, mode);
 950 }
 951
 952 static int qat_alg_skcipher_cbc_setkey(struct crypto_skcipher *tfm,
 953                                        const u8 *key, unsigned int keylen)
 954 {
 955         return qat_alg_skcipher_setkey(tfm, key, keylen,
 956                                        ICP_QAT_HW_CIPHER_CBC_MODE);
 957 }
 958
 959 static int qat_alg_skcipher_ctr_setkey(struct crypto_skcipher *tfm,
 960                                        const u8 *key, unsigned int keylen)
 961 {
 962         return qat_alg_skcipher_setkey(tfm, key, keylen,
 963                                        ICP_QAT_HW_CIPHER_CTR_MODE);
 964 }
 965
 966 static int qat_alg_skcipher_xts_setkey(struct crypto_skcipher *tfm,
 967                                        const u8 *key, unsigned int keylen)
 968 {
 969         struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
 970         int ret;
 971
 972         ret = xts_verify_key(tfm, key, keylen);
 973         if (ret)
 974                 return ret;
 975
 976         if (keylen >> 1 == AES_KEYSIZE_192) {
 977                 ret = crypto_skcipher_setkey(ctx->ftfm, key, keylen);
 978                 if (ret)
 979                         return ret;
 980
 981                 ctx->fallback = true;
 982
 983                 return 0;
 984         }
 985
 986         ctx->fallback = false;
 987
 988         ret = qat_alg_skcipher_setkey(tfm, key, keylen,
 989                                       ICP_QAT_HW_CIPHER_XTS_MODE);
 990         if (ret)
 991                 return ret;
 992
 993         if (HW_CAP_AES_V2(ctx->inst->accel_dev))
 994                 ret = crypto_cipher_setkey(ctx->tweak, key + (keylen / 2),
 995                                            keylen / 2);
 996
 997         return ret;
 998 }
 999
1000 static void qat_alg_set_req_iv(struct qat_crypto_request *qat_req)
1001 {
1002         struct icp_qat_fw_la_cipher_req_params *cipher_param;
1003         struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
1004         bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);
1005         u8 *iv = qat_req->skcipher_req->iv;
1006
1007         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1008
1009         if (aes_v2_capable && ctx->mode == ICP_QAT_HW_CIPHER_XTS_MODE)
1010                 crypto_cipher_encrypt_one(ctx->tweak,
1011                                           (u8 *)cipher_param->u.cipher_IV_array,
1012                                           iv);
1013         else
1014                 memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);
1015 }
1016
1017 static int qat_alg_skcipher_encrypt(struct skcipher_request *req)
1018 {
1019         struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1020         struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
1021         struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1022         struct qat_crypto_request *qat_req = skcipher_request_ctx(req);
1023         struct icp_qat_fw_la_cipher_req_params *cipher_param;
1024         gfp_t f = qat_algs_alloc_flags(&req->base);
1025         struct icp_qat_fw_la_bulk_req *msg;
1026         int ret;
1027
1028         if (req->cryptlen == 0)
1029                 return 0;
1030
1031         ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, req->src, req->dst,
1032                                  &qat_req->buf, NULL, f);
1033         if (unlikely(ret))
1034                 return ret;
1035
1036         msg = &qat_req->req;
1037         *msg = ctx->enc_fw_req;
1038         qat_req->skcipher_ctx = ctx;
1039         qat_req->skcipher_req = req;
1040         qat_req->cb = qat_skcipher_alg_callback;
1041         qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
1042         qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1043         qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1044         qat_req->encryption = true;
1045         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1046         cipher_param->cipher_length = req->cryptlen;
1047         cipher_param->cipher_offset = 0;
1048
1049         qat_alg_set_req_iv(qat_req);
1050
1051         ret = qat_alg_send_sym_message(qat_req, ctx->inst, &req->base);
1052         if (ret == -ENOSPC)
1053                 qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf);
1054
1055         return ret;
1056 }
1057
1058 static int qat_alg_skcipher_blk_encrypt(struct skcipher_request *req)
1059 {
1060         if (req->cryptlen % AES_BLOCK_SIZE != 0)
1061                 return -EINVAL;
1062
1063         return qat_alg_skcipher_encrypt(req);
1064 }
1065
1066 static int qat_alg_skcipher_xts_encrypt(struct skcipher_request *req)
1067 {
1068         struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1069         struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(stfm);
1070         struct skcipher_request *nreq = skcipher_request_ctx(req);
1071
1072         if (req->cryptlen < XTS_BLOCK_SIZE)
1073                 return -EINVAL;
1074
1075         if (ctx->fallback) {
1076                 memcpy(nreq, req, sizeof(*req));
1077                 skcipher_request_set_tfm(nreq, ctx->ftfm);
1078                 return crypto_skcipher_encrypt(nreq);
1079         }
1080
1081         return qat_alg_skcipher_encrypt(req);
1082 }
1083
1084 static int qat_alg_skcipher_decrypt(struct skcipher_request *req)
1085 {
1086         struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1087         struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
1088         struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1089         struct qat_crypto_request *qat_req = skcipher_request_ctx(req);
1090         struct icp_qat_fw_la_cipher_req_params *cipher_param;
1091         gfp_t f = qat_algs_alloc_flags(&req->base);
1092         struct icp_qat_fw_la_bulk_req *msg;
1093         int ret;
1094
1095         if (req->cryptlen == 0)
1096                 return 0;
1097
1098         ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, req->src, req->dst,
1099                                  &qat_req->buf, NULL, f);
1100         if (unlikely(ret))
1101                 return ret;
1102
1103         msg = &qat_req->req;
1104         *msg = ctx->dec_fw_req;
1105         qat_req->skcipher_ctx = ctx;
1106         qat_req->skcipher_req = req;
1107         qat_req->cb = qat_skcipher_alg_callback;
1108         qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
1109         qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1110         qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1111         qat_req->encryption = false;
1112         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1113         cipher_param->cipher_length = req->cryptlen;
1114         cipher_param->cipher_offset = 0;
1115
1116         qat_alg_set_req_iv(qat_req);
1117         qat_alg_update_iv(qat_req);
1118
1119         ret = qat_alg_send_sym_message(qat_req, ctx->inst, &req->base);
1120         if (ret == -ENOSPC)
1121                 qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf);
1122
1123         return ret;
1124 }
1125
1126 static int qat_alg_skcipher_blk_decrypt(struct skcipher_request *req)
1127 {
1128         if (req->cryptlen % AES_BLOCK_SIZE != 0)
1129                 return -EINVAL;
1130
1131         return qat_alg_skcipher_decrypt(req);
1132 }
1133
1134 static int qat_alg_skcipher_xts_decrypt(struct skcipher_request *req)
1135 {
1136         struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1137         struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(stfm);
1138         struct skcipher_request *nreq = skcipher_request_ctx(req);
1139
1140         if (req->cryptlen < XTS_BLOCK_SIZE)
1141                 return -EINVAL;
1142
1143         if (ctx->fallback) {
1144                 memcpy(nreq, req, sizeof(*req));
1145                 skcipher_request_set_tfm(nreq, ctx->ftfm);
1146                 return crypto_skcipher_decrypt(nreq);
1147         }
1148
1149         return qat_alg_skcipher_decrypt(req);
1150 }
1151
1152 static int qat_alg_aead_init(struct crypto_aead *tfm,
1153                              enum icp_qat_hw_auth_algo hash,
1154                              const char *hash_name)
1155 {
1156         struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
1157
1158         ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
1159         if (IS_ERR(ctx->hash_tfm))
1160                 return PTR_ERR(ctx->hash_tfm);
1161         ctx->qat_hash_alg = hash;
1162         crypto_aead_set_reqsize(tfm, sizeof(struct qat_crypto_request));
1163         return 0;
1164 }
1165
1166 static int qat_alg_aead_sha1_init(struct crypto_aead *tfm)
1167 {
1168         return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1");
1169 }
1170
1171 static int qat_alg_aead_sha256_init(struct crypto_aead *tfm)
1172 {
1173         return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256");
1174 }
1175
1176 static int qat_alg_aead_sha512_init(struct crypto_aead *tfm)
1177 {
1178         return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512");
1179 }
1180
1181 static void qat_alg_aead_exit(struct crypto_aead *tfm)
1182 {
1183         struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
1184         struct qat_crypto_instance *inst = ctx->inst;
1185         struct device *dev;
1186
1187         crypto_free_shash(ctx->hash_tfm);
1188
1189         if (!inst)
1190                 return;
1191
1192         dev = &GET_DEV(inst->accel_dev);
1193         if (ctx->enc_cd) {
1194                 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
1195                 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1196                                   ctx->enc_cd, ctx->enc_cd_paddr);
1197         }
1198         if (ctx->dec_cd) {
1199                 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
1200                 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1201                                   ctx->dec_cd, ctx->dec_cd_paddr);
1202         }
1203         qat_crypto_put_instance(inst);
1204 }
1205
1206 static int qat_alg_skcipher_init_tfm(struct crypto_skcipher *tfm)
1207 {
1208         crypto_skcipher_set_reqsize(tfm, sizeof(struct qat_crypto_request));
1209         return 0;
1210 }
1211
1212 static int qat_alg_skcipher_init_xts_tfm(struct crypto_skcipher *tfm)
1213 {
1214         struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1215         int reqsize;
1216
1217         ctx->ftfm = crypto_alloc_skcipher("xts(aes)", 0,
1218                                           CRYPTO_ALG_NEED_FALLBACK);
1219         if (IS_ERR(ctx->ftfm))
1220                 return PTR_ERR(ctx->ftfm);
1221
1222         ctx->tweak = crypto_alloc_cipher("aes", 0, 0);
1223         if (IS_ERR(ctx->tweak)) {
1224                 crypto_free_skcipher(ctx->ftfm);
1225                 return PTR_ERR(ctx->tweak);
1226         }
1227
1228         reqsize = max(sizeof(struct qat_crypto_request),
1229                       sizeof(struct skcipher_request) +
1230                       crypto_skcipher_reqsize(ctx->ftfm));
1231         crypto_skcipher_set_reqsize(tfm, reqsize);
1232
1233         return 0;
1234 }
1235
1236 static void qat_alg_skcipher_exit_tfm(struct crypto_skcipher *tfm)
1237 {
1238         struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1239         struct qat_crypto_instance *inst = ctx->inst;
1240         struct device *dev;
1241
1242         if (!inst)
1243                 return;
1244
1245         dev = &GET_DEV(inst->accel_dev);
1246         if (ctx->enc_cd) {
1247                 memset(ctx->enc_cd, 0,
1248                        sizeof(struct icp_qat_hw_cipher_algo_blk));
1249                 dma_free_coherent(dev,
1250                                   sizeof(struct icp_qat_hw_cipher_algo_blk),
1251                                   ctx->enc_cd, ctx->enc_cd_paddr);
1252         }
1253         if (ctx->dec_cd) {
1254                 memset(ctx->dec_cd, 0,
1255                        sizeof(struct icp_qat_hw_cipher_algo_blk));
1256                 dma_free_coherent(dev,
1257                                   sizeof(struct icp_qat_hw_cipher_algo_blk),
1258                                   ctx->dec_cd, ctx->dec_cd_paddr);
1259         }
1260         qat_crypto_put_instance(inst);
1261 }
1262
1263 static void qat_alg_skcipher_exit_xts_tfm(struct crypto_skcipher *tfm)
1264 {
1265         struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1266
1267         if (ctx->ftfm)
1268                 crypto_free_skcipher(ctx->ftfm);
1269
1270         if (ctx->tweak)
1271                 crypto_free_cipher(ctx->tweak);
1272
1273         qat_alg_skcipher_exit_tfm(tfm);
1274 }
1275
1276 static struct aead_alg qat_aeads[] = { {
1277         .base = {
1278                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1279                 .cra_driver_name = "qat_aes_cbc_hmac_sha1",
1280                 .cra_priority = 4001,
1281                 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1282                 .cra_blocksize = AES_BLOCK_SIZE,
1283                 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1284                 .cra_module = THIS_MODULE,
1285         },
1286         .init = qat_alg_aead_sha1_init,
1287         .exit = qat_alg_aead_exit,
1288         .setkey = qat_alg_aead_setkey,
1289         .decrypt = qat_alg_aead_dec,
1290         .encrypt = qat_alg_aead_enc,
1291         .ivsize = AES_BLOCK_SIZE,
1292         .maxauthsize = SHA1_DIGEST_SIZE,
1293 }, {
1294         .base = {
1295                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1296                 .cra_driver_name = "qat_aes_cbc_hmac_sha256",
1297                 .cra_priority = 4001,
1298                 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1299                 .cra_blocksize = AES_BLOCK_SIZE,
1300                 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1301                 .cra_module = THIS_MODULE,
1302         },
1303         .init = qat_alg_aead_sha256_init,
1304         .exit = qat_alg_aead_exit,
1305         .setkey = qat_alg_aead_setkey,
1306         .decrypt = qat_alg_aead_dec,
1307         .encrypt = qat_alg_aead_enc,
1308         .ivsize = AES_BLOCK_SIZE,
1309         .maxauthsize = SHA256_DIGEST_SIZE,
1310 }, {
1311         .base = {
1312                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1313                 .cra_driver_name = "qat_aes_cbc_hmac_sha512",
1314                 .cra_priority = 4001,
1315                 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1316                 .cra_blocksize = AES_BLOCK_SIZE,
1317                 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1318                 .cra_module = THIS_MODULE,
1319         },
1320         .init = qat_alg_aead_sha512_init,
1321         .exit = qat_alg_aead_exit,
1322         .setkey = qat_alg_aead_setkey,
1323         .decrypt = qat_alg_aead_dec,
1324         .encrypt = qat_alg_aead_enc,
1325         .ivsize = AES_BLOCK_SIZE,
1326         .maxauthsize = SHA512_DIGEST_SIZE,
1327 } };
1328
1329 static struct skcipher_alg qat_skciphers[] = { {
1330         .base.cra_name = "cbc(aes)",
1331         .base.cra_driver_name = "qat_aes_cbc",
1332         .base.cra_priority = 4001,
1333         .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1334         .base.cra_blocksize = AES_BLOCK_SIZE,
1335         .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1336         .base.cra_alignmask = 0,
1337         .base.cra_module = THIS_MODULE,
1338
1339         .init = qat_alg_skcipher_init_tfm,
1340         .exit = qat_alg_skcipher_exit_tfm,
1341         .setkey = qat_alg_skcipher_cbc_setkey,
1342         .decrypt = qat_alg_skcipher_blk_decrypt,
1343         .encrypt = qat_alg_skcipher_blk_encrypt,
1344         .min_keysize = AES_MIN_KEY_SIZE,
1345         .max_keysize = AES_MAX_KEY_SIZE,
1346         .ivsize = AES_BLOCK_SIZE,
1347 }, {
1348         .base.cra_name = "ctr(aes)",
1349         .base.cra_driver_name = "qat_aes_ctr",
1350         .base.cra_priority = 4001,
1351         .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1352         .base.cra_blocksize = 1,
1353         .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1354         .base.cra_alignmask = 0,
1355         .base.cra_module = THIS_MODULE,
1356
1357         .init = qat_alg_skcipher_init_tfm,
1358         .exit = qat_alg_skcipher_exit_tfm,
1359         .setkey = qat_alg_skcipher_ctr_setkey,
1360         .decrypt = qat_alg_skcipher_decrypt,
1361         .encrypt = qat_alg_skcipher_encrypt,
1362         .min_keysize = AES_MIN_KEY_SIZE,
1363         .max_keysize = AES_MAX_KEY_SIZE,
1364         .ivsize = AES_BLOCK_SIZE,
1365 }, {
1366         .base.cra_name = "xts(aes)",
1367         .base.cra_driver_name = "qat_aes_xts",
1368         .base.cra_priority = 4001,
1369         .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK |
1370                           CRYPTO_ALG_ALLOCATES_MEMORY,
1371         .base.cra_blocksize = AES_BLOCK_SIZE,
1372         .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1373         .base.cra_alignmask = 0,
1374         .base.cra_module = THIS_MODULE,
1375
1376         .init = qat_alg_skcipher_init_xts_tfm,
1377         .exit = qat_alg_skcipher_exit_xts_tfm,
1378         .setkey = qat_alg_skcipher_xts_setkey,
1379         .decrypt = qat_alg_skcipher_xts_decrypt,
1380         .encrypt = qat_alg_skcipher_xts_encrypt,
1381         .min_keysize = 2 * AES_MIN_KEY_SIZE,
1382         .max_keysize = 2 * AES_MAX_KEY_SIZE,
1383         .ivsize = AES_BLOCK_SIZE,
1384 } };
1385
1386 int qat_algs_register(void)
1387 {
1388         int ret = 0;
1389
1390         mutex_lock(&algs_lock);
1391         if (++active_devs != 1)
1392                 goto unlock;
1393
1394         ret = crypto_register_skciphers(qat_skciphers,
1395                                         ARRAY_SIZE(qat_skciphers));
1396         if (ret)
1397                 goto unlock;
1398
1399         ret = crypto_register_aeads(qat_aeads, ARRAY_SIZE(qat_aeads));
1400         if (ret)
1401                 goto unreg_algs;
1402
1403 unlock:
1404         mutex_unlock(&algs_lock);
1405         return ret;
1406
1407 unreg_algs:
1408         crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers));
1409         goto unlock;
1410 }
1411
1412 void qat_algs_unregister(void)
1413 {
1414         mutex_lock(&algs_lock);
1415         if (--active_devs != 0)
1416                 goto unlock;
1417
1418         crypto_unregister_aeads(qat_aeads, ARRAY_SIZE(qat_aeads));
1419         crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers));
1420
1421 unlock:
1422         mutex_unlock(&algs_lock);
1423 }