2 * This file is part of the Chelsio T6 Crypto driver for Linux.
4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 * Written and Maintained by:
42 #define pr_fmt(fmt) "chcr:" fmt
44 #include <linux/kernel.h>
45 #include <linux/module.h>
46 #include <linux/crypto.h>
47 #include <linux/cryptohash.h>
48 #include <linux/skbuff.h>
49 #include <linux/rtnetlink.h>
50 #include <linux/highmem.h>
51 #include <linux/scatterlist.h>
53 #include <crypto/aes.h>
54 #include <crypto/algapi.h>
55 #include <crypto/hash.h>
56 #include <crypto/gcm.h>
57 #include <crypto/sha.h>
58 #include <crypto/authenc.h>
59 #include <crypto/ctr.h>
60 #include <crypto/gf128mul.h>
61 #include <crypto/internal/aead.h>
62 #include <crypto/null.h>
63 #include <crypto/internal/skcipher.h>
64 #include <crypto/aead.h>
65 #include <crypto/scatterwalk.h>
66 #include <crypto/internal/hash.h>
70 #include "chcr_core.h"
71 #include "chcr_algo.h"
72 #include "chcr_crypto.h"
74 #define IV AES_BLOCK_SIZE
76 static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
78 return ctx->crypto_ctx->aeadctx;
81 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx)
83 return ctx->crypto_ctx->ablkctx;
86 static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx)
88 return ctx->crypto_ctx->hmacctx;
91 static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx)
93 return gctx->ctx->gcm;
96 static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx)
98 return gctx->ctx->authenc;
101 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
103 return ctx->dev->u_ctx;
106 static inline int is_ofld_imm(const struct sk_buff *skb)
108 return (skb->len <= SGE_MAX_WR_LEN);
112 * sgl_len - calculates the size of an SGL of the given capacity
113 * @n: the number of SGL entries
114 * Calculates the number of flits needed for a scatter/gather list that
115 * can hold the given number of entries.
117 static inline unsigned int sgl_len(unsigned int n)
120 return (3 * n) / 2 + (n & 1) + 2;
123 static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen,
129 unsigned int skip_len = 0;
132 if (sg_dma_len(sg) <= skip) {
133 skip -= sg_dma_len(sg);
142 while (sg && reqlen) {
143 less = min(reqlen, sg_dma_len(sg) - skip_len);
144 nents += DIV_ROUND_UP(less, entlen);
152 static inline void chcr_handle_ahash_resp(struct ahash_request *req,
153 unsigned char *input,
156 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
157 int digestsize, updated_digestsize;
158 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
159 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm));
163 reqctx = ahash_request_ctx(req);
164 digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));
165 if (reqctx->is_sg_map)
166 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
167 if (reqctx->dma_addr)
168 dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->dma_addr,
169 reqctx->dma_len, DMA_TO_DEVICE);
170 reqctx->dma_addr = 0;
171 updated_digestsize = digestsize;
172 if (digestsize == SHA224_DIGEST_SIZE)
173 updated_digestsize = SHA256_DIGEST_SIZE;
174 else if (digestsize == SHA384_DIGEST_SIZE)
175 updated_digestsize = SHA512_DIGEST_SIZE;
176 if (reqctx->result == 1) {
178 memcpy(req->result, input + sizeof(struct cpl_fw6_pld),
181 memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld),
185 req->base.complete(&req->base, err);
189 static inline void chcr_handle_aead_resp(struct aead_request *req,
190 unsigned char *input,
193 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
194 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
195 struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
198 chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op);
200 dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma,
201 reqctx->b0_len, DMA_BIDIRECTIONAL);
202 if (reqctx->verify == VERIFY_SW) {
203 chcr_verify_tag(req, input, &err);
204 reqctx->verify = VERIFY_HW;
206 req->base.complete(&req->base, err);
209 static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
211 u8 temp[SHA512_DIGEST_SIZE];
212 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
213 int authsize = crypto_aead_authsize(tfm);
214 struct cpl_fw6_pld *fw6_pld;
217 fw6_pld = (struct cpl_fw6_pld *)input;
218 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) ||
219 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) {
220 cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize);
223 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp,
224 authsize, req->assoclen +
225 req->cryptlen - authsize);
226 cmp = crypto_memneq(temp, (fw6_pld + 1), authsize);
235 * chcr_handle_resp - Unmap the DMA buffers associated with the request
236 * @req: crypto request
238 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
241 struct crypto_tfm *tfm = req->tfm;
242 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
243 struct adapter *adap = padap(ctx->dev);
245 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
246 case CRYPTO_ALG_TYPE_AEAD:
247 chcr_handle_aead_resp(aead_request_cast(req), input, err);
250 case CRYPTO_ALG_TYPE_ABLKCIPHER:
251 err = chcr_handle_cipher_resp(ablkcipher_request_cast(req),
255 case CRYPTO_ALG_TYPE_AHASH:
256 chcr_handle_ahash_resp(ahash_request_cast(req), input, err);
258 atomic_inc(&adap->chcr_stats.complete);
262 static void get_aes_decrypt_key(unsigned char *dec_key,
263 const unsigned char *key,
264 unsigned int keylength)
272 case AES_KEYLENGTH_128BIT:
273 nk = KEYLENGTH_4BYTES;
274 nr = NUMBER_OF_ROUNDS_10;
276 case AES_KEYLENGTH_192BIT:
277 nk = KEYLENGTH_6BYTES;
278 nr = NUMBER_OF_ROUNDS_12;
280 case AES_KEYLENGTH_256BIT:
281 nk = KEYLENGTH_8BYTES;
282 nr = NUMBER_OF_ROUNDS_14;
287 for (i = 0; i < nk; i++)
288 w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]);
291 temp = w_ring[nk - 1];
292 while (i + nk < (nr + 1) * 4) {
295 temp = (temp << 8) | (temp >> 24);
296 temp = aes_ks_subword(temp);
297 temp ^= round_constant[i / nk];
298 } else if (nk == 8 && (i % 4 == 0)) {
299 temp = aes_ks_subword(temp);
301 w_ring[i % nk] ^= temp;
302 temp = w_ring[i % nk];
306 for (k = 0, j = i % nk; k < nk; k++) {
307 *((u32 *)dec_key + k) = htonl(w_ring[j]);
314 static struct crypto_shash *chcr_alloc_shash(unsigned int ds)
316 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
319 case SHA1_DIGEST_SIZE:
320 base_hash = crypto_alloc_shash("sha1", 0, 0);
322 case SHA224_DIGEST_SIZE:
323 base_hash = crypto_alloc_shash("sha224", 0, 0);
325 case SHA256_DIGEST_SIZE:
326 base_hash = crypto_alloc_shash("sha256", 0, 0);
328 case SHA384_DIGEST_SIZE:
329 base_hash = crypto_alloc_shash("sha384", 0, 0);
331 case SHA512_DIGEST_SIZE:
332 base_hash = crypto_alloc_shash("sha512", 0, 0);
339 static int chcr_compute_partial_hash(struct shash_desc *desc,
340 char *iopad, char *result_hash,
343 struct sha1_state sha1_st;
344 struct sha256_state sha256_st;
345 struct sha512_state sha512_st;
348 if (digest_size == SHA1_DIGEST_SIZE) {
349 error = crypto_shash_init(desc) ?:
350 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?:
351 crypto_shash_export(desc, (void *)&sha1_st);
352 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE);
353 } else if (digest_size == SHA224_DIGEST_SIZE) {
354 error = crypto_shash_init(desc) ?:
355 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
356 crypto_shash_export(desc, (void *)&sha256_st);
357 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
359 } else if (digest_size == SHA256_DIGEST_SIZE) {
360 error = crypto_shash_init(desc) ?:
361 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
362 crypto_shash_export(desc, (void *)&sha256_st);
363 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
365 } else if (digest_size == SHA384_DIGEST_SIZE) {
366 error = crypto_shash_init(desc) ?:
367 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
368 crypto_shash_export(desc, (void *)&sha512_st);
369 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
371 } else if (digest_size == SHA512_DIGEST_SIZE) {
372 error = crypto_shash_init(desc) ?:
373 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
374 crypto_shash_export(desc, (void *)&sha512_st);
375 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
378 pr_err("Unknown digest size %d\n", digest_size);
383 static void chcr_change_order(char *buf, int ds)
387 if (ds == SHA512_DIGEST_SIZE) {
388 for (i = 0; i < (ds / sizeof(u64)); i++)
389 *((__be64 *)buf + i) =
390 cpu_to_be64(*((u64 *)buf + i));
392 for (i = 0; i < (ds / sizeof(u32)); i++)
393 *((__be32 *)buf + i) =
394 cpu_to_be32(*((u32 *)buf + i));
398 static inline int is_hmac(struct crypto_tfm *tfm)
400 struct crypto_alg *alg = tfm->__crt_alg;
401 struct chcr_alg_template *chcr_crypto_alg =
402 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template,
404 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC)
409 static inline void dsgl_walk_init(struct dsgl_walk *walk,
410 struct cpl_rx_phys_dsgl *dsgl)
414 walk->to = (struct phys_sge_pairs *)(dsgl + 1);
417 static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid)
419 struct cpl_rx_phys_dsgl *phys_cpl;
421 phys_cpl = walk->dsgl;
423 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
424 | CPL_RX_PHYS_DSGL_ISRDMA_V(0));
425 phys_cpl->pcirlxorder_to_noofsgentr =
426 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
427 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
428 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
429 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
430 CPL_RX_PHYS_DSGL_DCAID_V(0) |
431 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents));
432 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
433 phys_cpl->rss_hdr_int.qid = htons(qid);
434 phys_cpl->rss_hdr_int.hash_val = 0;
437 static inline void dsgl_walk_add_page(struct dsgl_walk *walk,
446 walk->to->len[j % 8] = htons(size);
447 walk->to->addr[j % 8] = cpu_to_be64(*addr);
454 static void dsgl_walk_add_sg(struct dsgl_walk *walk,
455 struct scatterlist *sg,
460 unsigned int left_size = slen, len = 0;
461 unsigned int j = walk->nents;
467 if (sg_dma_len(sg) <= skip) {
468 skip -= sg_dma_len(sg);
477 while (left_size && sg) {
478 len = min_t(u32, left_size, sg_dma_len(sg) - skip_len);
481 ent_len = min_t(u32, len, CHCR_DST_SG_SIZE);
482 walk->to->len[j % 8] = htons(ent_len);
483 walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) +
492 walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) -
493 skip_len) + skip_len;
494 left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len);
501 static inline void ulptx_walk_init(struct ulptx_walk *walk,
502 struct ulptx_sgl *ulp)
507 walk->pair = ulp->sge;
508 walk->last_sg = NULL;
509 walk->last_sg_len = 0;
512 static inline void ulptx_walk_end(struct ulptx_walk *walk)
514 walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
515 ULPTX_NSGE_V(walk->nents));
519 static inline void ulptx_walk_add_page(struct ulptx_walk *walk,
526 if (walk->nents == 0) {
527 walk->sgl->len0 = cpu_to_be32(size);
528 walk->sgl->addr0 = cpu_to_be64(*addr);
530 walk->pair->addr[walk->pair_idx] = cpu_to_be64(*addr);
531 walk->pair->len[walk->pair_idx] = cpu_to_be32(size);
532 walk->pair_idx = !walk->pair_idx;
539 static void ulptx_walk_add_sg(struct ulptx_walk *walk,
540 struct scatterlist *sg,
552 if (sg_dma_len(sg) <= skip) {
553 skip -= sg_dma_len(sg);
561 if (walk->nents == 0) {
562 small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len);
563 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
564 walk->sgl->len0 = cpu_to_be32(sgmin);
565 walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len);
569 walk->last_sg_len = sgmin + skip_len;
571 if (sg_dma_len(sg) == skip_len) {
578 small = min(sg_dma_len(sg) - skip_len, len);
579 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
580 walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin);
581 walk->pair->addr[walk->pair_idx] =
582 cpu_to_be64(sg_dma_address(sg) + skip_len);
583 walk->pair_idx = !walk->pair_idx;
590 walk->last_sg_len = skip_len;
591 if (sg_dma_len(sg) == skip_len) {
598 static inline int get_aead_subtype(struct crypto_aead *aead)
600 struct aead_alg *alg = crypto_aead_alg(aead);
601 struct chcr_alg_template *chcr_crypto_alg =
602 container_of(alg, struct chcr_alg_template, alg.aead);
603 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
606 static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm)
608 struct crypto_alg *alg = tfm->__crt_alg;
609 struct chcr_alg_template *chcr_crypto_alg =
610 container_of(alg, struct chcr_alg_template, alg.crypto);
612 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
615 static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
617 struct adapter *adap = netdev2adap(dev);
618 struct sge_uld_txq_info *txq_info =
619 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
620 struct sge_uld_txq *txq;
624 txq = &txq_info->uldtxq[idx];
625 spin_lock(&txq->sendq.lock);
628 spin_unlock(&txq->sendq.lock);
633 static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
634 struct _key_ctx *key_ctx)
636 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
637 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len);
640 ablkctx->key + (ablkctx->enckey_len >> 1),
641 ablkctx->enckey_len >> 1);
642 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1),
643 ablkctx->rrkey, ablkctx->enckey_len >> 1);
647 static int chcr_sg_ent_in_wr(struct scatterlist *src,
648 struct scatterlist *dst,
651 unsigned int srcskip,
652 unsigned int dstskip)
654 int srclen = 0, dstlen = 0;
655 int srcsg = minsg, dstsg = minsg;
656 int offset = 0, less;
658 if (sg_dma_len(src) == srcskip) {
663 if (sg_dma_len(dst) == dstskip) {
669 space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) {
670 srclen += (sg_dma_len(src) - srcskip);
673 while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) &&
674 space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) {
675 if (srclen <= dstlen)
677 less = min_t(unsigned int, sg_dma_len(dst) - offset -
678 dstskip, CHCR_DST_SG_SIZE);
681 if (offset == sg_dma_len(dst)) {
691 return min(srclen, dstlen);
694 static int chcr_cipher_fallback(struct crypto_skcipher *cipher,
696 struct scatterlist *src,
697 struct scatterlist *dst,
700 unsigned short op_type)
704 SKCIPHER_REQUEST_ON_STACK(subreq, cipher);
705 skcipher_request_set_tfm(subreq, cipher);
706 skcipher_request_set_callback(subreq, flags, NULL, NULL);
707 skcipher_request_set_crypt(subreq, src, dst,
710 err = op_type ? crypto_skcipher_decrypt(subreq) :
711 crypto_skcipher_encrypt(subreq);
712 skcipher_request_zero(subreq);
717 static inline void create_wreq(struct chcr_context *ctx,
718 struct chcr_wr *chcr_req,
719 struct crypto_async_request *req,
726 struct uld_ctx *u_ctx = ULD_CTX(ctx);
727 int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx];
730 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE;
731 chcr_req->wreq.pld_size_hash_size =
732 htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
733 chcr_req->wreq.len16_pkd =
734 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16)));
735 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
736 chcr_req->wreq.rx_chid_to_rx_q_id =
737 FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid,
738 !!lcb, ctx->tx_qidx);
740 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id,
742 chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) -
743 ((sizeof(chcr_req->wreq)) >> 4)));
745 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm);
746 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
747 sizeof(chcr_req->key_ctx) + sc_len);
751 * create_cipher_wr - form the WR for cipher operations
753 * @ctx: crypto driver context of the request.
754 * @qid: ingress qid where response of this WR should be received.
755 * @op_type: encryption or decryption
757 static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
759 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req);
760 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
761 struct sk_buff *skb = NULL;
762 struct chcr_wr *chcr_req;
763 struct cpl_rx_phys_dsgl *phys_cpl;
764 struct ulptx_sgl *ulptx;
765 struct chcr_blkcipher_req_ctx *reqctx =
766 ablkcipher_request_ctx(wrparam->req);
767 unsigned int temp = 0, transhdr_len, dst_size;
770 unsigned int kctx_len;
771 gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
772 GFP_KERNEL : GFP_ATOMIC;
773 struct adapter *adap = padap(c_ctx(tfm)->dev);
775 nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE,
777 dst_size = get_space_for_phys_dsgl(nents + 1);
778 kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
779 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
780 nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes,
781 CHCR_SRC_SG_SIZE, reqctx->src_ofst);
782 temp = reqctx->imm ? (DIV_ROUND_UP((IV + wrparam->req->nbytes), 16)
783 * 16) : (sgl_len(nents + MIN_CIPHER_SG) * 8);
784 transhdr_len += temp;
785 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16;
786 skb = alloc_skb(SGE_MAX_WR_LEN, flags);
791 chcr_req = __skb_put_zero(skb, transhdr_len);
792 chcr_req->sec_cpl.op_ivinsrtofst =
793 FILL_SEC_CPL_OP_IVINSR(c_ctx(tfm)->dev->rx_channel_id, 2, 1);
795 chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes);
796 chcr_req->sec_cpl.aadstart_cipherstop_hi =
797 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0);
799 chcr_req->sec_cpl.cipherstop_lo_authinsert =
800 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
801 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0,
804 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
807 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
808 if ((reqctx->op == CHCR_DECRYPT_OP) &&
809 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
810 CRYPTO_ALG_SUB_TYPE_CTR)) &&
811 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
812 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) {
813 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx);
815 if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) ||
816 (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) {
817 memcpy(chcr_req->key_ctx.key, ablkctx->key,
818 ablkctx->enckey_len);
820 memcpy(chcr_req->key_ctx.key, ablkctx->key +
821 (ablkctx->enckey_len >> 1),
822 ablkctx->enckey_len >> 1);
823 memcpy(chcr_req->key_ctx.key +
824 (ablkctx->enckey_len >> 1),
826 ablkctx->enckey_len >> 1);
829 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
830 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
831 chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam);
832 chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid);
834 atomic_inc(&adap->chcr_stats.cipher_rqst);
835 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len
836 +(reqctx->imm ? (IV + wrparam->bytes) : 0);
837 create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0,
839 ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC);
843 return ERR_PTR(error);
846 static inline int chcr_keyctx_ck_size(unsigned int keylen)
850 if (keylen == AES_KEYSIZE_128)
851 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
852 else if (keylen == AES_KEYSIZE_192)
853 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
854 else if (keylen == AES_KEYSIZE_256)
855 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
861 static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher,
865 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
866 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
869 crypto_skcipher_clear_flags(ablkctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
870 crypto_skcipher_set_flags(ablkctx->sw_cipher, cipher->base.crt_flags &
871 CRYPTO_TFM_REQ_MASK);
872 err = crypto_skcipher_setkey(ablkctx->sw_cipher, key, keylen);
873 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
875 crypto_skcipher_get_flags(ablkctx->sw_cipher) &
880 static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher,
884 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
885 unsigned int ck_size, context_size;
889 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
893 ck_size = chcr_keyctx_ck_size(keylen);
894 alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0;
895 memcpy(ablkctx->key, key, keylen);
896 ablkctx->enckey_len = keylen;
897 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3);
898 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
899 keylen + alignment) >> 4;
901 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
903 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
906 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
907 ablkctx->enckey_len = 0;
912 static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher,
916 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
917 unsigned int ck_size, context_size;
921 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
924 ck_size = chcr_keyctx_ck_size(keylen);
925 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
926 memcpy(ablkctx->key, key, keylen);
927 ablkctx->enckey_len = keylen;
928 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
929 keylen + alignment) >> 4;
931 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
933 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
937 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
938 ablkctx->enckey_len = 0;
943 static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher,
947 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
948 unsigned int ck_size, context_size;
952 if (keylen < CTR_RFC3686_NONCE_SIZE)
954 memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE),
955 CTR_RFC3686_NONCE_SIZE);
957 keylen -= CTR_RFC3686_NONCE_SIZE;
958 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
962 ck_size = chcr_keyctx_ck_size(keylen);
963 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
964 memcpy(ablkctx->key, key, keylen);
965 ablkctx->enckey_len = keylen;
966 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
967 keylen + alignment) >> 4;
969 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
971 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
975 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
976 ablkctx->enckey_len = 0;
980 static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add)
982 unsigned int size = AES_BLOCK_SIZE;
983 __be32 *b = (__be32 *)(dstiv + size);
986 memcpy(dstiv, srciv, AES_BLOCK_SIZE);
987 for (; size >= 4; size -= 4) {
988 prev = be32_to_cpu(*--b);
998 static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes)
1000 __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE);
1002 u32 temp = be32_to_cpu(*--b);
1005 c = (u64)temp + 1; // No of block can processed withou overflow
1006 if ((bytes / AES_BLOCK_SIZE) > c)
1007 bytes = c * AES_BLOCK_SIZE;
1011 static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv)
1013 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1014 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
1015 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1016 struct crypto_cipher *cipher;
1019 unsigned int keylen;
1020 int round = reqctx->last_req_len / AES_BLOCK_SIZE;
1021 int round8 = round / 8;
1023 cipher = ablkctx->aes_generic;
1024 memcpy(iv, reqctx->iv, AES_BLOCK_SIZE);
1026 keylen = ablkctx->enckey_len / 2;
1027 key = ablkctx->key + keylen;
1028 ret = crypto_cipher_setkey(cipher, key, keylen);
1031 /*H/W sends the encrypted IV in dsgl when AADIVDROP bit is 0*/
1032 for (i = 0; i < round8; i++)
1033 gf128mul_x8_ble((le128 *)iv, (le128 *)iv);
1035 for (i = 0; i < (round % 8); i++)
1036 gf128mul_x_ble((le128 *)iv, (le128 *)iv);
1038 crypto_cipher_decrypt_one(cipher, iv, iv);
1043 static int chcr_update_cipher_iv(struct ablkcipher_request *req,
1044 struct cpl_fw6_pld *fw6_pld, u8 *iv)
1046 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1047 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1048 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm));
1051 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
1052 ctr_add_iv(iv, req->info, (reqctx->processed /
1054 else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686)
1055 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1056 CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed /
1057 AES_BLOCK_SIZE) + 1);
1058 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
1059 ret = chcr_update_tweak(req, iv);
1060 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
1062 sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv,
1064 reqctx->processed - AES_BLOCK_SIZE);
1066 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
1073 /* We need separate function for final iv because in rfc3686 Initial counter
1074 * starts from 1 and buffer size of iv is 8 byte only which remains constant
1075 * for subsequent update requests
1078 static int chcr_final_cipher_iv(struct ablkcipher_request *req,
1079 struct cpl_fw6_pld *fw6_pld, u8 *iv)
1081 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1082 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1083 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm));
1086 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
1087 ctr_add_iv(iv, req->info, (reqctx->processed /
1089 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
1090 ret = chcr_update_tweak(req, iv);
1091 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
1093 sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv,
1095 reqctx->processed - AES_BLOCK_SIZE);
1097 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
1105 static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
1106 unsigned char *input, int err)
1108 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1109 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1110 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
1111 struct sk_buff *skb;
1112 struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input;
1113 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1114 struct cipher_wr_param wrparam;
1119 if (req->nbytes == reqctx->processed) {
1120 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1122 err = chcr_final_cipher_iv(req, fw6_pld, req->info);
1126 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1127 c_ctx(tfm)->tx_qidx))) {
1128 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1135 bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 1,
1136 SPACE_LEFT(ablkctx->enckey_len),
1137 reqctx->src_ofst, reqctx->dst_ofst);
1138 if ((bytes + reqctx->processed) >= req->nbytes)
1139 bytes = req->nbytes - reqctx->processed;
1141 bytes = ROUND_16(bytes);
1143 /*CTR mode counter overfloa*/
1144 bytes = req->nbytes - reqctx->processed;
1146 dma_sync_single_for_cpu(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1147 reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
1148 err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv);
1149 dma_sync_single_for_device(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1150 reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
1154 if (unlikely(bytes == 0)) {
1155 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1157 err = chcr_cipher_fallback(ablkctx->sw_cipher,
1167 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1168 CRYPTO_ALG_SUB_TYPE_CTR)
1169 bytes = adjust_ctr_overflow(reqctx->iv, bytes);
1170 wrparam.qid = u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx];
1172 wrparam.bytes = bytes;
1173 skb = create_cipher_wr(&wrparam);
1175 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
1179 skb->dev = u_ctx->lldi.ports[0];
1180 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx);
1182 reqctx->last_req_len = bytes;
1183 reqctx->processed += bytes;
1186 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1188 req->base.complete(&req->base, err);
1192 static int process_cipher(struct ablkcipher_request *req,
1194 struct sk_buff **skb,
1195 unsigned short op_type)
1197 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1198 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
1199 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1200 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
1201 struct cipher_wr_param wrparam;
1202 int bytes, err = -EINVAL;
1204 reqctx->processed = 0;
1207 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
1208 (req->nbytes == 0) ||
1209 (req->nbytes % crypto_ablkcipher_blocksize(tfm))) {
1210 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
1211 ablkctx->enckey_len, req->nbytes, ivsize);
1214 chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1215 if (req->nbytes < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) +
1217 sizeof(struct cpl_rx_phys_dsgl) +
1220 /* Can be sent as Imm*/
1221 unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len;
1223 dnents = sg_nents_xlen(req->dst, req->nbytes,
1224 CHCR_DST_SG_SIZE, 0);
1226 phys_dsgl = get_space_for_phys_dsgl(dnents);
1227 kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
1228 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
1229 reqctx->imm = (transhdr_len + IV + req->nbytes) <=
1231 bytes = IV + req->nbytes;
1238 bytes = chcr_sg_ent_in_wr(req->src, req->dst,
1240 SPACE_LEFT(ablkctx->enckey_len),
1242 if ((bytes + reqctx->processed) >= req->nbytes)
1243 bytes = req->nbytes - reqctx->processed;
1245 bytes = ROUND_16(bytes);
1247 bytes = req->nbytes;
1249 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1250 CRYPTO_ALG_SUB_TYPE_CTR) {
1251 bytes = adjust_ctr_overflow(req->info, bytes);
1253 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1254 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) {
1255 memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE);
1256 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->info,
1257 CTR_RFC3686_IV_SIZE);
1259 /* initialize counter portion of counter block */
1260 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1261 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1265 memcpy(reqctx->iv, req->info, IV);
1267 if (unlikely(bytes == 0)) {
1268 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1270 err = chcr_cipher_fallback(ablkctx->sw_cipher,
1279 reqctx->op = op_type;
1280 reqctx->srcsg = req->src;
1281 reqctx->dstsg = req->dst;
1282 reqctx->src_ofst = 0;
1283 reqctx->dst_ofst = 0;
1286 wrparam.bytes = bytes;
1287 *skb = create_cipher_wr(&wrparam);
1289 err = PTR_ERR(*skb);
1292 reqctx->processed = bytes;
1293 reqctx->last_req_len = bytes;
1297 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1302 static int chcr_aes_encrypt(struct ablkcipher_request *req)
1304 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1305 struct sk_buff *skb = NULL;
1307 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1309 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1310 c_ctx(tfm)->tx_qidx))) {
1311 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1315 err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx],
1316 &skb, CHCR_ENCRYPT_OP);
1319 skb->dev = u_ctx->lldi.ports[0];
1320 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx);
1322 return -EINPROGRESS;
1325 static int chcr_aes_decrypt(struct ablkcipher_request *req)
1327 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1328 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1329 struct sk_buff *skb = NULL;
1332 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1333 c_ctx(tfm)->tx_qidx))) {
1334 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1338 err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx],
1339 &skb, CHCR_DECRYPT_OP);
1342 skb->dev = u_ctx->lldi.ports[0];
1343 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx);
1345 return -EINPROGRESS;
1348 static int chcr_device_init(struct chcr_context *ctx)
1350 struct uld_ctx *u_ctx = NULL;
1351 struct adapter *adap;
1353 int txq_perchan, txq_idx, ntxq;
1354 int err = 0, rxq_perchan, rxq_idx;
1356 id = smp_processor_id();
1358 u_ctx = assign_chcr_device();
1360 pr_err("chcr device assignment fails\n");
1363 ctx->dev = u_ctx->dev;
1364 adap = padap(ctx->dev);
1365 ntxq = min_not_zero((unsigned int)u_ctx->lldi.nrxq,
1366 adap->vres.ncrypto_fc);
1367 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
1368 txq_perchan = ntxq / u_ctx->lldi.nchan;
1369 rxq_idx = ctx->dev->tx_channel_id * rxq_perchan;
1370 rxq_idx += id % rxq_perchan;
1371 txq_idx = ctx->dev->tx_channel_id * txq_perchan;
1372 txq_idx += id % txq_perchan;
1373 spin_lock(&ctx->dev->lock_chcr_dev);
1374 ctx->rx_qidx = rxq_idx;
1375 ctx->tx_qidx = txq_idx;
1376 ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id;
1377 ctx->dev->rx_channel_id = 0;
1378 spin_unlock(&ctx->dev->lock_chcr_dev);
1384 static int chcr_cra_init(struct crypto_tfm *tfm)
1386 struct crypto_alg *alg = tfm->__crt_alg;
1387 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1388 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1390 ablkctx->sw_cipher = crypto_alloc_skcipher(alg->cra_name, 0,
1391 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
1392 if (IS_ERR(ablkctx->sw_cipher)) {
1393 pr_err("failed to allocate fallback for %s\n", alg->cra_name);
1394 return PTR_ERR(ablkctx->sw_cipher);
1397 if (get_cryptoalg_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_XTS) {
1398 /* To update tweak*/
1399 ablkctx->aes_generic = crypto_alloc_cipher("aes-generic", 0, 0);
1400 if (IS_ERR(ablkctx->aes_generic)) {
1401 pr_err("failed to allocate aes cipher for tweak\n");
1402 return PTR_ERR(ablkctx->aes_generic);
1405 ablkctx->aes_generic = NULL;
1407 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx);
1408 return chcr_device_init(crypto_tfm_ctx(tfm));
1411 static int chcr_rfc3686_init(struct crypto_tfm *tfm)
1413 struct crypto_alg *alg = tfm->__crt_alg;
1414 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1415 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1417 /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes))
1418 * cannot be used as fallback in chcr_handle_cipher_response
1420 ablkctx->sw_cipher = crypto_alloc_skcipher("ctr(aes)", 0,
1421 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
1422 if (IS_ERR(ablkctx->sw_cipher)) {
1423 pr_err("failed to allocate fallback for %s\n", alg->cra_name);
1424 return PTR_ERR(ablkctx->sw_cipher);
1426 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx);
1427 return chcr_device_init(crypto_tfm_ctx(tfm));
1431 static void chcr_cra_exit(struct crypto_tfm *tfm)
1433 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1434 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1436 crypto_free_skcipher(ablkctx->sw_cipher);
1437 if (ablkctx->aes_generic)
1438 crypto_free_cipher(ablkctx->aes_generic);
1441 static int get_alg_config(struct algo_param *params,
1442 unsigned int auth_size)
1444 switch (auth_size) {
1445 case SHA1_DIGEST_SIZE:
1446 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
1447 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1;
1448 params->result_size = SHA1_DIGEST_SIZE;
1450 case SHA224_DIGEST_SIZE:
1451 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1452 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224;
1453 params->result_size = SHA256_DIGEST_SIZE;
1455 case SHA256_DIGEST_SIZE:
1456 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1457 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256;
1458 params->result_size = SHA256_DIGEST_SIZE;
1460 case SHA384_DIGEST_SIZE:
1461 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1462 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384;
1463 params->result_size = SHA512_DIGEST_SIZE;
1465 case SHA512_DIGEST_SIZE:
1466 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1467 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512;
1468 params->result_size = SHA512_DIGEST_SIZE;
1471 pr_err("chcr : ERROR, unsupported digest size\n");
1477 static inline void chcr_free_shash(struct crypto_shash *base_hash)
1479 crypto_free_shash(base_hash);
1483 * create_hash_wr - Create hash work request
1484 * @req - Cipher req base
1486 static struct sk_buff *create_hash_wr(struct ahash_request *req,
1487 struct hash_wr_param *param)
1489 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1490 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1491 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm));
1492 struct sk_buff *skb = NULL;
1493 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm));
1494 struct chcr_wr *chcr_req;
1495 struct ulptx_sgl *ulptx;
1496 unsigned int nents = 0, transhdr_len, iopad_alignment = 0;
1497 unsigned int digestsize = crypto_ahash_digestsize(tfm);
1498 unsigned int kctx_len = 0, temp = 0;
1499 u8 hash_size_in_response = 0;
1500 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1502 struct adapter *adap = padap(h_ctx(tfm)->dev);
1505 iopad_alignment = KEYCTX_ALIGN_PAD(digestsize);
1506 kctx_len = param->alg_prm.result_size + iopad_alignment;
1507 if (param->opad_needed)
1508 kctx_len += param->alg_prm.result_size + iopad_alignment;
1510 if (req_ctx->result)
1511 hash_size_in_response = digestsize;
1513 hash_size_in_response = param->alg_prm.result_size;
1514 transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
1515 req_ctx->imm = (transhdr_len + param->bfr_len + param->sg_len) <=
1517 nents = sg_nents_xlen(req->src, param->sg_len, CHCR_SRC_SG_SIZE, 0);
1518 nents += param->bfr_len ? 1 : 0;
1519 transhdr_len += req_ctx->imm ? (DIV_ROUND_UP((param->bfr_len +
1520 param->sg_len), 16) * 16) :
1521 (sgl_len(nents) * 8);
1522 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16;
1524 skb = alloc_skb(SGE_MAX_WR_LEN, flags);
1526 return ERR_PTR(-ENOMEM);
1527 chcr_req = __skb_put_zero(skb, transhdr_len);
1529 chcr_req->sec_cpl.op_ivinsrtofst =
1530 FILL_SEC_CPL_OP_IVINSR(h_ctx(tfm)->dev->rx_channel_id, 2, 0);
1531 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len);
1533 chcr_req->sec_cpl.aadstart_cipherstop_hi =
1534 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
1535 chcr_req->sec_cpl.cipherstop_lo_authinsert =
1536 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
1537 chcr_req->sec_cpl.seqno_numivs =
1538 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode,
1539 param->opad_needed, 0);
1541 chcr_req->sec_cpl.ivgen_hdrlen =
1542 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0);
1544 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash,
1545 param->alg_prm.result_size);
1547 if (param->opad_needed)
1548 memcpy(chcr_req->key_ctx.key +
1549 ((param->alg_prm.result_size <= 32) ? 32 :
1550 CHCR_HASH_MAX_DIGEST_SIZE),
1551 hmacctx->opad, param->alg_prm.result_size);
1553 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
1554 param->alg_prm.mk_size, 0,
1557 sizeof(chcr_req->key_ctx)) >> 4));
1558 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1);
1559 ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + kctx_len +
1561 if (param->bfr_len != 0) {
1562 req_ctx->dma_addr = dma_map_single(&u_ctx->lldi.pdev->dev,
1563 req_ctx->reqbfr, param->bfr_len,
1565 if (dma_mapping_error(&u_ctx->lldi.pdev->dev,
1566 req_ctx->dma_addr)) {
1570 req_ctx->dma_len = param->bfr_len;
1572 req_ctx->dma_addr = 0;
1574 chcr_add_hash_src_ent(req, ulptx, param);
1575 /* Request upto max wr size */
1576 temp = kctx_len + DUMMY_BYTES + (req_ctx->imm ? (param->sg_len
1577 + param->bfr_len) : 0);
1578 atomic_inc(&adap->chcr_stats.digest_rqst);
1579 create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->imm,
1580 hash_size_in_response, transhdr_len,
1586 return ERR_PTR(error);
1589 static int chcr_ahash_update(struct ahash_request *req)
1591 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1592 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1593 struct uld_ctx *u_ctx = NULL;
1594 struct sk_buff *skb;
1595 u8 remainder = 0, bs;
1596 unsigned int nbytes = req->nbytes;
1597 struct hash_wr_param params;
1600 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1602 u_ctx = ULD_CTX(h_ctx(rtfm));
1603 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1604 h_ctx(rtfm)->tx_qidx))) {
1605 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1609 if (nbytes + req_ctx->reqlen >= bs) {
1610 remainder = (nbytes + req_ctx->reqlen) % bs;
1611 nbytes = nbytes + req_ctx->reqlen - remainder;
1613 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr
1614 + req_ctx->reqlen, nbytes, 0);
1615 req_ctx->reqlen += nbytes;
1618 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1621 params.opad_needed = 0;
1624 params.sg_len = nbytes - req_ctx->reqlen;
1625 params.bfr_len = req_ctx->reqlen;
1627 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
1628 req_ctx->result = 0;
1629 req_ctx->data_len += params.sg_len + params.bfr_len;
1630 skb = create_hash_wr(req, ¶ms);
1632 error = PTR_ERR(skb);
1638 swap(req_ctx->reqbfr, req_ctx->skbfr);
1639 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
1640 req_ctx->reqbfr, remainder, req->nbytes -
1643 req_ctx->reqlen = remainder;
1644 skb->dev = u_ctx->lldi.ports[0];
1645 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
1648 return -EINPROGRESS;
1650 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1654 static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1)
1656 memset(bfr_ptr, 0, bs);
1659 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3);
1661 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3);
1664 static int chcr_ahash_final(struct ahash_request *req)
1666 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1667 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1668 struct hash_wr_param params;
1669 struct sk_buff *skb;
1670 struct uld_ctx *u_ctx = NULL;
1671 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1673 u_ctx = ULD_CTX(h_ctx(rtfm));
1674 if (is_hmac(crypto_ahash_tfm(rtfm)))
1675 params.opad_needed = 1;
1677 params.opad_needed = 0;
1679 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
1680 req_ctx->result = 1;
1681 params.bfr_len = req_ctx->reqlen;
1682 req_ctx->data_len += params.bfr_len + params.sg_len;
1683 if (req_ctx->reqlen == 0) {
1684 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1688 params.bfr_len = bs;
1691 params.scmd1 = req_ctx->data_len;
1695 skb = create_hash_wr(req, ¶ms);
1697 return PTR_ERR(skb);
1699 skb->dev = u_ctx->lldi.ports[0];
1700 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
1702 return -EINPROGRESS;
1705 static int chcr_ahash_finup(struct ahash_request *req)
1707 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1708 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1709 struct uld_ctx *u_ctx = NULL;
1710 struct sk_buff *skb;
1711 struct hash_wr_param params;
1715 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1716 u_ctx = ULD_CTX(h_ctx(rtfm));
1718 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1719 h_ctx(rtfm)->tx_qidx))) {
1720 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1724 if (is_hmac(crypto_ahash_tfm(rtfm)))
1725 params.opad_needed = 1;
1727 params.opad_needed = 0;
1729 params.sg_len = req->nbytes;
1730 params.bfr_len = req_ctx->reqlen;
1731 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
1732 req_ctx->data_len += params.bfr_len + params.sg_len;
1733 req_ctx->result = 1;
1734 if ((req_ctx->reqlen + req->nbytes) == 0) {
1735 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1739 params.bfr_len = bs;
1741 params.scmd1 = req_ctx->data_len;
1745 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1749 skb = create_hash_wr(req, ¶ms);
1751 error = PTR_ERR(skb);
1754 skb->dev = u_ctx->lldi.ports[0];
1755 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
1758 return -EINPROGRESS;
1760 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1764 static int chcr_ahash_digest(struct ahash_request *req)
1766 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1767 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1768 struct uld_ctx *u_ctx = NULL;
1769 struct sk_buff *skb;
1770 struct hash_wr_param params;
1775 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1777 u_ctx = ULD_CTX(h_ctx(rtfm));
1778 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1779 h_ctx(rtfm)->tx_qidx))) {
1780 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1784 if (is_hmac(crypto_ahash_tfm(rtfm)))
1785 params.opad_needed = 1;
1787 params.opad_needed = 0;
1788 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1794 params.sg_len = req->nbytes;
1797 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm));
1798 req_ctx->result = 1;
1799 req_ctx->data_len += params.bfr_len + params.sg_len;
1801 if (req->nbytes == 0) {
1802 create_last_hash_block(req_ctx->reqbfr, bs, 0);
1804 params.bfr_len = bs;
1807 skb = create_hash_wr(req, ¶ms);
1809 error = PTR_ERR(skb);
1812 skb->dev = u_ctx->lldi.ports[0];
1813 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
1815 return -EINPROGRESS;
1817 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1821 static int chcr_ahash_export(struct ahash_request *areq, void *out)
1823 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1824 struct chcr_ahash_req_ctx *state = out;
1826 state->reqlen = req_ctx->reqlen;
1827 state->data_len = req_ctx->data_len;
1828 state->is_sg_map = 0;
1830 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen);
1831 memcpy(state->partial_hash, req_ctx->partial_hash,
1832 CHCR_HASH_MAX_DIGEST_SIZE);
1836 static int chcr_ahash_import(struct ahash_request *areq, const void *in)
1838 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1839 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in;
1841 req_ctx->reqlen = state->reqlen;
1842 req_ctx->data_len = state->data_len;
1843 req_ctx->reqbfr = req_ctx->bfr1;
1844 req_ctx->skbfr = req_ctx->bfr2;
1845 req_ctx->is_sg_map = 0;
1846 req_ctx->result = 0;
1847 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128);
1848 memcpy(req_ctx->partial_hash, state->partial_hash,
1849 CHCR_HASH_MAX_DIGEST_SIZE);
1853 static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
1854 unsigned int keylen)
1856 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm));
1857 unsigned int digestsize = crypto_ahash_digestsize(tfm);
1858 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1859 unsigned int i, err = 0, updated_digestsize;
1861 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash);
1863 /* use the key to calculate the ipad and opad. ipad will sent with the
1864 * first request's data. opad will be sent with the final hash result
1865 * ipad in hmacctx->ipad and opad in hmacctx->opad location
1867 shash->tfm = hmacctx->base_hash;
1868 shash->flags = crypto_shash_get_flags(hmacctx->base_hash);
1870 err = crypto_shash_digest(shash, key, keylen,
1874 keylen = digestsize;
1876 memcpy(hmacctx->ipad, key, keylen);
1878 memset(hmacctx->ipad + keylen, 0, bs - keylen);
1879 memcpy(hmacctx->opad, hmacctx->ipad, bs);
1881 for (i = 0; i < bs / sizeof(int); i++) {
1882 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA;
1883 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA;
1886 updated_digestsize = digestsize;
1887 if (digestsize == SHA224_DIGEST_SIZE)
1888 updated_digestsize = SHA256_DIGEST_SIZE;
1889 else if (digestsize == SHA384_DIGEST_SIZE)
1890 updated_digestsize = SHA512_DIGEST_SIZE;
1891 err = chcr_compute_partial_hash(shash, hmacctx->ipad,
1892 hmacctx->ipad, digestsize);
1895 chcr_change_order(hmacctx->ipad, updated_digestsize);
1897 err = chcr_compute_partial_hash(shash, hmacctx->opad,
1898 hmacctx->opad, digestsize);
1901 chcr_change_order(hmacctx->opad, updated_digestsize);
1906 static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
1907 unsigned int key_len)
1909 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
1910 unsigned short context_size = 0;
1913 err = chcr_cipher_fallback_setkey(cipher, key, key_len);
1917 memcpy(ablkctx->key, key, key_len);
1918 ablkctx->enckey_len = key_len;
1919 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2);
1920 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
1921 ablkctx->key_ctx_hdr =
1922 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
1923 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
1924 CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
1925 CHCR_KEYCTX_NO_KEY, 1,
1927 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
1930 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1931 ablkctx->enckey_len = 0;
1936 static int chcr_sha_init(struct ahash_request *areq)
1938 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1939 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1940 int digestsize = crypto_ahash_digestsize(tfm);
1942 req_ctx->data_len = 0;
1943 req_ctx->reqlen = 0;
1944 req_ctx->reqbfr = req_ctx->bfr1;
1945 req_ctx->skbfr = req_ctx->bfr2;
1946 req_ctx->skb = NULL;
1947 req_ctx->result = 0;
1948 req_ctx->is_sg_map = 0;
1949 copy_hash_init_values(req_ctx->partial_hash, digestsize);
1953 static int chcr_sha_cra_init(struct crypto_tfm *tfm)
1955 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1956 sizeof(struct chcr_ahash_req_ctx));
1957 return chcr_device_init(crypto_tfm_ctx(tfm));
1960 static int chcr_hmac_init(struct ahash_request *areq)
1962 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1963 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq);
1964 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm));
1965 unsigned int digestsize = crypto_ahash_digestsize(rtfm);
1966 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1968 chcr_sha_init(areq);
1969 req_ctx->data_len = bs;
1970 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1971 if (digestsize == SHA224_DIGEST_SIZE)
1972 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1973 SHA256_DIGEST_SIZE);
1974 else if (digestsize == SHA384_DIGEST_SIZE)
1975 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1976 SHA512_DIGEST_SIZE);
1978 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1984 static int chcr_hmac_cra_init(struct crypto_tfm *tfm)
1986 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1987 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1988 unsigned int digestsize =
1989 crypto_ahash_digestsize(__crypto_ahash_cast(tfm));
1991 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1992 sizeof(struct chcr_ahash_req_ctx));
1993 hmacctx->base_hash = chcr_alloc_shash(digestsize);
1994 if (IS_ERR(hmacctx->base_hash))
1995 return PTR_ERR(hmacctx->base_hash);
1996 return chcr_device_init(crypto_tfm_ctx(tfm));
1999 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
2001 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
2002 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
2004 if (hmacctx->base_hash) {
2005 chcr_free_shash(hmacctx->base_hash);
2006 hmacctx->base_hash = NULL;
2010 static int chcr_aead_common_init(struct aead_request *req,
2011 unsigned short op_type)
2013 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2014 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2015 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2016 int error = -EINVAL;
2017 unsigned int dst_size;
2018 unsigned int authsize = crypto_aead_authsize(tfm);
2020 dst_size = req->assoclen + req->cryptlen + (op_type ?
2021 -authsize : authsize);
2022 /* validate key size */
2023 if (aeadctx->enckey_len == 0)
2025 if (op_type && req->cryptlen < authsize)
2027 error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
2033 reqctx->aad_nents = sg_nents_xlen(req->src, req->assoclen,
2034 CHCR_SRC_SG_SIZE, 0);
2035 reqctx->src_nents = sg_nents_xlen(req->src, req->cryptlen,
2036 CHCR_SRC_SG_SIZE, req->assoclen);
2042 static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents,
2043 int aadmax, int wrlen,
2044 unsigned short op_type)
2046 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
2048 if (((req->cryptlen - (op_type ? authsize : 0)) == 0) ||
2049 dst_nents > MAX_DSGL_ENT ||
2050 (req->assoclen > aadmax) ||
2051 (wrlen > SGE_MAX_WR_LEN))
2056 static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type)
2058 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2059 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2060 struct aead_request *subreq = aead_request_ctx(req);
2062 aead_request_set_tfm(subreq, aeadctx->sw_cipher);
2063 aead_request_set_callback(subreq, req->base.flags,
2064 req->base.complete, req->base.data);
2065 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
2067 aead_request_set_ad(subreq, req->assoclen);
2068 return op_type ? crypto_aead_decrypt(subreq) :
2069 crypto_aead_encrypt(subreq);
2072 static struct sk_buff *create_authenc_wr(struct aead_request *req,
2075 unsigned short op_type)
2077 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2078 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2079 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
2080 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2081 struct sk_buff *skb = NULL;
2082 struct chcr_wr *chcr_req;
2083 struct cpl_rx_phys_dsgl *phys_cpl;
2084 struct ulptx_sgl *ulptx;
2085 unsigned int transhdr_len;
2086 unsigned int dst_size = 0, temp;
2087 unsigned int kctx_len = 0, dnents;
2088 unsigned int assoclen = req->assoclen;
2089 unsigned int authsize = crypto_aead_authsize(tfm);
2090 int error = -EINVAL;
2092 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2094 struct adapter *adap = padap(a_ctx(tfm)->dev);
2096 if (req->cryptlen == 0)
2100 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) {
2104 dst_size = assoclen + req->cryptlen + (op_type ? -authsize :
2106 error = chcr_aead_common_init(req, op_type);
2108 return ERR_PTR(error);
2110 dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
2111 dnents += sg_nents_xlen(req->dst, req->cryptlen +
2112 (op_type ? -authsize : authsize), CHCR_DST_SG_SIZE,
2114 dnents += MIN_AUTH_SG; // For IV
2119 dst_size = get_space_for_phys_dsgl(dnents);
2120 kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4)
2121 - sizeof(chcr_req->key_ctx);
2122 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2123 reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) <
2125 temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen), 16)
2126 * 16) : (sgl_len(reqctx->src_nents + reqctx->aad_nents
2128 transhdr_len += temp;
2129 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16;
2131 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
2132 transhdr_len, op_type)) {
2133 atomic_inc(&adap->chcr_stats.fallback);
2134 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
2136 return ERR_PTR(chcr_aead_fallback(req, op_type));
2138 skb = alloc_skb(SGE_MAX_WR_LEN, flags);
2144 chcr_req = __skb_put_zero(skb, transhdr_len);
2146 temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
2149 * Input order is AAD,IV and Payload. where IV should be included as
2150 * the part of authdata. All other fields should be filled according
2151 * to the hardware spec
2153 chcr_req->sec_cpl.op_ivinsrtofst =
2154 FILL_SEC_CPL_OP_IVINSR(a_ctx(tfm)->dev->rx_channel_id, 2,
2156 chcr_req->sec_cpl.pldlen = htonl(assoclen + IV + req->cryptlen);
2157 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2158 assoclen ? 1 : 0, assoclen,
2160 (temp & 0x1F0) >> 4);
2161 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(
2163 null ? 0 : assoclen + IV + 1,
2165 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
2166 (op_type == CHCR_ENCRYPT_OP) ? 1 : 0,
2167 CHCR_SCMD_CIPHER_MODE_AES_CBC,
2168 actx->auth_mode, aeadctx->hmac_ctrl,
2170 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2173 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2174 if (op_type == CHCR_ENCRYPT_OP)
2175 memcpy(chcr_req->key_ctx.key, aeadctx->key,
2176 aeadctx->enckey_len);
2178 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey,
2179 aeadctx->enckey_len);
2181 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) <<
2182 4), actx->h_iopad, kctx_len -
2183 (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4));
2184 memcpy(reqctx->iv, req->iv, IV);
2185 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2186 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
2187 chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid);
2188 chcr_add_aead_src_ent(req, ulptx, assoclen, op_type);
2189 atomic_inc(&adap->chcr_stats.cipher_rqst);
2190 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size +
2191 kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0);
2192 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
2193 transhdr_len, temp, 0);
2195 reqctx->op = op_type;
2199 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
2202 return ERR_PTR(error);
2205 static int chcr_aead_dma_map(struct device *dev,
2206 struct aead_request *req,
2207 unsigned short op_type)
2210 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2211 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2212 unsigned int authsize = crypto_aead_authsize(tfm);
2215 dst_size = req->assoclen + req->cryptlen + (op_type ?
2216 -authsize : authsize);
2217 if (!req->cryptlen || !dst_size)
2219 reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV,
2221 if (dma_mapping_error(dev, reqctx->iv_dma))
2224 if (req->src == req->dst) {
2225 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2230 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2234 error = dma_map_sg(dev, req->dst, sg_nents(req->dst),
2237 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2245 dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
2249 static void chcr_aead_dma_unmap(struct device *dev,
2250 struct aead_request *req,
2251 unsigned short op_type)
2253 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2254 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2255 unsigned int authsize = crypto_aead_authsize(tfm);
2258 dst_size = req->assoclen + req->cryptlen + (op_type ?
2259 -authsize : authsize);
2260 if (!req->cryptlen || !dst_size)
2263 dma_unmap_single(dev, reqctx->iv_dma, IV,
2265 if (req->src == req->dst) {
2266 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2269 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2271 dma_unmap_sg(dev, req->dst, sg_nents(req->dst),
2276 static inline void chcr_add_aead_src_ent(struct aead_request *req,
2277 struct ulptx_sgl *ulptx,
2278 unsigned int assoclen,
2279 unsigned short op_type)
2281 struct ulptx_walk ulp_walk;
2282 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2285 u8 *buf = (u8 *)ulptx;
2287 if (reqctx->b0_dma) {
2288 memcpy(buf, reqctx->scratch_pad, reqctx->b0_len);
2289 buf += reqctx->b0_len;
2291 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2294 memcpy(buf, reqctx->iv, IV);
2296 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2297 buf, req->cryptlen, req->assoclen);
2299 ulptx_walk_init(&ulp_walk, ulptx);
2301 ulptx_walk_add_page(&ulp_walk, reqctx->b0_len,
2303 ulptx_walk_add_sg(&ulp_walk, req->src, assoclen, 0);
2304 ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma);
2305 ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen,
2307 ulptx_walk_end(&ulp_walk);
2311 static inline void chcr_add_aead_dst_ent(struct aead_request *req,
2312 struct cpl_rx_phys_dsgl *phys_cpl,
2313 unsigned int assoclen,
2314 unsigned short op_type,
2317 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2318 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2319 struct dsgl_walk dsgl_walk;
2320 unsigned int authsize = crypto_aead_authsize(tfm);
2323 dsgl_walk_init(&dsgl_walk, phys_cpl);
2325 dsgl_walk_add_page(&dsgl_walk, reqctx->b0_len, &reqctx->b0_dma);
2326 dsgl_walk_add_sg(&dsgl_walk, req->dst, assoclen, 0);
2327 dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma);
2328 temp = req->cryptlen + (op_type ? -authsize : authsize);
2329 dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, req->assoclen);
2330 dsgl_walk_end(&dsgl_walk, qid);
2333 static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
2334 struct ulptx_sgl *ulptx,
2335 struct cipher_wr_param *wrparam)
2337 struct ulptx_walk ulp_walk;
2338 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
2341 u8 *buf = (u8 *)ulptx;
2343 memcpy(buf, reqctx->iv, IV);
2345 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2346 buf, wrparam->bytes, reqctx->processed);
2348 ulptx_walk_init(&ulp_walk, ulptx);
2349 ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma);
2350 ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes,
2352 reqctx->srcsg = ulp_walk.last_sg;
2353 reqctx->src_ofst = ulp_walk.last_sg_len;
2354 ulptx_walk_end(&ulp_walk);
2358 static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req,
2359 struct cpl_rx_phys_dsgl *phys_cpl,
2360 struct cipher_wr_param *wrparam,
2363 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
2364 struct dsgl_walk dsgl_walk;
2366 dsgl_walk_init(&dsgl_walk, phys_cpl);
2367 dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma);
2368 dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes,
2370 reqctx->dstsg = dsgl_walk.last_sg;
2371 reqctx->dst_ofst = dsgl_walk.last_sg_len;
2373 dsgl_walk_end(&dsgl_walk, qid);
2376 static inline void chcr_add_hash_src_ent(struct ahash_request *req,
2377 struct ulptx_sgl *ulptx,
2378 struct hash_wr_param *param)
2380 struct ulptx_walk ulp_walk;
2381 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
2384 u8 *buf = (u8 *)ulptx;
2386 if (param->bfr_len) {
2387 memcpy(buf, reqctx->reqbfr, param->bfr_len);
2388 buf += param->bfr_len;
2390 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2391 buf, param->sg_len, 0);
2393 ulptx_walk_init(&ulp_walk, ulptx);
2395 ulptx_walk_add_page(&ulp_walk, param->bfr_len,
2397 ulptx_walk_add_sg(&ulp_walk, req->src, param->sg_len,
2399 // reqctx->srcsg = ulp_walk.last_sg;
2400 // reqctx->src_ofst = ulp_walk.last_sg_len;
2401 ulptx_walk_end(&ulp_walk);
2406 static inline int chcr_hash_dma_map(struct device *dev,
2407 struct ahash_request *req)
2409 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
2414 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2418 req_ctx->is_sg_map = 1;
2422 static inline void chcr_hash_dma_unmap(struct device *dev,
2423 struct ahash_request *req)
2425 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
2430 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2432 req_ctx->is_sg_map = 0;
2437 static int chcr_cipher_dma_map(struct device *dev,
2438 struct ablkcipher_request *req)
2441 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
2443 reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV,
2445 if (dma_mapping_error(dev, reqctx->iv_dma))
2448 if (req->src == req->dst) {
2449 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2454 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2458 error = dma_map_sg(dev, req->dst, sg_nents(req->dst),
2461 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2469 dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
2472 static void chcr_cipher_dma_unmap(struct device *dev,
2473 struct ablkcipher_request *req)
2475 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
2477 dma_unmap_single(dev, reqctx->iv_dma, IV,
2479 if (req->src == req->dst) {
2480 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2483 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2485 dma_unmap_sg(dev, req->dst, sg_nents(req->dst),
2490 static int set_msg_len(u8 *block, unsigned int msglen, int csize)
2494 memset(block, 0, csize);
2499 else if (msglen > (unsigned int)(1 << (8 * csize)))
2502 data = cpu_to_be32(msglen);
2503 memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
2508 static void generate_b0(struct aead_request *req,
2509 struct chcr_aead_ctx *aeadctx,
2510 unsigned short op_type)
2512 unsigned int l, lp, m;
2514 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2515 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2516 u8 *b0 = reqctx->scratch_pad;
2518 m = crypto_aead_authsize(aead);
2520 memcpy(b0, reqctx->iv, 16);
2525 /* set m, bits 3-5 */
2526 *b0 |= (8 * ((m - 2) / 2));
2528 /* set adata, bit 6, if associated data is used */
2531 rc = set_msg_len(b0 + 16 - l,
2532 (op_type == CHCR_DECRYPT_OP) ?
2533 req->cryptlen - m : req->cryptlen, l);
2536 static inline int crypto_ccm_check_iv(const u8 *iv)
2538 /* 2 <= L <= 8, so 1 <= L' <= 7. */
2539 if (iv[0] < 1 || iv[0] > 7)
2545 static int ccm_format_packet(struct aead_request *req,
2546 struct chcr_aead_ctx *aeadctx,
2547 unsigned int sub_type,
2548 unsigned short op_type)
2550 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2553 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2555 memcpy(reqctx->iv + 1, &aeadctx->salt[0], 3);
2556 memcpy(reqctx->iv + 4, req->iv, 8);
2557 memset(reqctx->iv + 12, 0, 4);
2558 *((unsigned short *)(reqctx->scratch_pad + 16)) =
2559 htons(req->assoclen - 8);
2561 memcpy(reqctx->iv, req->iv, 16);
2562 *((unsigned short *)(reqctx->scratch_pad + 16)) =
2563 htons(req->assoclen);
2565 generate_b0(req, aeadctx, op_type);
2566 /* zero the ctr value */
2567 memset(reqctx->iv + 15 - reqctx->iv[0], 0, reqctx->iv[0] + 1);
2571 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
2572 unsigned int dst_size,
2573 struct aead_request *req,
2574 unsigned short op_type)
2576 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2577 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2578 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
2579 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
2580 unsigned int c_id = a_ctx(tfm)->dev->rx_channel_id;
2581 unsigned int ccm_xtra;
2582 unsigned char tag_offset = 0, auth_offset = 0;
2583 unsigned int assoclen;
2585 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2586 assoclen = req->assoclen - 8;
2588 assoclen = req->assoclen;
2589 ccm_xtra = CCM_B0_SIZE +
2590 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0);
2592 auth_offset = req->cryptlen ?
2593 (assoclen + IV + 1 + ccm_xtra) : 0;
2594 if (op_type == CHCR_DECRYPT_OP) {
2595 if (crypto_aead_authsize(tfm) != req->cryptlen)
2596 tag_offset = crypto_aead_authsize(tfm);
2602 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id,
2603 2, assoclen + 1 + ccm_xtra);
2605 htonl(assoclen + IV + req->cryptlen + ccm_xtra);
2606 /* For CCM there wil be b0 always. So AAD start will be 1 always */
2607 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2608 1, assoclen + ccm_xtra, assoclen
2609 + IV + 1 + ccm_xtra, 0);
2611 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0,
2612 auth_offset, tag_offset,
2613 (op_type == CHCR_ENCRYPT_OP) ? 0 :
2614 crypto_aead_authsize(tfm));
2615 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
2616 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1,
2617 cipher_mode, mac_mode,
2618 aeadctx->hmac_ctrl, IV >> 1);
2620 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0,
2624 int aead_ccm_validate_input(unsigned short op_type,
2625 struct aead_request *req,
2626 struct chcr_aead_ctx *aeadctx,
2627 unsigned int sub_type)
2629 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2630 if (crypto_ccm_check_iv(req->iv)) {
2631 pr_err("CCM: IV check fails\n");
2635 if (req->assoclen != 16 && req->assoclen != 20) {
2636 pr_err("RFC4309: Invalid AAD length %d\n",
2644 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
2647 unsigned short op_type)
2649 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2650 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2651 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2652 struct sk_buff *skb = NULL;
2653 struct chcr_wr *chcr_req;
2654 struct cpl_rx_phys_dsgl *phys_cpl;
2655 struct ulptx_sgl *ulptx;
2656 unsigned int transhdr_len;
2657 unsigned int dst_size = 0, kctx_len, dnents, temp;
2658 unsigned int sub_type, assoclen = req->assoclen;
2659 unsigned int authsize = crypto_aead_authsize(tfm);
2660 int error = -EINVAL;
2661 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2663 struct adapter *adap = padap(a_ctx(tfm)->dev);
2666 sub_type = get_aead_subtype(tfm);
2667 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2669 dst_size = assoclen + req->cryptlen + (op_type ? -authsize :
2671 error = chcr_aead_common_init(req, op_type);
2673 return ERR_PTR(error);
2676 reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0);
2677 error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type);
2681 dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
2682 dnents += sg_nents_xlen(req->dst, req->cryptlen
2683 + (op_type ? -authsize : authsize),
2684 CHCR_DST_SG_SIZE, req->assoclen);
2685 dnents += MIN_CCM_SG; // For IV and B0
2689 dst_size = get_space_for_phys_dsgl(dnents);
2690 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2;
2691 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2692 reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen +
2693 reqctx->b0_len) <= SGE_MAX_WR_LEN;
2694 temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen +
2695 reqctx->b0_len), 16) * 16) :
2696 (sgl_len(reqctx->src_nents + reqctx->aad_nents +
2698 transhdr_len += temp;
2699 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16;
2701 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE -
2702 reqctx->b0_len, transhdr_len, op_type)) {
2703 atomic_inc(&adap->chcr_stats.fallback);
2704 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
2706 return ERR_PTR(chcr_aead_fallback(req, op_type));
2708 skb = alloc_skb(SGE_MAX_WR_LEN, flags);
2715 chcr_req = (struct chcr_wr *) __skb_put_zero(skb, transhdr_len);
2717 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type);
2719 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2720 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
2721 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
2722 16), aeadctx->key, aeadctx->enckey_len);
2724 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2725 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
2726 error = ccm_format_packet(req, aeadctx, sub_type, op_type);
2730 reqctx->b0_dma = dma_map_single(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev,
2731 &reqctx->scratch_pad, reqctx->b0_len,
2733 if (dma_mapping_error(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev,
2739 chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid);
2740 chcr_add_aead_src_ent(req, ulptx, assoclen, op_type);
2742 atomic_inc(&adap->chcr_stats.aead_rqst);
2743 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size +
2744 kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen +
2745 reqctx->b0_len) : 0);
2746 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0,
2747 transhdr_len, temp, 0);
2749 reqctx->op = op_type;
2755 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type);
2756 return ERR_PTR(error);
2759 static struct sk_buff *create_gcm_wr(struct aead_request *req,
2762 unsigned short op_type)
2764 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2765 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2766 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2767 struct sk_buff *skb = NULL;
2768 struct chcr_wr *chcr_req;
2769 struct cpl_rx_phys_dsgl *phys_cpl;
2770 struct ulptx_sgl *ulptx;
2771 unsigned int transhdr_len, dnents = 0;
2772 unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen;
2773 unsigned int authsize = crypto_aead_authsize(tfm);
2774 int error = -EINVAL;
2775 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2777 struct adapter *adap = padap(a_ctx(tfm)->dev);
2779 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
2780 assoclen = req->assoclen - 8;
2783 dst_size = assoclen + req->cryptlen + (op_type ? -authsize : authsize);
2784 error = chcr_aead_common_init(req, op_type);
2786 return ERR_PTR(error);
2788 dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
2789 dnents += sg_nents_xlen(req->dst,
2790 req->cryptlen + (op_type ? -authsize : authsize),
2791 CHCR_DST_SG_SIZE, req->assoclen);
2792 dnents += MIN_GCM_SG; // For IV
2796 dst_size = get_space_for_phys_dsgl(dnents);
2797 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) +
2799 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2800 reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) <=
2802 temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV +
2803 req->cryptlen), 16) * 16) : (sgl_len(reqctx->src_nents +
2804 reqctx->aad_nents + MIN_GCM_SG) * 8);
2805 transhdr_len += temp;
2806 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16;
2807 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
2808 transhdr_len, op_type)) {
2809 atomic_inc(&adap->chcr_stats.fallback);
2810 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
2812 return ERR_PTR(chcr_aead_fallback(req, op_type));
2814 skb = alloc_skb(SGE_MAX_WR_LEN, flags);
2820 chcr_req = __skb_put_zero(skb, transhdr_len);
2822 //Offset of tag from end
2823 temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
2824 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
2825 a_ctx(tfm)->dev->rx_channel_id, 2,
2827 chcr_req->sec_cpl.pldlen =
2828 htonl(assoclen + IV + req->cryptlen);
2829 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2830 assoclen ? 1 : 0, assoclen,
2831 assoclen + IV + 1, 0);
2832 chcr_req->sec_cpl.cipherstop_lo_authinsert =
2833 FILL_SEC_CPL_AUTHINSERT(0, assoclen + IV + 1,
2835 chcr_req->sec_cpl.seqno_numivs =
2836 FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type ==
2837 CHCR_ENCRYPT_OP) ? 1 : 0,
2838 CHCR_SCMD_CIPHER_MODE_AES_GCM,
2839 CHCR_SCMD_AUTH_MODE_GHASH,
2840 aeadctx->hmac_ctrl, IV >> 1);
2841 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2843 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2844 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
2845 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
2846 16), GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE);
2848 /* prepare a 16 byte iv */
2849 /* S A L T | IV | 0x00000001 */
2850 if (get_aead_subtype(tfm) ==
2851 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
2852 memcpy(reqctx->iv, aeadctx->salt, 4);
2853 memcpy(reqctx->iv + 4, req->iv, GCM_RFC4106_IV_SIZE);
2855 memcpy(reqctx->iv, req->iv, GCM_AES_IV_SIZE);
2857 *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01);
2859 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2860 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
2862 chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid);
2863 chcr_add_aead_src_ent(req, ulptx, assoclen, op_type);
2864 atomic_inc(&adap->chcr_stats.aead_rqst);
2865 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size +
2866 kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0);
2867 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
2868 transhdr_len, temp, reqctx->verify);
2870 reqctx->op = op_type;
2874 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type);
2875 return ERR_PTR(error);
2880 static int chcr_aead_cra_init(struct crypto_aead *tfm)
2882 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2883 struct aead_alg *alg = crypto_aead_alg(tfm);
2885 aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0,
2886 CRYPTO_ALG_NEED_FALLBACK |
2888 if (IS_ERR(aeadctx->sw_cipher))
2889 return PTR_ERR(aeadctx->sw_cipher);
2890 crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx),
2891 sizeof(struct aead_request) +
2892 crypto_aead_reqsize(aeadctx->sw_cipher)));
2893 return chcr_device_init(a_ctx(tfm));
2896 static void chcr_aead_cra_exit(struct crypto_aead *tfm)
2898 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2900 crypto_free_aead(aeadctx->sw_cipher);
2903 static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm,
2904 unsigned int authsize)
2906 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2908 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP;
2909 aeadctx->mayverify = VERIFY_HW;
2910 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2912 static int chcr_authenc_setauthsize(struct crypto_aead *tfm,
2913 unsigned int authsize)
2915 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2916 u32 maxauth = crypto_aead_maxauthsize(tfm);
2918 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not
2919 * true for sha1. authsize == 12 condition should be before
2920 * authsize == (maxauth >> 1)
2922 if (authsize == ICV_4) {
2923 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2924 aeadctx->mayverify = VERIFY_HW;
2925 } else if (authsize == ICV_6) {
2926 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
2927 aeadctx->mayverify = VERIFY_HW;
2928 } else if (authsize == ICV_10) {
2929 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
2930 aeadctx->mayverify = VERIFY_HW;
2931 } else if (authsize == ICV_12) {
2932 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2933 aeadctx->mayverify = VERIFY_HW;
2934 } else if (authsize == ICV_14) {
2935 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2936 aeadctx->mayverify = VERIFY_HW;
2937 } else if (authsize == (maxauth >> 1)) {
2938 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2939 aeadctx->mayverify = VERIFY_HW;
2940 } else if (authsize == maxauth) {
2941 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2942 aeadctx->mayverify = VERIFY_HW;
2944 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2945 aeadctx->mayverify = VERIFY_SW;
2947 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2951 static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
2953 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2957 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2958 aeadctx->mayverify = VERIFY_HW;
2961 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2962 aeadctx->mayverify = VERIFY_HW;
2965 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2966 aeadctx->mayverify = VERIFY_HW;
2969 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2970 aeadctx->mayverify = VERIFY_HW;
2973 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2974 aeadctx->mayverify = VERIFY_HW;
2978 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2979 aeadctx->mayverify = VERIFY_SW;
2983 crypto_tfm_set_flags((struct crypto_tfm *) tfm,
2984 CRYPTO_TFM_RES_BAD_KEY_LEN);
2987 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2990 static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm,
2991 unsigned int authsize)
2993 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2997 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2998 aeadctx->mayverify = VERIFY_HW;
3001 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3002 aeadctx->mayverify = VERIFY_HW;
3005 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3006 aeadctx->mayverify = VERIFY_HW;
3009 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
3010 CRYPTO_TFM_RES_BAD_KEY_LEN);
3013 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3016 static int chcr_ccm_setauthsize(struct crypto_aead *tfm,
3017 unsigned int authsize)
3019 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3023 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3024 aeadctx->mayverify = VERIFY_HW;
3027 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
3028 aeadctx->mayverify = VERIFY_HW;
3031 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3032 aeadctx->mayverify = VERIFY_HW;
3035 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
3036 aeadctx->mayverify = VERIFY_HW;
3039 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3040 aeadctx->mayverify = VERIFY_HW;
3043 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3044 aeadctx->mayverify = VERIFY_HW;
3047 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3048 aeadctx->mayverify = VERIFY_HW;
3051 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
3052 CRYPTO_TFM_RES_BAD_KEY_LEN);
3055 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3058 static int chcr_ccm_common_setkey(struct crypto_aead *aead,
3060 unsigned int keylen)
3062 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3063 unsigned char ck_size, mk_size;
3064 int key_ctx_size = 0;
3066 key_ctx_size = sizeof(struct _key_ctx) +
3067 ((DIV_ROUND_UP(keylen, 16)) << 4) * 2;
3068 if (keylen == AES_KEYSIZE_128) {
3069 mk_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3070 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3071 } else if (keylen == AES_KEYSIZE_192) {
3072 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3073 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192;
3074 } else if (keylen == AES_KEYSIZE_256) {
3075 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3076 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
3078 crypto_tfm_set_flags((struct crypto_tfm *)aead,
3079 CRYPTO_TFM_RES_BAD_KEY_LEN);
3080 aeadctx->enckey_len = 0;
3083 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0,
3085 memcpy(aeadctx->key, key, keylen);
3086 aeadctx->enckey_len = keylen;
3091 static int chcr_aead_ccm_setkey(struct crypto_aead *aead,
3093 unsigned int keylen)
3095 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3098 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3099 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
3100 CRYPTO_TFM_REQ_MASK);
3101 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3102 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
3103 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
3104 CRYPTO_TFM_RES_MASK);
3107 return chcr_ccm_common_setkey(aead, key, keylen);
3110 static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
3111 unsigned int keylen)
3113 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3117 crypto_tfm_set_flags((struct crypto_tfm *)aead,
3118 CRYPTO_TFM_RES_BAD_KEY_LEN);
3119 aeadctx->enckey_len = 0;
3122 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3123 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
3124 CRYPTO_TFM_REQ_MASK);
3125 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3126 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
3127 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
3128 CRYPTO_TFM_RES_MASK);
3132 memcpy(aeadctx->salt, key + keylen, 3);
3133 return chcr_ccm_common_setkey(aead, key, keylen);
3136 static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
3137 unsigned int keylen)
3139 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3140 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
3141 struct crypto_cipher *cipher;
3142 unsigned int ck_size;
3143 int ret = 0, key_ctx_size = 0;
3145 aeadctx->enckey_len = 0;
3146 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3147 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead)
3148 & CRYPTO_TFM_REQ_MASK);
3149 ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3150 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
3151 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
3152 CRYPTO_TFM_RES_MASK);
3156 if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
3158 keylen -= 4; /* nonce/salt is present in the last 4 bytes */
3159 memcpy(aeadctx->salt, key + keylen, 4);
3161 if (keylen == AES_KEYSIZE_128) {
3162 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3163 } else if (keylen == AES_KEYSIZE_192) {
3164 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3165 } else if (keylen == AES_KEYSIZE_256) {
3166 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3168 crypto_tfm_set_flags((struct crypto_tfm *)aead,
3169 CRYPTO_TFM_RES_BAD_KEY_LEN);
3170 pr_err("GCM: Invalid key length %d\n", keylen);
3175 memcpy(aeadctx->key, key, keylen);
3176 aeadctx->enckey_len = keylen;
3177 key_ctx_size = sizeof(struct _key_ctx) +
3178 ((DIV_ROUND_UP(keylen, 16)) << 4) +
3180 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
3181 CHCR_KEYCTX_MAC_KEY_SIZE_128,
3184 /* Calculate the H = CIPH(K, 0 repeated 16 times).
3185 * It will go in key context
3187 cipher = crypto_alloc_cipher("aes-generic", 0, 0);
3188 if (IS_ERR(cipher)) {
3189 aeadctx->enckey_len = 0;
3194 ret = crypto_cipher_setkey(cipher, key, keylen);
3196 aeadctx->enckey_len = 0;
3199 memset(gctx->ghash_h, 0, AEAD_H_SIZE);
3200 crypto_cipher_encrypt_one(cipher, gctx->ghash_h, gctx->ghash_h);
3203 crypto_free_cipher(cipher);
3208 static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
3209 unsigned int keylen)
3211 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc));
3212 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3213 /* it contains auth and cipher key both*/
3214 struct crypto_authenc_keys keys;
3216 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize;
3217 int err = 0, i, key_ctx_len = 0;
3218 unsigned char ck_size = 0;
3219 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 };
3220 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
3221 struct algo_param param;
3225 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3226 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3227 & CRYPTO_TFM_REQ_MASK);
3228 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3229 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK);
3230 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher)
3231 & CRYPTO_TFM_RES_MASK);
3235 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
3236 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
3240 if (get_alg_config(¶m, max_authsize)) {
3241 pr_err("chcr : Unsupported digest size\n");
3244 if (keys.enckeylen == AES_KEYSIZE_128) {
3245 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3246 } else if (keys.enckeylen == AES_KEYSIZE_192) {
3247 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3248 } else if (keys.enckeylen == AES_KEYSIZE_256) {
3249 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3251 pr_err("chcr : Unsupported cipher key\n");
3255 /* Copy only encryption key. We use authkey to generate h(ipad) and
3256 * h(opad) so authkey is not needed again. authkeylen size have the
3257 * size of the hash digest size.
3259 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3260 aeadctx->enckey_len = keys.enckeylen;
3261 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3262 aeadctx->enckey_len << 3);
3264 base_hash = chcr_alloc_shash(max_authsize);
3265 if (IS_ERR(base_hash)) {
3266 pr_err("chcr : Base driver cannot be loaded\n");
3267 aeadctx->enckey_len = 0;
3271 SHASH_DESC_ON_STACK(shash, base_hash);
3272 shash->tfm = base_hash;
3273 shash->flags = crypto_shash_get_flags(base_hash);
3274 bs = crypto_shash_blocksize(base_hash);
3275 align = KEYCTX_ALIGN_PAD(max_authsize);
3276 o_ptr = actx->h_iopad + param.result_size + align;
3278 if (keys.authkeylen > bs) {
3279 err = crypto_shash_digest(shash, keys.authkey,
3283 pr_err("chcr : Base driver cannot be loaded\n");
3286 keys.authkeylen = max_authsize;
3288 memcpy(o_ptr, keys.authkey, keys.authkeylen);
3290 /* Compute the ipad-digest*/
3291 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
3292 memcpy(pad, o_ptr, keys.authkeylen);
3293 for (i = 0; i < bs >> 2; i++)
3294 *((unsigned int *)pad + i) ^= IPAD_DATA;
3296 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad,
3299 /* Compute the opad-digest */
3300 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
3301 memcpy(pad, o_ptr, keys.authkeylen);
3302 for (i = 0; i < bs >> 2; i++)
3303 *((unsigned int *)pad + i) ^= OPAD_DATA;
3305 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize))
3308 /* convert the ipad and opad digest to network order */
3309 chcr_change_order(actx->h_iopad, param.result_size);
3310 chcr_change_order(o_ptr, param.result_size);
3311 key_ctx_len = sizeof(struct _key_ctx) +
3312 ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4) +
3313 (param.result_size + align) * 2;
3314 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size,
3315 0, 1, key_ctx_len >> 4);
3316 actx->auth_mode = param.auth_mode;
3317 chcr_free_shash(base_hash);
3322 aeadctx->enckey_len = 0;
3323 if (!IS_ERR(base_hash))
3324 chcr_free_shash(base_hash);
3328 static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
3329 const u8 *key, unsigned int keylen)
3331 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc));
3332 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3333 struct crypto_authenc_keys keys;
3335 /* it contains auth and cipher key both*/
3336 int key_ctx_len = 0;
3337 unsigned char ck_size = 0;
3339 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3340 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3341 & CRYPTO_TFM_REQ_MASK);
3342 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3343 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK);
3344 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher)
3345 & CRYPTO_TFM_RES_MASK);
3349 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
3350 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
3353 if (keys.enckeylen == AES_KEYSIZE_128) {
3354 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3355 } else if (keys.enckeylen == AES_KEYSIZE_192) {
3356 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3357 } else if (keys.enckeylen == AES_KEYSIZE_256) {
3358 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3360 pr_err("chcr : Unsupported cipher key\n");
3363 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3364 aeadctx->enckey_len = keys.enckeylen;
3365 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3366 aeadctx->enckey_len << 3);
3367 key_ctx_len = sizeof(struct _key_ctx)
3368 + ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4);
3370 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0,
3371 0, key_ctx_len >> 4);
3372 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP;
3375 aeadctx->enckey_len = 0;
3378 static int chcr_aead_encrypt(struct aead_request *req)
3380 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3381 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3383 reqctx->verify = VERIFY_HW;
3385 switch (get_aead_subtype(tfm)) {
3386 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC:
3387 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL:
3388 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
3390 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3391 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3392 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
3393 create_aead_ccm_wr);
3395 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
3400 static int chcr_aead_decrypt(struct aead_request *req)
3402 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3403 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3404 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3407 if (aeadctx->mayverify == VERIFY_SW) {
3408 size = crypto_aead_maxauthsize(tfm);
3409 reqctx->verify = VERIFY_SW;
3412 reqctx->verify = VERIFY_HW;
3415 switch (get_aead_subtype(tfm)) {
3416 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC:
3417 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL:
3418 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
3420 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3421 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3422 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
3423 create_aead_ccm_wr);
3425 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
3430 static int chcr_aead_op(struct aead_request *req,
3431 unsigned short op_type,
3433 create_wr_t create_wr_fn)
3435 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3436 struct uld_ctx *u_ctx;
3437 struct sk_buff *skb;
3439 if (!a_ctx(tfm)->dev) {
3440 pr_err("chcr : %s : No crypto device.\n", __func__);
3443 u_ctx = ULD_CTX(a_ctx(tfm));
3444 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
3445 a_ctx(tfm)->tx_qidx)) {
3446 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3450 /* Form a WR from req */
3451 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size,
3454 if (IS_ERR(skb) || !skb)
3455 return PTR_ERR(skb);
3457 skb->dev = u_ctx->lldi.ports[0];
3458 set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx);
3460 return -EINPROGRESS;
3462 static struct chcr_alg_template driver_algs[] = {
3465 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC,
3468 .cra_name = "cbc(aes)",
3469 .cra_driver_name = "cbc-aes-chcr",
3470 .cra_blocksize = AES_BLOCK_SIZE,
3471 .cra_init = chcr_cra_init,
3472 .cra_exit = chcr_cra_exit,
3473 .cra_u.ablkcipher = {
3474 .min_keysize = AES_MIN_KEY_SIZE,
3475 .max_keysize = AES_MAX_KEY_SIZE,
3476 .ivsize = AES_BLOCK_SIZE,
3477 .setkey = chcr_aes_cbc_setkey,
3478 .encrypt = chcr_aes_encrypt,
3479 .decrypt = chcr_aes_decrypt,
3484 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS,
3487 .cra_name = "xts(aes)",
3488 .cra_driver_name = "xts-aes-chcr",
3489 .cra_blocksize = AES_BLOCK_SIZE,
3490 .cra_init = chcr_cra_init,
3492 .cra_u .ablkcipher = {
3493 .min_keysize = 2 * AES_MIN_KEY_SIZE,
3494 .max_keysize = 2 * AES_MAX_KEY_SIZE,
3495 .ivsize = AES_BLOCK_SIZE,
3496 .setkey = chcr_aes_xts_setkey,
3497 .encrypt = chcr_aes_encrypt,
3498 .decrypt = chcr_aes_decrypt,
3503 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR,
3506 .cra_name = "ctr(aes)",
3507 .cra_driver_name = "ctr-aes-chcr",
3509 .cra_init = chcr_cra_init,
3510 .cra_exit = chcr_cra_exit,
3511 .cra_u.ablkcipher = {
3512 .min_keysize = AES_MIN_KEY_SIZE,
3513 .max_keysize = AES_MAX_KEY_SIZE,
3514 .ivsize = AES_BLOCK_SIZE,
3515 .setkey = chcr_aes_ctr_setkey,
3516 .encrypt = chcr_aes_encrypt,
3517 .decrypt = chcr_aes_decrypt,
3522 .type = CRYPTO_ALG_TYPE_ABLKCIPHER |
3523 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686,
3526 .cra_name = "rfc3686(ctr(aes))",
3527 .cra_driver_name = "rfc3686-ctr-aes-chcr",
3529 .cra_init = chcr_rfc3686_init,
3530 .cra_exit = chcr_cra_exit,
3531 .cra_u.ablkcipher = {
3532 .min_keysize = AES_MIN_KEY_SIZE +
3533 CTR_RFC3686_NONCE_SIZE,
3534 .max_keysize = AES_MAX_KEY_SIZE +
3535 CTR_RFC3686_NONCE_SIZE,
3536 .ivsize = CTR_RFC3686_IV_SIZE,
3537 .setkey = chcr_aes_rfc3686_setkey,
3538 .encrypt = chcr_aes_encrypt,
3539 .decrypt = chcr_aes_decrypt,
3546 .type = CRYPTO_ALG_TYPE_AHASH,
3549 .halg.digestsize = SHA1_DIGEST_SIZE,
3552 .cra_driver_name = "sha1-chcr",
3553 .cra_blocksize = SHA1_BLOCK_SIZE,
3558 .type = CRYPTO_ALG_TYPE_AHASH,
3561 .halg.digestsize = SHA256_DIGEST_SIZE,
3563 .cra_name = "sha256",
3564 .cra_driver_name = "sha256-chcr",
3565 .cra_blocksize = SHA256_BLOCK_SIZE,
3570 .type = CRYPTO_ALG_TYPE_AHASH,
3573 .halg.digestsize = SHA224_DIGEST_SIZE,
3575 .cra_name = "sha224",
3576 .cra_driver_name = "sha224-chcr",
3577 .cra_blocksize = SHA224_BLOCK_SIZE,
3582 .type = CRYPTO_ALG_TYPE_AHASH,
3585 .halg.digestsize = SHA384_DIGEST_SIZE,
3587 .cra_name = "sha384",
3588 .cra_driver_name = "sha384-chcr",
3589 .cra_blocksize = SHA384_BLOCK_SIZE,
3594 .type = CRYPTO_ALG_TYPE_AHASH,
3597 .halg.digestsize = SHA512_DIGEST_SIZE,
3599 .cra_name = "sha512",
3600 .cra_driver_name = "sha512-chcr",
3601 .cra_blocksize = SHA512_BLOCK_SIZE,
3607 .type = CRYPTO_ALG_TYPE_HMAC,
3610 .halg.digestsize = SHA1_DIGEST_SIZE,
3612 .cra_name = "hmac(sha1)",
3613 .cra_driver_name = "hmac-sha1-chcr",
3614 .cra_blocksize = SHA1_BLOCK_SIZE,
3619 .type = CRYPTO_ALG_TYPE_HMAC,
3622 .halg.digestsize = SHA224_DIGEST_SIZE,
3624 .cra_name = "hmac(sha224)",
3625 .cra_driver_name = "hmac-sha224-chcr",
3626 .cra_blocksize = SHA224_BLOCK_SIZE,
3631 .type = CRYPTO_ALG_TYPE_HMAC,
3634 .halg.digestsize = SHA256_DIGEST_SIZE,
3636 .cra_name = "hmac(sha256)",
3637 .cra_driver_name = "hmac-sha256-chcr",
3638 .cra_blocksize = SHA256_BLOCK_SIZE,
3643 .type = CRYPTO_ALG_TYPE_HMAC,
3646 .halg.digestsize = SHA384_DIGEST_SIZE,
3648 .cra_name = "hmac(sha384)",
3649 .cra_driver_name = "hmac-sha384-chcr",
3650 .cra_blocksize = SHA384_BLOCK_SIZE,
3655 .type = CRYPTO_ALG_TYPE_HMAC,
3658 .halg.digestsize = SHA512_DIGEST_SIZE,
3660 .cra_name = "hmac(sha512)",
3661 .cra_driver_name = "hmac-sha512-chcr",
3662 .cra_blocksize = SHA512_BLOCK_SIZE,
3666 /* Add AEAD Algorithms */
3668 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM,
3672 .cra_name = "gcm(aes)",
3673 .cra_driver_name = "gcm-aes-chcr",
3675 .cra_priority = CHCR_AEAD_PRIORITY,
3676 .cra_ctxsize = sizeof(struct chcr_context) +
3677 sizeof(struct chcr_aead_ctx) +
3678 sizeof(struct chcr_gcm_ctx),
3680 .ivsize = GCM_AES_IV_SIZE,
3681 .maxauthsize = GHASH_DIGEST_SIZE,
3682 .setkey = chcr_gcm_setkey,
3683 .setauthsize = chcr_gcm_setauthsize,
3687 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106,
3691 .cra_name = "rfc4106(gcm(aes))",
3692 .cra_driver_name = "rfc4106-gcm-aes-chcr",
3694 .cra_priority = CHCR_AEAD_PRIORITY + 1,
3695 .cra_ctxsize = sizeof(struct chcr_context) +
3696 sizeof(struct chcr_aead_ctx) +
3697 sizeof(struct chcr_gcm_ctx),
3700 .ivsize = GCM_RFC4106_IV_SIZE,
3701 .maxauthsize = GHASH_DIGEST_SIZE,
3702 .setkey = chcr_gcm_setkey,
3703 .setauthsize = chcr_4106_4309_setauthsize,
3707 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM,
3711 .cra_name = "ccm(aes)",
3712 .cra_driver_name = "ccm-aes-chcr",
3714 .cra_priority = CHCR_AEAD_PRIORITY,
3715 .cra_ctxsize = sizeof(struct chcr_context) +
3716 sizeof(struct chcr_aead_ctx),
3719 .ivsize = AES_BLOCK_SIZE,
3720 .maxauthsize = GHASH_DIGEST_SIZE,
3721 .setkey = chcr_aead_ccm_setkey,
3722 .setauthsize = chcr_ccm_setauthsize,
3726 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309,
3730 .cra_name = "rfc4309(ccm(aes))",
3731 .cra_driver_name = "rfc4309-ccm-aes-chcr",
3733 .cra_priority = CHCR_AEAD_PRIORITY + 1,
3734 .cra_ctxsize = sizeof(struct chcr_context) +
3735 sizeof(struct chcr_aead_ctx),
3739 .maxauthsize = GHASH_DIGEST_SIZE,
3740 .setkey = chcr_aead_rfc4309_setkey,
3741 .setauthsize = chcr_4106_4309_setauthsize,
3745 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3749 .cra_name = "authenc(hmac(sha1),cbc(aes))",
3751 "authenc-hmac-sha1-cbc-aes-chcr",
3752 .cra_blocksize = AES_BLOCK_SIZE,
3753 .cra_priority = CHCR_AEAD_PRIORITY,
3754 .cra_ctxsize = sizeof(struct chcr_context) +
3755 sizeof(struct chcr_aead_ctx) +
3756 sizeof(struct chcr_authenc_ctx),
3759 .ivsize = AES_BLOCK_SIZE,
3760 .maxauthsize = SHA1_DIGEST_SIZE,
3761 .setkey = chcr_authenc_setkey,
3762 .setauthsize = chcr_authenc_setauthsize,
3766 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3771 .cra_name = "authenc(hmac(sha256),cbc(aes))",
3773 "authenc-hmac-sha256-cbc-aes-chcr",
3774 .cra_blocksize = AES_BLOCK_SIZE,
3775 .cra_priority = CHCR_AEAD_PRIORITY,
3776 .cra_ctxsize = sizeof(struct chcr_context) +
3777 sizeof(struct chcr_aead_ctx) +
3778 sizeof(struct chcr_authenc_ctx),
3781 .ivsize = AES_BLOCK_SIZE,
3782 .maxauthsize = SHA256_DIGEST_SIZE,
3783 .setkey = chcr_authenc_setkey,
3784 .setauthsize = chcr_authenc_setauthsize,
3788 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3792 .cra_name = "authenc(hmac(sha224),cbc(aes))",
3794 "authenc-hmac-sha224-cbc-aes-chcr",
3795 .cra_blocksize = AES_BLOCK_SIZE,
3796 .cra_priority = CHCR_AEAD_PRIORITY,
3797 .cra_ctxsize = sizeof(struct chcr_context) +
3798 sizeof(struct chcr_aead_ctx) +
3799 sizeof(struct chcr_authenc_ctx),
3801 .ivsize = AES_BLOCK_SIZE,
3802 .maxauthsize = SHA224_DIGEST_SIZE,
3803 .setkey = chcr_authenc_setkey,
3804 .setauthsize = chcr_authenc_setauthsize,
3808 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3812 .cra_name = "authenc(hmac(sha384),cbc(aes))",
3814 "authenc-hmac-sha384-cbc-aes-chcr",
3815 .cra_blocksize = AES_BLOCK_SIZE,
3816 .cra_priority = CHCR_AEAD_PRIORITY,
3817 .cra_ctxsize = sizeof(struct chcr_context) +
3818 sizeof(struct chcr_aead_ctx) +
3819 sizeof(struct chcr_authenc_ctx),
3822 .ivsize = AES_BLOCK_SIZE,
3823 .maxauthsize = SHA384_DIGEST_SIZE,
3824 .setkey = chcr_authenc_setkey,
3825 .setauthsize = chcr_authenc_setauthsize,
3829 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3833 .cra_name = "authenc(hmac(sha512),cbc(aes))",
3835 "authenc-hmac-sha512-cbc-aes-chcr",
3836 .cra_blocksize = AES_BLOCK_SIZE,
3837 .cra_priority = CHCR_AEAD_PRIORITY,
3838 .cra_ctxsize = sizeof(struct chcr_context) +
3839 sizeof(struct chcr_aead_ctx) +
3840 sizeof(struct chcr_authenc_ctx),
3843 .ivsize = AES_BLOCK_SIZE,
3844 .maxauthsize = SHA512_DIGEST_SIZE,
3845 .setkey = chcr_authenc_setkey,
3846 .setauthsize = chcr_authenc_setauthsize,
3850 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_NULL,
3854 .cra_name = "authenc(digest_null,cbc(aes))",
3856 "authenc-digest_null-cbc-aes-chcr",
3857 .cra_blocksize = AES_BLOCK_SIZE,
3858 .cra_priority = CHCR_AEAD_PRIORITY,
3859 .cra_ctxsize = sizeof(struct chcr_context) +
3860 sizeof(struct chcr_aead_ctx) +
3861 sizeof(struct chcr_authenc_ctx),
3864 .ivsize = AES_BLOCK_SIZE,
3866 .setkey = chcr_aead_digest_null_setkey,
3867 .setauthsize = chcr_authenc_null_setauthsize,
3873 * chcr_unregister_alg - Deregister crypto algorithms with
3876 static int chcr_unregister_alg(void)
3880 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3881 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
3882 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3883 if (driver_algs[i].is_registered)
3884 crypto_unregister_alg(
3885 &driver_algs[i].alg.crypto);
3887 case CRYPTO_ALG_TYPE_AEAD:
3888 if (driver_algs[i].is_registered)
3889 crypto_unregister_aead(
3890 &driver_algs[i].alg.aead);
3892 case CRYPTO_ALG_TYPE_AHASH:
3893 if (driver_algs[i].is_registered)
3894 crypto_unregister_ahash(
3895 &driver_algs[i].alg.hash);
3898 driver_algs[i].is_registered = 0;
3903 #define SZ_AHASH_CTX sizeof(struct chcr_context)
3904 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx))
3905 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx)
3906 #define AHASH_CRA_FLAGS (CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC)
3909 * chcr_register_alg - Register crypto algorithms with kernel framework.
3911 static int chcr_register_alg(void)
3913 struct crypto_alg ai;
3914 struct ahash_alg *a_hash;
3918 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3919 if (driver_algs[i].is_registered)
3921 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
3922 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3923 driver_algs[i].alg.crypto.cra_priority =
3925 driver_algs[i].alg.crypto.cra_module = THIS_MODULE;
3926 driver_algs[i].alg.crypto.cra_flags =
3927 CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC |
3928 CRYPTO_ALG_NEED_FALLBACK;
3929 driver_algs[i].alg.crypto.cra_ctxsize =
3930 sizeof(struct chcr_context) +
3931 sizeof(struct ablk_ctx);
3932 driver_algs[i].alg.crypto.cra_alignmask = 0;
3933 driver_algs[i].alg.crypto.cra_type =
3934 &crypto_ablkcipher_type;
3935 err = crypto_register_alg(&driver_algs[i].alg.crypto);
3936 name = driver_algs[i].alg.crypto.cra_driver_name;
3938 case CRYPTO_ALG_TYPE_AEAD:
3939 driver_algs[i].alg.aead.base.cra_flags =
3940 CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
3941 CRYPTO_ALG_NEED_FALLBACK;
3942 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt;
3943 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt;
3944 driver_algs[i].alg.aead.init = chcr_aead_cra_init;
3945 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit;
3946 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE;
3947 err = crypto_register_aead(&driver_algs[i].alg.aead);
3948 name = driver_algs[i].alg.aead.base.cra_driver_name;
3950 case CRYPTO_ALG_TYPE_AHASH:
3951 a_hash = &driver_algs[i].alg.hash;
3952 a_hash->update = chcr_ahash_update;
3953 a_hash->final = chcr_ahash_final;
3954 a_hash->finup = chcr_ahash_finup;
3955 a_hash->digest = chcr_ahash_digest;
3956 a_hash->export = chcr_ahash_export;
3957 a_hash->import = chcr_ahash_import;
3958 a_hash->halg.statesize = SZ_AHASH_REQ_CTX;
3959 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY;
3960 a_hash->halg.base.cra_module = THIS_MODULE;
3961 a_hash->halg.base.cra_flags = AHASH_CRA_FLAGS;
3962 a_hash->halg.base.cra_alignmask = 0;
3963 a_hash->halg.base.cra_exit = NULL;
3964 a_hash->halg.base.cra_type = &crypto_ahash_type;
3966 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) {
3967 a_hash->halg.base.cra_init = chcr_hmac_cra_init;
3968 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit;
3969 a_hash->init = chcr_hmac_init;
3970 a_hash->setkey = chcr_ahash_setkey;
3971 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX;
3973 a_hash->init = chcr_sha_init;
3974 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX;
3975 a_hash->halg.base.cra_init = chcr_sha_cra_init;
3977 err = crypto_register_ahash(&driver_algs[i].alg.hash);
3978 ai = driver_algs[i].alg.hash.halg.base;
3979 name = ai.cra_driver_name;
3983 pr_err("chcr : %s : Algorithm registration failed\n",
3987 driver_algs[i].is_registered = 1;
3993 chcr_unregister_alg();
3998 * start_crypto - Register the crypto algorithms.
3999 * This should called once when the first device comesup. After this
4000 * kernel will start calling driver APIs for crypto operations.
4002 int start_crypto(void)
4004 return chcr_register_alg();
4008 * stop_crypto - Deregister all the crypto algorithms with kernel.
4009 * This should be called once when the last device goes down. After this
4010 * kernel will not call the driver API for crypto operations.
4012 int stop_crypto(void)
4014 chcr_unregister_alg();