drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (C) 2020 Marvell. */
   3
   4 #include "otx2_cptvf.h"
   5 #include "otx2_cpt_common.h"
   6
   7 /* Default timeout when waiting for free pending entry in us */
   8 #define CPT_PENTRY_TIMEOUT      1000
   9 #define CPT_PENTRY_STEP         50
  10
  11 /* Default threshold for stopping and resuming sender requests */
  12 #define CPT_IQ_STOP_MARGIN      128
  13 #define CPT_IQ_RESUME_MARGIN    512
  14
  15 /* Default command timeout in seconds */
  16 #define CPT_COMMAND_TIMEOUT     4
  17 #define CPT_TIME_IN_RESET_COUNT 5
  18
  19 static void otx2_cpt_dump_sg_list(struct pci_dev *pdev,
  20                                   struct otx2_cpt_req_info *req)
  21 {
  22         int i;
  23
  24         pr_debug("Gather list size %d\n", req->in_cnt);
  25         for (i = 0; i < req->in_cnt; i++) {
  26                 pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%llx\n", i,
  27                          req->in[i].size, req->in[i].vptr,
  28                          req->in[i].dma_addr);
  29                 pr_debug("Buffer hexdump (%d bytes)\n",
  30                          req->in[i].size);
  31                 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,
  32                                      req->in[i].vptr, req->in[i].size, false);
  33         }
  34         pr_debug("Scatter list size %d\n", req->out_cnt);
  35         for (i = 0; i < req->out_cnt; i++) {
  36                 pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%llx\n", i,
  37                          req->out[i].size, req->out[i].vptr,
  38                          req->out[i].dma_addr);
  39                 pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size);
  40                 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,
  41                                      req->out[i].vptr, req->out[i].size, false);
  42         }
  43 }
  44
  45 static inline struct otx2_cpt_pending_entry *get_free_pending_entry(
  46                                         struct otx2_cpt_pending_queue *q,
  47                                         int qlen)
  48 {
  49         struct otx2_cpt_pending_entry *ent = NULL;
  50
  51         ent = &q->head[q->rear];
  52         if (unlikely(ent->busy))
  53                 return NULL;
  54
  55         q->rear++;
  56         if (unlikely(q->rear == qlen))
  57                 q->rear = 0;
  58
  59         return ent;
  60 }
  61
  62 static inline u32 modulo_inc(u32 index, u32 length, u32 inc)
  63 {
  64         if (WARN_ON(inc > length))
  65                 inc = length;
  66
  67         index += inc;
  68         if (unlikely(index >= length))
  69                 index -= length;
  70
  71         return index;
  72 }
  73
  74 static inline void free_pentry(struct otx2_cpt_pending_entry *pentry)
  75 {
  76         pentry->completion_addr = NULL;
  77         pentry->info = NULL;
  78         pentry->callback = NULL;
  79         pentry->areq = NULL;
  80         pentry->resume_sender = false;
  81         pentry->busy = false;
  82 }
  83
  84 static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
  85                            struct otx2_cpt_pending_queue *pqueue,
  86                            struct otx2_cptlf_info *lf)
  87 {
  88         struct otx2_cptvf_request *cpt_req = &req->req;
  89         struct otx2_cpt_pending_entry *pentry = NULL;
  90         union otx2_cpt_ctrl_info *ctrl = &req->ctrl;
  91         struct otx2_cpt_inst_info *info = NULL;
  92         union otx2_cpt_res_s *result = NULL;
  93         struct otx2_cpt_iq_command iq_cmd;
  94         union otx2_cpt_inst_s cptinst;
  95         int retry, ret = 0;
  96         u8 resume_sender;
  97         gfp_t gfp;
  98
  99         gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
 100                                                               GFP_ATOMIC;
 101         if (unlikely(!otx2_cptlf_started(lf->lfs)))
 102                 return -ENODEV;
 103
 104         info = lf->lfs->ops->cpt_sg_info_create(pdev, req, gfp);
 105         if (unlikely(!info)) {
 106                 dev_err(&pdev->dev, "Setting up cpt inst info failed");
 107                 return -ENOMEM;
 108         }
 109         cpt_req->dlen = info->dlen;
 110
 111         result = info->completion_addr;
 112         result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT;
 113
 114         spin_lock_bh(&pqueue->lock);
 115         pentry = get_free_pending_entry(pqueue, pqueue->qlen);
 116         retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP;
 117         while (unlikely(!pentry) && retry--) {
 118                 spin_unlock_bh(&pqueue->lock);
 119                 udelay(CPT_PENTRY_STEP);
 120                 spin_lock_bh(&pqueue->lock);
 121                 pentry = get_free_pending_entry(pqueue, pqueue->qlen);
 122         }
 123
 124         if (unlikely(!pentry)) {
 125                 ret = -ENOSPC;
 126                 goto destroy_info;
 127         }
 128
 129         /*
 130          * Check if we are close to filling in entire pending queue,
 131          * if so then tell the sender to stop/sleep by returning -EBUSY
 132          * We do it only for context which can sleep (GFP_KERNEL)
 133          */
 134         if (gfp == GFP_KERNEL &&
 135             pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) {
 136                 pentry->resume_sender = true;
 137         } else
 138                 pentry->resume_sender = false;
 139         resume_sender = pentry->resume_sender;
 140         pqueue->pending_count++;
 141
 142         pentry->completion_addr = info->completion_addr;
 143         pentry->info = info;
 144         pentry->callback = req->callback;
 145         pentry->areq = req->areq;
 146         pentry->busy = true;
 147         info->pentry = pentry;
 148         info->time_in = jiffies;
 149         info->req = req;
 150
 151         /* Fill in the command */
 152         iq_cmd.cmd.u = 0;
 153         iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
 154         iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
 155         iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
 156         iq_cmd.cmd.s.dlen   = cpu_to_be16(cpt_req->dlen);
 157
 158         /* 64-bit swap for microcode data reads, not needed for addresses*/
 159         cpu_to_be64s(&iq_cmd.cmd.u);
 160         iq_cmd.dptr = info->dptr_baddr | info->gthr_sz << 60;
 161         iq_cmd.rptr = info->rptr_baddr | info->sctr_sz << 60;
 162         iq_cmd.cptr.s.cptr = cpt_req->cptr_dma;
 163         iq_cmd.cptr.s.grp = ctrl->s.grp;
 164
 165         /* Fill in the CPT_INST_S type command for HW interpretation */
 166         otx2_cpt_fill_inst(&cptinst, &iq_cmd, info->comp_baddr);
 167
 168         /* Print debug info if enabled */
 169         otx2_cpt_dump_sg_list(pdev, req);
 170         pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX2_CPT_INST_SIZE);
 171         print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX2_CPT_INST_SIZE, false);
 172         pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen);
 173         print_hex_dump_debug("", 0, 16, 1, info->in_buffer,
 174                              cpt_req->dlen, false);
 175
 176         /* Send CPT command */
 177         lf->lfs->ops->send_cmd(&cptinst, 1, lf);
 178
 179         /*
 180          * We allocate and prepare pending queue entry in critical section
 181          * together with submitting CPT instruction to CPT instruction queue
 182          * to make sure that order of CPT requests is the same in both
 183          * pending and instruction queues
 184          */
 185         spin_unlock_bh(&pqueue->lock);
 186
 187         ret = resume_sender ? -EBUSY : -EINPROGRESS;
 188         return ret;
 189
 190 destroy_info:
 191         spin_unlock_bh(&pqueue->lock);
 192         otx2_cpt_info_destroy(pdev, info);
 193         return ret;
 194 }
 195
 196 int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
 197                         int cpu_num)
 198 {
 199         struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);
 200         struct otx2_cptlfs_info *lfs = &cptvf->lfs;
 201
 202         return process_request(lfs->pdev, req, &lfs->lf[cpu_num].pqueue,
 203                                &lfs->lf[cpu_num]);
 204 }
 205
 206 static int cpt_process_ccode(struct otx2_cptlfs_info *lfs,
 207                              union otx2_cpt_res_s *cpt_status,
 208                              struct otx2_cpt_inst_info *info,
 209                              u32 *res_code)
 210 {
 211         u8 uc_ccode = lfs->ops->cpt_get_uc_compcode(cpt_status);
 212         u8 ccode = lfs->ops->cpt_get_compcode(cpt_status);
 213         struct pci_dev *pdev = lfs->pdev;
 214
 215         switch (ccode) {
 216         case OTX2_CPT_COMP_E_FAULT:
 217                 dev_err(&pdev->dev,
 218                         "Request failed with DMA fault\n");
 219                 otx2_cpt_dump_sg_list(pdev, info->req);
 220                 break;
 221
 222         case OTX2_CPT_COMP_E_HWERR:
 223                 dev_err(&pdev->dev,
 224                         "Request failed with hardware error\n");
 225                 otx2_cpt_dump_sg_list(pdev, info->req);
 226                 break;
 227
 228         case OTX2_CPT_COMP_E_INSTERR:
 229                 dev_err(&pdev->dev,
 230                         "Request failed with instruction error\n");
 231                 otx2_cpt_dump_sg_list(pdev, info->req);
 232                 break;
 233
 234         case OTX2_CPT_COMP_E_NOTDONE:
 235                 /* check for timeout */
 236                 if (time_after_eq(jiffies, info->time_in +
 237                                   CPT_COMMAND_TIMEOUT * HZ))
 238                         dev_warn(&pdev->dev,
 239                                  "Request timed out 0x%p", info->req);
 240                 else if (info->extra_time < CPT_TIME_IN_RESET_COUNT) {
 241                         info->time_in = jiffies;
 242                         info->extra_time++;
 243                 }
 244                 return 1;
 245
 246         case OTX2_CPT_COMP_E_GOOD:
 247         case OTX2_CPT_COMP_E_WARN:
 248                 /*
 249                  * Check microcode completion code, it is only valid
 250                  * when completion code is CPT_COMP_E::GOOD
 251                  */
 252                 if (uc_ccode != OTX2_CPT_UCC_SUCCESS) {
 253                         /*
 254                          * If requested hmac is truncated and ucode returns
 255                          * s/g write length error then we report success
 256                          * because ucode writes as many bytes of calculated
 257                          * hmac as available in gather buffer and reports
 258                          * s/g write length error if number of bytes in gather
 259                          * buffer is less than full hmac size.
 260                          */
 261                         if (info->req->is_trunc_hmac &&
 262                             uc_ccode == OTX2_CPT_UCC_SG_WRITE_LENGTH) {
 263                                 *res_code = 0;
 264                                 break;
 265                         }
 266
 267                         dev_err(&pdev->dev,
 268                                 "Request failed with software error code 0x%x\n",
 269                                 cpt_status->s.uc_compcode);
 270                         otx2_cpt_dump_sg_list(pdev, info->req);
 271                         break;
 272                 }
 273                 /* Request has been processed with success */
 274                 *res_code = 0;
 275                 break;
 276
 277         default:
 278                 dev_err(&pdev->dev,
 279                         "Request returned invalid status %d\n", ccode);
 280                 break;
 281         }
 282         return 0;
 283 }
 284
 285 static inline void process_pending_queue(struct otx2_cptlfs_info *lfs,
 286                                          struct otx2_cpt_pending_queue *pqueue)
 287 {
 288         struct otx2_cpt_pending_entry *resume_pentry = NULL;
 289         void (*callback)(int status, void *arg, void *req);
 290         struct otx2_cpt_pending_entry *pentry = NULL;
 291         union otx2_cpt_res_s *cpt_status = NULL;
 292         struct otx2_cpt_inst_info *info = NULL;
 293         struct otx2_cpt_req_info *req = NULL;
 294         struct crypto_async_request *areq;
 295         struct pci_dev *pdev = lfs->pdev;
 296         u32 res_code, resume_index;
 297
 298         while (1) {
 299                 spin_lock_bh(&pqueue->lock);
 300                 pentry = &pqueue->head[pqueue->front];
 301
 302                 if (WARN_ON(!pentry)) {
 303                         spin_unlock_bh(&pqueue->lock);
 304                         break;
 305                 }
 306
 307                 res_code = -EINVAL;
 308                 if (unlikely(!pentry->busy)) {
 309                         spin_unlock_bh(&pqueue->lock);
 310                         break;
 311                 }
 312
 313                 if (unlikely(!pentry->callback)) {
 314                         dev_err(&pdev->dev, "Callback NULL\n");
 315                         goto process_pentry;
 316                 }
 317
 318                 info = pentry->info;
 319                 if (unlikely(!info)) {
 320                         dev_err(&pdev->dev, "Pending entry post arg NULL\n");
 321                         goto process_pentry;
 322                 }
 323
 324                 req = info->req;
 325                 if (unlikely(!req)) {
 326                         dev_err(&pdev->dev, "Request NULL\n");
 327                         goto process_pentry;
 328                 }
 329
 330                 cpt_status = pentry->completion_addr;
 331                 if (unlikely(!cpt_status)) {
 332                         dev_err(&pdev->dev, "Completion address NULL\n");
 333                         goto process_pentry;
 334                 }
 335
 336                 if (cpt_process_ccode(lfs, cpt_status, info, &res_code)) {
 337                         spin_unlock_bh(&pqueue->lock);
 338                         return;
 339                 }
 340                 info->pdev = pdev;
 341
 342 process_pentry:
 343                 /*
 344                  * Check if we should inform sending side to resume
 345                  * We do it CPT_IQ_RESUME_MARGIN elements in advance before
 346                  * pending queue becomes empty
 347                  */
 348                 resume_index = modulo_inc(pqueue->front, pqueue->qlen,
 349                                           CPT_IQ_RESUME_MARGIN);
 350                 resume_pentry = &pqueue->head[resume_index];
 351                 if (resume_pentry &&
 352                     resume_pentry->resume_sender) {
 353                         resume_pentry->resume_sender = false;
 354                         callback = resume_pentry->callback;
 355                         areq = resume_pentry->areq;
 356
 357                         if (callback) {
 358                                 spin_unlock_bh(&pqueue->lock);
 359
 360                                 /*
 361                                  * EINPROGRESS is an indication for sending
 362                                  * side that it can resume sending requests
 363                                  */
 364                                 callback(-EINPROGRESS, areq, info);
 365                                 spin_lock_bh(&pqueue->lock);
 366                         }
 367                 }
 368
 369                 callback = pentry->callback;
 370                 areq = pentry->areq;
 371                 free_pentry(pentry);
 372
 373                 pqueue->pending_count--;
 374                 pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1);
 375                 spin_unlock_bh(&pqueue->lock);
 376
 377                 /*
 378                  * Call callback after current pending entry has been
 379                  * processed, we don't do it if the callback pointer is
 380                  * invalid.
 381                  */
 382                 if (callback)
 383                         callback(res_code, areq, info);
 384         }
 385 }
 386
 387 void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe)
 388 {
 389         process_pending_queue(wqe->lfs,
 390                               &wqe->lfs->lf[wqe->lf_num].pqueue);
 391 }
 392
 393 int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev)
 394 {
 395         struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);
 396
 397         return cptvf->lfs.kcrypto_eng_grp_num;
 398 }