drivers/crypto/hisilicon/qm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2019 HiSilicon Limited. */
   3 #include <asm/page.h>
   4 #include <linux/acpi.h>
   5 #include <linux/aer.h>
   6 #include <linux/bitmap.h>
   7 #include <linux/debugfs.h>
   8 #include <linux/dma-mapping.h>
   9 #include <linux/idr.h>
  10 #include <linux/io.h>
  11 #include <linux/irqreturn.h>
  12 #include <linux/log2.h>
  13 #include <linux/seq_file.h>
  14 #include <linux/slab.h>
  15 #include <linux/uacce.h>
  16 #include <linux/uaccess.h>
  17 #include <uapi/misc/uacce/hisi_qm.h>
  18 #include "qm.h"
  19
  20 /* eq/aeq irq enable */
  21 #define QM_VF_AEQ_INT_SOURCE            0x0
  22 #define QM_VF_AEQ_INT_MASK              0x4
  23 #define QM_VF_EQ_INT_SOURCE             0x8
  24 #define QM_VF_EQ_INT_MASK               0xc
  25 #define QM_IRQ_NUM_V1                   1
  26 #define QM_IRQ_NUM_PF_V2                4
  27 #define QM_IRQ_NUM_VF_V2                2
  28
  29 #define QM_EQ_EVENT_IRQ_VECTOR          0
  30 #define QM_AEQ_EVENT_IRQ_VECTOR         1
  31 #define QM_ABNORMAL_EVENT_IRQ_VECTOR    3
  32
  33 /* mailbox */
  34 #define QM_MB_CMD_SQC                   0x0
  35 #define QM_MB_CMD_CQC                   0x1
  36 #define QM_MB_CMD_EQC                   0x2
  37 #define QM_MB_CMD_AEQC                  0x3
  38 #define QM_MB_CMD_SQC_BT                0x4
  39 #define QM_MB_CMD_CQC_BT                0x5
  40 #define QM_MB_CMD_SQC_VFT_V2            0x6
  41 #define QM_MB_CMD_STOP_QP               0x8
  42
  43 #define QM_MB_CMD_SEND_BASE             0x300
  44 #define QM_MB_EVENT_SHIFT               8
  45 #define QM_MB_BUSY_SHIFT                13
  46 #define QM_MB_OP_SHIFT                  14
  47 #define QM_MB_CMD_DATA_ADDR_L           0x304
  48 #define QM_MB_CMD_DATA_ADDR_H           0x308
  49
  50 /* sqc shift */
  51 #define QM_SQ_HOP_NUM_SHIFT             0
  52 #define QM_SQ_PAGE_SIZE_SHIFT           4
  53 #define QM_SQ_BUF_SIZE_SHIFT            8
  54 #define QM_SQ_SQE_SIZE_SHIFT            12
  55 #define QM_SQ_PRIORITY_SHIFT            0
  56 #define QM_SQ_ORDERS_SHIFT              4
  57 #define QM_SQ_TYPE_SHIFT                8
  58 #define QM_QC_PASID_ENABLE              0x1
  59 #define QM_QC_PASID_ENABLE_SHIFT        7
  60
  61 #define QM_SQ_TYPE_MASK                 GENMASK(3, 0)
  62 #define QM_SQ_TAIL_IDX(sqc)             ((le16_to_cpu((sqc)->w11) >> 6) & 0x1)
  63
  64 /* cqc shift */
  65 #define QM_CQ_HOP_NUM_SHIFT             0
  66 #define QM_CQ_PAGE_SIZE_SHIFT           4
  67 #define QM_CQ_BUF_SIZE_SHIFT            8
  68 #define QM_CQ_CQE_SIZE_SHIFT            12
  69 #define QM_CQ_PHASE_SHIFT               0
  70 #define QM_CQ_FLAG_SHIFT                1
  71
  72 #define QM_CQE_PHASE(cqe)               (le16_to_cpu((cqe)->w7) & 0x1)
  73 #define QM_QC_CQE_SIZE                  4
  74 #define QM_CQ_TAIL_IDX(cqc)             ((le16_to_cpu((cqc)->w11) >> 6) & 0x1)
  75
  76 /* eqc shift */
  77 #define QM_EQE_AEQE_SIZE                (2UL << 12)
  78 #define QM_EQC_PHASE_SHIFT              16
  79
  80 #define QM_EQE_PHASE(eqe)               ((le32_to_cpu((eqe)->dw0) >> 16) & 0x1)
  81 #define QM_EQE_CQN_MASK                 GENMASK(15, 0)
  82
  83 #define QM_AEQE_PHASE(aeqe)             ((le32_to_cpu((aeqe)->dw0) >> 16) & 0x1)
  84 #define QM_AEQE_TYPE_SHIFT              17
  85
  86 #define QM_DOORBELL_CMD_SQ              0
  87 #define QM_DOORBELL_CMD_CQ              1
  88 #define QM_DOORBELL_CMD_EQ              2
  89 #define QM_DOORBELL_CMD_AEQ             3
  90
  91 #define QM_DOORBELL_BASE_V1             0x340
  92 #define QM_DB_CMD_SHIFT_V1              16
  93 #define QM_DB_INDEX_SHIFT_V1            32
  94 #define QM_DB_PRIORITY_SHIFT_V1         48
  95 #define QM_DOORBELL_SQ_CQ_BASE_V2       0x1000
  96 #define QM_DOORBELL_EQ_AEQ_BASE_V2      0x2000
  97 #define QM_QUE_ISO_CFG_V                0x0030
  98 #define QM_QUE_ISO_EN                   0x100154
  99 #define QM_CAPBILITY                    0x100158
 100 #define QM_QP_NUN_MASK                  GENMASK(10, 0)
 101 #define QM_QP_DB_INTERVAL               0x10000
 102 #define QM_QP_MAX_NUM_SHIFT             11
 103 #define QM_DB_CMD_SHIFT_V2              12
 104 #define QM_DB_RAND_SHIFT_V2             16
 105 #define QM_DB_INDEX_SHIFT_V2            32
 106 #define QM_DB_PRIORITY_SHIFT_V2         48
 107
 108 #define QM_MEM_START_INIT               0x100040
 109 #define QM_MEM_INIT_DONE                0x100044
 110 #define QM_VFT_CFG_RDY                  0x10006c
 111 #define QM_VFT_CFG_OP_WR                0x100058
 112 #define QM_VFT_CFG_TYPE                 0x10005c
 113 #define QM_SQC_VFT                      0x0
 114 #define QM_CQC_VFT                      0x1
 115 #define QM_VFT_CFG                      0x100060
 116 #define QM_VFT_CFG_OP_ENABLE            0x100054
 117
 118 #define QM_VFT_CFG_DATA_L               0x100064
 119 #define QM_VFT_CFG_DATA_H               0x100068
 120 #define QM_SQC_VFT_BUF_SIZE             (7ULL << 8)
 121 #define QM_SQC_VFT_SQC_SIZE             (5ULL << 12)
 122 #define QM_SQC_VFT_INDEX_NUMBER         (1ULL << 16)
 123 #define QM_SQC_VFT_START_SQN_SHIFT      28
 124 #define QM_SQC_VFT_VALID                (1ULL << 44)
 125 #define QM_SQC_VFT_SQN_SHIFT            45
 126 #define QM_CQC_VFT_BUF_SIZE             (7ULL << 8)
 127 #define QM_CQC_VFT_SQC_SIZE             (5ULL << 12)
 128 #define QM_CQC_VFT_INDEX_NUMBER         (1ULL << 16)
 129 #define QM_CQC_VFT_VALID                (1ULL << 28)
 130
 131 #define QM_SQC_VFT_BASE_SHIFT_V2        28
 132 #define QM_SQC_VFT_BASE_MASK_V2         GENMASK(15, 0)
 133 #define QM_SQC_VFT_NUM_SHIFT_V2         45
 134 #define QM_SQC_VFT_NUM_MASK_v2          GENMASK(9, 0)
 135
 136 #define QM_DFX_CNT_CLR_CE               0x100118
 137
 138 #define QM_ABNORMAL_INT_SOURCE          0x100000
 139 #define QM_ABNORMAL_INT_SOURCE_CLR      GENMASK(14, 0)
 140 #define QM_ABNORMAL_INT_MASK            0x100004
 141 #define QM_ABNORMAL_INT_MASK_VALUE      0x7fff
 142 #define QM_ABNORMAL_INT_STATUS          0x100008
 143 #define QM_ABNORMAL_INT_SET             0x10000c
 144 #define QM_ABNORMAL_INF00               0x100010
 145 #define QM_FIFO_OVERFLOW_TYPE           0xc0
 146 #define QM_FIFO_OVERFLOW_TYPE_SHIFT     6
 147 #define QM_FIFO_OVERFLOW_VF             0x3f
 148 #define QM_ABNORMAL_INF01               0x100014
 149 #define QM_DB_TIMEOUT_TYPE              0xc0
 150 #define QM_DB_TIMEOUT_TYPE_SHIFT        6
 151 #define QM_DB_TIMEOUT_VF                0x3f
 152 #define QM_RAS_CE_ENABLE                0x1000ec
 153 #define QM_RAS_FE_ENABLE                0x1000f0
 154 #define QM_RAS_NFE_ENABLE               0x1000f4
 155 #define QM_RAS_CE_THRESHOLD             0x1000f8
 156 #define QM_RAS_CE_TIMES_PER_IRQ         1
 157 #define QM_RAS_MSI_INT_SEL              0x1040f4
 158
 159 #define QM_RESET_WAIT_TIMEOUT           400
 160 #define QM_PEH_VENDOR_ID                0x1000d8
 161 #define ACC_VENDOR_ID_VALUE             0x5a5a
 162 #define QM_PEH_DFX_INFO0                0x1000fc
 163 #define ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT 3
 164 #define ACC_PEH_MSI_DISABLE             GENMASK(31, 0)
 165 #define ACC_MASTER_GLOBAL_CTRL_SHUTDOWN 0x1
 166 #define ACC_MASTER_TRANS_RETURN_RW      3
 167 #define ACC_MASTER_TRANS_RETURN         0x300150
 168 #define ACC_MASTER_GLOBAL_CTRL          0x300000
 169 #define ACC_AM_CFG_PORT_WR_EN           0x30001c
 170 #define QM_RAS_NFE_MBIT_DISABLE         ~QM_ECC_MBIT
 171 #define ACC_AM_ROB_ECC_INT_STS          0x300104
 172 #define ACC_ROB_ECC_ERR_MULTPL          BIT(1)
 173
 174 #define QM_DFX_MB_CNT_VF                0x104010
 175 #define QM_DFX_DB_CNT_VF                0x104020
 176 #define QM_DFX_SQE_CNT_VF_SQN           0x104030
 177 #define QM_DFX_CQE_CNT_VF_CQN           0x104040
 178 #define QM_DFX_QN_SHIFT                 16
 179 #define CURRENT_FUN_MASK                GENMASK(5, 0)
 180 #define CURRENT_Q_MASK                  GENMASK(31, 16)
 181
 182 #define POLL_PERIOD                     10
 183 #define POLL_TIMEOUT                    1000
 184 #define WAIT_PERIOD_US_MAX              200
 185 #define WAIT_PERIOD_US_MIN              100
 186 #define MAX_WAIT_COUNTS                 1000
 187 #define QM_CACHE_WB_START               0x204
 188 #define QM_CACHE_WB_DONE                0x208
 189
 190 #define PCI_BAR_2                       2
 191 #define PCI_BAR_4                       4
 192 #define QM_SQE_DATA_ALIGN_MASK          GENMASK(6, 0)
 193 #define QMC_ALIGN(sz)                   ALIGN(sz, 32)
 194
 195 #define QM_DBG_READ_LEN         256
 196 #define QM_DBG_WRITE_LEN                1024
 197 #define QM_DBG_TMP_BUF_LEN              22
 198 #define QM_PCI_COMMAND_INVALID          ~0
 199
 200 #define WAIT_PERIOD                     20
 201 #define REMOVE_WAIT_DELAY               10
 202 #define QM_SQE_ADDR_MASK                GENMASK(7, 0)
 203 #define QM_EQ_DEPTH                     (1024 * 2)
 204
 205 #define QM_DRIVER_REMOVING              0
 206 #define QM_RST_SCHED                    1
 207 #define QM_RESETTING                    2
 208
 209 #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \
 210         (((hop_num) << QM_CQ_HOP_NUM_SHIFT)     | \
 211         ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT)      | \
 212         ((buf_sz) << QM_CQ_BUF_SIZE_SHIFT)      | \
 213         ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
 214
 215 #define QM_MK_CQC_DW3_V2(cqe_sz) \
 216         ((QM_Q_DEPTH - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
 217
 218 #define QM_MK_SQC_W13(priority, orders, alg_type) \
 219         (((priority) << QM_SQ_PRIORITY_SHIFT)   | \
 220         ((orders) << QM_SQ_ORDERS_SHIFT)        | \
 221         (((alg_type) & QM_SQ_TYPE_MASK) << QM_SQ_TYPE_SHIFT))
 222
 223 #define QM_MK_SQC_DW3_V1(hop_num, pg_sz, buf_sz, sqe_sz) \
 224         (((hop_num) << QM_SQ_HOP_NUM_SHIFT)     | \
 225         ((pg_sz) << QM_SQ_PAGE_SIZE_SHIFT)      | \
 226         ((buf_sz) << QM_SQ_BUF_SIZE_SHIFT)      | \
 227         ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
 228
 229 #define QM_MK_SQC_DW3_V2(sqe_sz) \
 230         ((QM_Q_DEPTH - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
 231
 232 #define INIT_QC_COMMON(qc, base, pasid) do {                    \
 233         (qc)->head = 0;                                         \
 234         (qc)->tail = 0;                                         \
 235         (qc)->base_l = cpu_to_le32(lower_32_bits(base));        \
 236         (qc)->base_h = cpu_to_le32(upper_32_bits(base));        \
 237         (qc)->dw3 = 0;                                          \
 238         (qc)->w8 = 0;                                           \
 239         (qc)->rsvd0 = 0;                                        \
 240         (qc)->pasid = cpu_to_le16(pasid);                       \
 241         (qc)->w11 = 0;                                          \
 242         (qc)->rsvd1 = 0;                                        \
 243 } while (0)
 244
 245 enum vft_type {
 246         SQC_VFT = 0,
 247         CQC_VFT,
 248 };
 249
 250 enum acc_err_result {
 251         ACC_ERR_NONE,
 252         ACC_ERR_NEED_RESET,
 253         ACC_ERR_RECOVERED,
 254 };
 255
 256 struct qm_cqe {
 257         __le32 rsvd0;
 258         __le16 cmd_id;
 259         __le16 rsvd1;
 260         __le16 sq_head;
 261         __le16 sq_num;
 262         __le16 rsvd2;
 263         __le16 w7;
 264 };
 265
 266 struct qm_eqe {
 267         __le32 dw0;
 268 };
 269
 270 struct qm_aeqe {
 271         __le32 dw0;
 272 };
 273
 274 struct qm_sqc {
 275         __le16 head;
 276         __le16 tail;
 277         __le32 base_l;
 278         __le32 base_h;
 279         __le32 dw3;
 280         __le16 w8;
 281         __le16 rsvd0;
 282         __le16 pasid;
 283         __le16 w11;
 284         __le16 cq_num;
 285         __le16 w13;
 286         __le32 rsvd1;
 287 };
 288
 289 struct qm_cqc {
 290         __le16 head;
 291         __le16 tail;
 292         __le32 base_l;
 293         __le32 base_h;
 294         __le32 dw3;
 295         __le16 w8;
 296         __le16 rsvd0;
 297         __le16 pasid;
 298         __le16 w11;
 299         __le32 dw6;
 300         __le32 rsvd1;
 301 };
 302
 303 struct qm_eqc {
 304         __le16 head;
 305         __le16 tail;
 306         __le32 base_l;
 307         __le32 base_h;
 308         __le32 dw3;
 309         __le32 rsvd[2];
 310         __le32 dw6;
 311 };
 312
 313 struct qm_aeqc {
 314         __le16 head;
 315         __le16 tail;
 316         __le32 base_l;
 317         __le32 base_h;
 318         __le32 dw3;
 319         __le32 rsvd[2];
 320         __le32 dw6;
 321 };
 322
 323 struct qm_mailbox {
 324         __le16 w0;
 325         __le16 queue_num;
 326         __le32 base_l;
 327         __le32 base_h;
 328         __le32 rsvd;
 329 };
 330
 331 struct qm_doorbell {
 332         __le16 queue_num;
 333         __le16 cmd;
 334         __le16 index;
 335         __le16 priority;
 336 };
 337
 338 struct hisi_qm_resource {
 339         struct hisi_qm *qm;
 340         int distance;
 341         struct list_head list;
 342 };
 343
 344 struct hisi_qm_hw_ops {
 345         int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number);
 346         void (*qm_db)(struct hisi_qm *qm, u16 qn,
 347                       u8 cmd, u16 index, u8 priority);
 348         u32 (*get_irq_num)(struct hisi_qm *qm);
 349         int (*debug_init)(struct hisi_qm *qm);
 350         void (*hw_error_init)(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe);
 351         void (*hw_error_uninit)(struct hisi_qm *qm);
 352         enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm);
 353         int (*stop_qp)(struct hisi_qp *qp);
 354 };
 355
 356 struct qm_dfx_item {
 357         const char *name;
 358         u32 offset;
 359 };
 360
 361 static struct qm_dfx_item qm_dfx_files[] = {
 362         {"err_irq", offsetof(struct qm_dfx, err_irq_cnt)},
 363         {"aeq_irq", offsetof(struct qm_dfx, aeq_irq_cnt)},
 364         {"abnormal_irq", offsetof(struct qm_dfx, abnormal_irq_cnt)},
 365         {"create_qp_err", offsetof(struct qm_dfx, create_qp_err_cnt)},
 366         {"mb_err", offsetof(struct qm_dfx, mb_err_cnt)},
 367 };
 368
 369 static const char * const qm_debug_file_name[] = {
 370         [CURRENT_QM]   = "current_qm",
 371         [CURRENT_Q]    = "current_q",
 372         [CLEAR_ENABLE] = "clear_enable",
 373 };
 374
 375 struct hisi_qm_hw_error {
 376         u32 int_msk;
 377         const char *msg;
 378 };
 379
 380 static const struct hisi_qm_hw_error qm_hw_error[] = {
 381         { .int_msk = BIT(0), .msg = "qm_axi_rresp" },
 382         { .int_msk = BIT(1), .msg = "qm_axi_bresp" },
 383         { .int_msk = BIT(2), .msg = "qm_ecc_mbit" },
 384         { .int_msk = BIT(3), .msg = "qm_ecc_1bit" },
 385         { .int_msk = BIT(4), .msg = "qm_acc_get_task_timeout" },
 386         { .int_msk = BIT(5), .msg = "qm_acc_do_task_timeout" },
 387         { .int_msk = BIT(6), .msg = "qm_acc_wb_not_ready_timeout" },
 388         { .int_msk = BIT(7), .msg = "qm_sq_cq_vf_invalid" },
 389         { .int_msk = BIT(8), .msg = "qm_cq_vf_invalid" },
 390         { .int_msk = BIT(9), .msg = "qm_sq_vf_invalid" },
 391         { .int_msk = BIT(10), .msg = "qm_db_timeout" },
 392         { .int_msk = BIT(11), .msg = "qm_of_fifo_of" },
 393         { .int_msk = BIT(12), .msg = "qm_db_random_invalid" },
 394         { .int_msk = BIT(13), .msg = "qm_mailbox_timeout" },
 395         { .int_msk = BIT(14), .msg = "qm_flr_timeout" },
 396         { /* sentinel */ }
 397 };
 398
 399 static const char * const qm_db_timeout[] = {
 400         "sq", "cq", "eq", "aeq",
 401 };
 402
 403 static const char * const qm_fifo_overflow[] = {
 404         "cq", "eq", "aeq",
 405 };
 406
 407 static const char * const qm_s[] = {
 408         "init", "start", "close", "stop",
 409 };
 410
 411 static const char * const qp_s[] = {
 412         "none", "init", "start", "stop", "close",
 413 };
 414
 415 static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new)
 416 {
 417         enum qm_state curr = atomic_read(&qm->status.flags);
 418         bool avail = false;
 419
 420         switch (curr) {
 421         case QM_INIT:
 422                 if (new == QM_START || new == QM_CLOSE)
 423                         avail = true;
 424                 break;
 425         case QM_START:
 426                 if (new == QM_STOP)
 427                         avail = true;
 428                 break;
 429         case QM_STOP:
 430                 if (new == QM_CLOSE || new == QM_START)
 431                         avail = true;
 432                 break;
 433         default:
 434                 break;
 435         }
 436
 437         dev_dbg(&qm->pdev->dev, "change qm state from %s to %s\n",
 438                 qm_s[curr], qm_s[new]);
 439
 440         if (!avail)
 441                 dev_warn(&qm->pdev->dev, "Can not change qm state from %s to %s\n",
 442                          qm_s[curr], qm_s[new]);
 443
 444         return avail;
 445 }
 446
 447 static bool qm_qp_avail_state(struct hisi_qm *qm, struct hisi_qp *qp,
 448                               enum qp_state new)
 449 {
 450         enum qm_state qm_curr = atomic_read(&qm->status.flags);
 451         enum qp_state qp_curr = 0;
 452         bool avail = false;
 453
 454         if (qp)
 455                 qp_curr = atomic_read(&qp->qp_status.flags);
 456
 457         switch (new) {
 458         case QP_INIT:
 459                 if (qm_curr == QM_START || qm_curr == QM_INIT)
 460                         avail = true;
 461                 break;
 462         case QP_START:
 463                 if ((qm_curr == QM_START && qp_curr == QP_INIT) ||
 464                     (qm_curr == QM_START && qp_curr == QP_STOP))
 465                         avail = true;
 466                 break;
 467         case QP_STOP:
 468                 if ((qm_curr == QM_START && qp_curr == QP_START) ||
 469                     (qp_curr == QP_INIT))
 470                         avail = true;
 471                 break;
 472         case QP_CLOSE:
 473                 if ((qm_curr == QM_START && qp_curr == QP_INIT) ||
 474                     (qm_curr == QM_START && qp_curr == QP_STOP) ||
 475                     (qm_curr == QM_STOP && qp_curr == QP_STOP)  ||
 476                     (qm_curr == QM_STOP && qp_curr == QP_INIT))
 477                         avail = true;
 478                 break;
 479         default:
 480                 break;
 481         }
 482
 483         dev_dbg(&qm->pdev->dev, "change qp state from %s to %s in QM %s\n",
 484                 qp_s[qp_curr], qp_s[new], qm_s[qm_curr]);
 485
 486         if (!avail)
 487                 dev_warn(&qm->pdev->dev,
 488                          "Can not change qp state from %s to %s in QM %s\n",
 489                          qp_s[qp_curr], qp_s[new], qm_s[qm_curr]);
 490
 491         return avail;
 492 }
 493
 494 /* return 0 mailbox ready, -ETIMEDOUT hardware timeout */
 495 static int qm_wait_mb_ready(struct hisi_qm *qm)
 496 {
 497         u32 val;
 498
 499         return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE,
 500                                           val, !((val >> QM_MB_BUSY_SHIFT) &
 501                                           0x1), POLL_PERIOD, POLL_TIMEOUT);
 502 }
 503
 504 /* 128 bit should be written to hardware at one time to trigger a mailbox */
 505 static void qm_mb_write(struct hisi_qm *qm, const void *src)
 506 {
 507         void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE;
 508         unsigned long tmp0 = 0, tmp1 = 0;
 509
 510         if (!IS_ENABLED(CONFIG_ARM64)) {
 511                 memcpy_toio(fun_base, src, 16);
 512                 wmb();
 513                 return;
 514         }
 515
 516         asm volatile("ldp %0, %1, %3\n"
 517                      "stp %0, %1, %2\n"
 518                      "dsb sy\n"
 519                      : "=&r" (tmp0),
 520                        "=&r" (tmp1),
 521                        "+Q" (*((char __iomem *)fun_base))
 522                      : "Q" (*((char *)src))
 523                      : "memory");
 524 }
 525
 526 static int qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue,
 527                  bool op)
 528 {
 529         struct qm_mailbox mailbox;
 530         int ret = 0;
 531
 532         dev_dbg(&qm->pdev->dev, "QM mailbox request to q%u: %u-%llx\n",
 533                 queue, cmd, (unsigned long long)dma_addr);
 534
 535         mailbox.w0 = cpu_to_le16(cmd |
 536                      (op ? 0x1 << QM_MB_OP_SHIFT : 0) |
 537                      (0x1 << QM_MB_BUSY_SHIFT));
 538         mailbox.queue_num = cpu_to_le16(queue);
 539         mailbox.base_l = cpu_to_le32(lower_32_bits(dma_addr));
 540         mailbox.base_h = cpu_to_le32(upper_32_bits(dma_addr));
 541         mailbox.rsvd = 0;
 542
 543         mutex_lock(&qm->mailbox_lock);
 544
 545         if (unlikely(qm_wait_mb_ready(qm))) {
 546                 ret = -EBUSY;
 547                 dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n");
 548                 goto busy_unlock;
 549         }
 550
 551         qm_mb_write(qm, &mailbox);
 552
 553         if (unlikely(qm_wait_mb_ready(qm))) {
 554                 ret = -EBUSY;
 555                 dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n");
 556                 goto busy_unlock;
 557         }
 558
 559 busy_unlock:
 560         mutex_unlock(&qm->mailbox_lock);
 561
 562         if (ret)
 563                 atomic64_inc(&qm->debug.dfx.mb_err_cnt);
 564         return ret;
 565 }
 566
 567 static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
 568 {
 569         u64 doorbell;
 570
 571         doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V1) |
 572                    ((u64)index << QM_DB_INDEX_SHIFT_V1)  |
 573                    ((u64)priority << QM_DB_PRIORITY_SHIFT_V1);
 574
 575         writeq(doorbell, qm->io_base + QM_DOORBELL_BASE_V1);
 576 }
 577
 578 static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
 579 {
 580         void __iomem *io_base = qm->io_base;
 581         u16 randata = 0;
 582         u64 doorbell;
 583
 584         if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ)
 585                 io_base = qm->db_io_base + (u64)qn * qm->db_interval +
 586                           QM_DOORBELL_SQ_CQ_BASE_V2;
 587         else
 588                 io_base += QM_DOORBELL_EQ_AEQ_BASE_V2;
 589
 590         doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) |
 591                    ((u64)randata << QM_DB_RAND_SHIFT_V2) |
 592                    ((u64)index << QM_DB_INDEX_SHIFT_V2)  |
 593                    ((u64)priority << QM_DB_PRIORITY_SHIFT_V2);
 594
 595         writeq(doorbell, io_base);
 596 }
 597
 598 static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
 599 {
 600         dev_dbg(&qm->pdev->dev, "QM doorbell request: qn=%u, cmd=%u, index=%u\n",
 601                 qn, cmd, index);
 602
 603         qm->ops->qm_db(qm, qn, cmd, index, priority);
 604 }
 605
 606 static int qm_dev_mem_reset(struct hisi_qm *qm)
 607 {
 608         u32 val;
 609
 610         writel(0x1, qm->io_base + QM_MEM_START_INIT);
 611         return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val,
 612                                           val & BIT(0), POLL_PERIOD,
 613                                           POLL_TIMEOUT);
 614 }
 615
 616 static u32 qm_get_irq_num_v1(struct hisi_qm *qm)
 617 {
 618         return QM_IRQ_NUM_V1;
 619 }
 620
 621 static u32 qm_get_irq_num_v2(struct hisi_qm *qm)
 622 {
 623         if (qm->fun_type == QM_HW_PF)
 624                 return QM_IRQ_NUM_PF_V2;
 625         else
 626                 return QM_IRQ_NUM_VF_V2;
 627 }
 628
 629 static struct hisi_qp *qm_to_hisi_qp(struct hisi_qm *qm, struct qm_eqe *eqe)
 630 {
 631         u16 cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK;
 632
 633         return &qm->qp_array[cqn];
 634 }
 635
 636 static void qm_cq_head_update(struct hisi_qp *qp)
 637 {
 638         if (qp->qp_status.cq_head == QM_Q_DEPTH - 1) {
 639                 qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase;
 640                 qp->qp_status.cq_head = 0;
 641         } else {
 642                 qp->qp_status.cq_head++;
 643         }
 644 }
 645
 646 static void qm_poll_qp(struct hisi_qp *qp, struct hisi_qm *qm)
 647 {
 648         if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP))
 649                 return;
 650
 651         if (qp->event_cb) {
 652                 qp->event_cb(qp);
 653                 return;
 654         }
 655
 656         if (qp->req_cb) {
 657                 struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head;
 658
 659                 while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) {
 660                         dma_rmb();
 661                         qp->req_cb(qp, qp->sqe + qm->sqe_size *
 662                                    le16_to_cpu(cqe->sq_head));
 663                         qm_cq_head_update(qp);
 664                         cqe = qp->cqe + qp->qp_status.cq_head;
 665                         qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ,
 666                               qp->qp_status.cq_head, 0);
 667                         atomic_dec(&qp->qp_status.used);
 668                 }
 669
 670                 /* set c_flag */
 671                 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ,
 672                       qp->qp_status.cq_head, 1);
 673         }
 674 }
 675
 676 static void qm_work_process(struct work_struct *work)
 677 {
 678         struct hisi_qm *qm = container_of(work, struct hisi_qm, work);
 679         struct qm_eqe *eqe = qm->eqe + qm->status.eq_head;
 680         struct hisi_qp *qp;
 681         int eqe_num = 0;
 682
 683         while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) {
 684                 eqe_num++;
 685                 qp = qm_to_hisi_qp(qm, eqe);
 686                 qm_poll_qp(qp, qm);
 687
 688                 if (qm->status.eq_head == QM_EQ_DEPTH - 1) {
 689                         qm->status.eqc_phase = !qm->status.eqc_phase;
 690                         eqe = qm->eqe;
 691                         qm->status.eq_head = 0;
 692                 } else {
 693                         eqe++;
 694                         qm->status.eq_head++;
 695                 }
 696
 697                 if (eqe_num == QM_EQ_DEPTH / 2 - 1) {
 698                         eqe_num = 0;
 699                         qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
 700                 }
 701         }
 702
 703         qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
 704 }
 705
 706 static irqreturn_t do_qm_irq(int irq, void *data)
 707 {
 708         struct hisi_qm *qm = (struct hisi_qm *)data;
 709
 710         /* the workqueue created by device driver of QM */
 711         if (qm->wq)
 712                 queue_work(qm->wq, &qm->work);
 713         else
 714                 schedule_work(&qm->work);
 715
 716         return IRQ_HANDLED;
 717 }
 718
 719 static irqreturn_t qm_irq(int irq, void *data)
 720 {
 721         struct hisi_qm *qm = data;
 722
 723         if (readl(qm->io_base + QM_VF_EQ_INT_SOURCE))
 724                 return do_qm_irq(irq, data);
 725
 726         atomic64_inc(&qm->debug.dfx.err_irq_cnt);
 727         dev_err(&qm->pdev->dev, "invalid int source\n");
 728         qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
 729
 730         return IRQ_NONE;
 731 }
 732
 733 static irqreturn_t qm_aeq_irq(int irq, void *data)
 734 {
 735         struct hisi_qm *qm = data;
 736         struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head;
 737         u32 type;
 738
 739         atomic64_inc(&qm->debug.dfx.aeq_irq_cnt);
 740         if (!readl(qm->io_base + QM_VF_AEQ_INT_SOURCE))
 741                 return IRQ_NONE;
 742
 743         while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) {
 744                 type = le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT;
 745                 if (type < ARRAY_SIZE(qm_fifo_overflow))
 746                         dev_err(&qm->pdev->dev, "%s overflow\n",
 747                                 qm_fifo_overflow[type]);
 748                 else
 749                         dev_err(&qm->pdev->dev, "unknown error type %u\n",
 750                                 type);
 751
 752                 if (qm->status.aeq_head == QM_Q_DEPTH - 1) {
 753                         qm->status.aeqc_phase = !qm->status.aeqc_phase;
 754                         aeqe = qm->aeqe;
 755                         qm->status.aeq_head = 0;
 756                 } else {
 757                         aeqe++;
 758                         qm->status.aeq_head++;
 759                 }
 760
 761                 qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
 762         }
 763
 764         return IRQ_HANDLED;
 765 }
 766
 767 static void qm_irq_unregister(struct hisi_qm *qm)
 768 {
 769         struct pci_dev *pdev = qm->pdev;
 770
 771         free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm);
 772
 773         if (qm->ver == QM_HW_V1)
 774                 return;
 775
 776         free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm);
 777
 778         if (qm->fun_type == QM_HW_PF)
 779                 free_irq(pci_irq_vector(pdev,
 780                          QM_ABNORMAL_EVENT_IRQ_VECTOR), qm);
 781 }
 782
 783 static void qm_init_qp_status(struct hisi_qp *qp)
 784 {
 785         struct hisi_qp_status *qp_status = &qp->qp_status;
 786
 787         qp_status->sq_tail = 0;
 788         qp_status->cq_head = 0;
 789         qp_status->cqc_phase = true;
 790         atomic_set(&qp_status->used, 0);
 791 }
 792
 793 static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base,
 794                             u32 number)
 795 {
 796         u64 tmp = 0;
 797
 798         if (number > 0) {
 799                 switch (type) {
 800                 case SQC_VFT:
 801                         if (qm->ver == QM_HW_V1) {
 802                                 tmp = QM_SQC_VFT_BUF_SIZE       |
 803                                       QM_SQC_VFT_SQC_SIZE       |
 804                                       QM_SQC_VFT_INDEX_NUMBER   |
 805                                       QM_SQC_VFT_VALID          |
 806                                       (u64)base << QM_SQC_VFT_START_SQN_SHIFT;
 807                         } else {
 808                                 tmp = (u64)base << QM_SQC_VFT_START_SQN_SHIFT |
 809                                       QM_SQC_VFT_VALID |
 810                                       (u64)(number - 1) << QM_SQC_VFT_SQN_SHIFT;
 811                         }
 812                         break;
 813                 case CQC_VFT:
 814                         if (qm->ver == QM_HW_V1) {
 815                                 tmp = QM_CQC_VFT_BUF_SIZE       |
 816                                       QM_CQC_VFT_SQC_SIZE       |
 817                                       QM_CQC_VFT_INDEX_NUMBER   |
 818                                       QM_CQC_VFT_VALID;
 819                         } else {
 820                                 tmp = QM_CQC_VFT_VALID;
 821                         }
 822                         break;
 823                 }
 824         }
 825
 826         writel(lower_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_L);
 827         writel(upper_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_H);
 828 }
 829
 830 static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type,
 831                              u32 fun_num, u32 base, u32 number)
 832 {
 833         unsigned int val;
 834         int ret;
 835
 836         ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
 837                                          val & BIT(0), POLL_PERIOD,
 838                                          POLL_TIMEOUT);
 839         if (ret)
 840                 return ret;
 841
 842         writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR);
 843         writel(type, qm->io_base + QM_VFT_CFG_TYPE);
 844         writel(fun_num, qm->io_base + QM_VFT_CFG);
 845
 846         qm_vft_data_cfg(qm, type, base, number);
 847
 848         writel(0x0, qm->io_base + QM_VFT_CFG_RDY);
 849         writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);
 850
 851         return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
 852                                           val & BIT(0), POLL_PERIOD,
 853                                           POLL_TIMEOUT);
 854 }
 855
 856 /* The config should be conducted after qm_dev_mem_reset() */
 857 static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
 858                               u32 number)
 859 {
 860         int ret, i;
 861
 862         for (i = SQC_VFT; i <= CQC_VFT; i++) {
 863                 ret = qm_set_vft_common(qm, i, fun_num, base, number);
 864                 if (ret)
 865                         return ret;
 866         }
 867
 868         return 0;
 869 }
 870
 871 static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number)
 872 {
 873         u64 sqc_vft;
 874         int ret;
 875
 876         ret = qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1);
 877         if (ret)
 878                 return ret;
 879
 880         sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
 881                   ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32);
 882         *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
 883         *number = (QM_SQC_VFT_NUM_MASK_v2 &
 884                    (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
 885
 886         return 0;
 887 }
 888
 889 static int qm_get_vf_qp_num(struct hisi_qm *qm, u32 fun_num)
 890 {
 891         u32 remain_q_num, vfq_num;
 892         u32 num_vfs = qm->vfs_num;
 893
 894         vfq_num = (qm->ctrl_qp_num - qm->qp_num) / num_vfs;
 895         if (vfq_num >= qm->max_qp_num)
 896                 return qm->max_qp_num;
 897
 898         remain_q_num = (qm->ctrl_qp_num - qm->qp_num) % num_vfs;
 899         if (vfq_num + remain_q_num <= qm->max_qp_num)
 900                 return fun_num == num_vfs ? vfq_num + remain_q_num : vfq_num;
 901
 902         /*
 903          * if vfq_num + remain_q_num > max_qp_num, the last VFs,
 904          * each with one more queue.
 905          */
 906         return fun_num + remain_q_num > num_vfs ? vfq_num + 1 : vfq_num;
 907 }
 908
 909 static struct hisi_qm *file_to_qm(struct debugfs_file *file)
 910 {
 911         struct qm_debug *debug = file->debug;
 912
 913         return container_of(debug, struct hisi_qm, debug);
 914 }
 915
 916 static u32 current_q_read(struct debugfs_file *file)
 917 {
 918         struct hisi_qm *qm = file_to_qm(file);
 919
 920         return readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) >> QM_DFX_QN_SHIFT;
 921 }
 922
 923 static int current_q_write(struct debugfs_file *file, u32 val)
 924 {
 925         struct hisi_qm *qm = file_to_qm(file);
 926         u32 tmp;
 927
 928         if (val >= qm->debug.curr_qm_qp_num)
 929                 return -EINVAL;
 930
 931         tmp = val << QM_DFX_QN_SHIFT |
 932               (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_FUN_MASK);
 933         writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
 934
 935         tmp = val << QM_DFX_QN_SHIFT |
 936               (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_FUN_MASK);
 937         writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
 938
 939         return 0;
 940 }
 941
 942 static u32 clear_enable_read(struct debugfs_file *file)
 943 {
 944         struct hisi_qm *qm = file_to_qm(file);
 945
 946         return readl(qm->io_base + QM_DFX_CNT_CLR_CE);
 947 }
 948
 949 /* rd_clr_ctrl 1 enable read clear, otherwise 0 disable it */
 950 static int clear_enable_write(struct debugfs_file *file, u32 rd_clr_ctrl)
 951 {
 952         struct hisi_qm *qm = file_to_qm(file);
 953
 954         if (rd_clr_ctrl > 1)
 955                 return -EINVAL;
 956
 957         writel(rd_clr_ctrl, qm->io_base + QM_DFX_CNT_CLR_CE);
 958
 959         return 0;
 960 }
 961
 962 static u32 current_qm_read(struct debugfs_file *file)
 963 {
 964         struct hisi_qm *qm = file_to_qm(file);
 965
 966         return readl(qm->io_base + QM_DFX_MB_CNT_VF);
 967 }
 968
 969 static int current_qm_write(struct debugfs_file *file, u32 val)
 970 {
 971         struct hisi_qm *qm = file_to_qm(file);
 972         u32 tmp;
 973
 974         if (val > qm->vfs_num)
 975                 return -EINVAL;
 976
 977         /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */
 978         if (!val)
 979                 qm->debug.curr_qm_qp_num = qm->qp_num;
 980         else
 981                 qm->debug.curr_qm_qp_num = qm_get_vf_qp_num(qm, val);
 982
 983         writel(val, qm->io_base + QM_DFX_MB_CNT_VF);
 984         writel(val, qm->io_base + QM_DFX_DB_CNT_VF);
 985
 986         tmp = val |
 987               (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK);
 988         writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
 989
 990         tmp = val |
 991               (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK);
 992         writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
 993
 994         return 0;
 995 }
 996
 997 static ssize_t qm_debug_read(struct file *filp, char __user *buf,
 998                              size_t count, loff_t *pos)
 999 {
1000         struct debugfs_file *file = filp->private_data;
1001         enum qm_debug_file index = file->index;
1002         char tbuf[QM_DBG_TMP_BUF_LEN];
1003         u32 val;
1004         int ret;
1005
1006         mutex_lock(&file->lock);
1007         switch (index) {
1008         case CURRENT_QM:
1009                 val = current_qm_read(file);
1010                 break;
1011         case CURRENT_Q:
1012                 val = current_q_read(file);
1013                 break;
1014         case CLEAR_ENABLE:
1015                 val = clear_enable_read(file);
1016                 break;
1017         default:
1018                 mutex_unlock(&file->lock);
1019                 return -EINVAL;
1020         }
1021         mutex_unlock(&file->lock);
1022
1023         ret = scnprintf(tbuf, QM_DBG_TMP_BUF_LEN, "%u\n", val);
1024         return simple_read_from_buffer(buf, count, pos, tbuf, ret);
1025 }
1026
1027 static ssize_t qm_debug_write(struct file *filp, const char __user *buf,
1028                               size_t count, loff_t *pos)
1029 {
1030         struct debugfs_file *file = filp->private_data;
1031         enum qm_debug_file index = file->index;
1032         unsigned long val;
1033         char tbuf[QM_DBG_TMP_BUF_LEN];
1034         int len, ret;
1035
1036         if (*pos != 0)
1037                 return 0;
1038
1039         if (count >= QM_DBG_TMP_BUF_LEN)
1040                 return -ENOSPC;
1041
1042         len = simple_write_to_buffer(tbuf, QM_DBG_TMP_BUF_LEN - 1, pos, buf,
1043                                      count);
1044         if (len < 0)
1045                 return len;
1046
1047         tbuf[len] = '\0';
1048         if (kstrtoul(tbuf, 0, &val))
1049                 return -EFAULT;
1050
1051         mutex_lock(&file->lock);
1052         switch (index) {
1053         case CURRENT_QM:
1054                 ret = current_qm_write(file, val);
1055                 break;
1056         case CURRENT_Q:
1057                 ret = current_q_write(file, val);
1058                 break;
1059         case CLEAR_ENABLE:
1060                 ret = clear_enable_write(file, val);
1061                 break;
1062         default:
1063                 ret = -EINVAL;
1064         }
1065         mutex_unlock(&file->lock);
1066
1067         if (ret)
1068                 return ret;
1069
1070         return count;
1071 }
1072
1073 static const struct file_operations qm_debug_fops = {
1074         .owner = THIS_MODULE,
1075         .open = simple_open,
1076         .read = qm_debug_read,
1077         .write = qm_debug_write,
1078 };
1079
1080 struct qm_dfx_registers {
1081         char  *reg_name;
1082         u64   reg_offset;
1083 };
1084
1085 #define CNT_CYC_REGS_NUM                10
1086 static struct qm_dfx_registers qm_dfx_regs[] = {
1087         /* XXX_CNT are reading clear register */
1088         {"QM_ECC_1BIT_CNT               ",  0x104000ull},
1089         {"QM_ECC_MBIT_CNT               ",  0x104008ull},
1090         {"QM_DFX_MB_CNT                 ",  0x104018ull},
1091         {"QM_DFX_DB_CNT                 ",  0x104028ull},
1092         {"QM_DFX_SQE_CNT                ",  0x104038ull},
1093         {"QM_DFX_CQE_CNT                ",  0x104048ull},
1094         {"QM_DFX_SEND_SQE_TO_ACC_CNT    ",  0x104050ull},
1095         {"QM_DFX_WB_SQE_FROM_ACC_CNT    ",  0x104058ull},
1096         {"QM_DFX_ACC_FINISH_CNT         ",  0x104060ull},
1097         {"QM_DFX_CQE_ERR_CNT            ",  0x1040b4ull},
1098         {"QM_DFX_FUNS_ACTIVE_ST         ",  0x200ull},
1099         {"QM_ECC_1BIT_INF               ",  0x104004ull},
1100         {"QM_ECC_MBIT_INF               ",  0x10400cull},
1101         {"QM_DFX_ACC_RDY_VLD0           ",  0x1040a0ull},
1102         {"QM_DFX_ACC_RDY_VLD1           ",  0x1040a4ull},
1103         {"QM_DFX_AXI_RDY_VLD            ",  0x1040a8ull},
1104         {"QM_DFX_FF_ST0                 ",  0x1040c8ull},
1105         {"QM_DFX_FF_ST1                 ",  0x1040ccull},
1106         {"QM_DFX_FF_ST2                 ",  0x1040d0ull},
1107         {"QM_DFX_FF_ST3                 ",  0x1040d4ull},
1108         {"QM_DFX_FF_ST4                 ",  0x1040d8ull},
1109         {"QM_DFX_FF_ST5                 ",  0x1040dcull},
1110         {"QM_DFX_FF_ST6                 ",  0x1040e0ull},
1111         {"QM_IN_IDLE_ST                 ",  0x1040e4ull},
1112         { NULL, 0}
1113 };
1114
1115 static struct qm_dfx_registers qm_vf_dfx_regs[] = {
1116         {"QM_DFX_FUNS_ACTIVE_ST         ",  0x200ull},
1117         { NULL, 0}
1118 };
1119
1120 static int qm_regs_show(struct seq_file *s, void *unused)
1121 {
1122         struct hisi_qm *qm = s->private;
1123         struct qm_dfx_registers *regs;
1124         u32 val;
1125
1126         if (qm->fun_type == QM_HW_PF)
1127                 regs = qm_dfx_regs;
1128         else
1129                 regs = qm_vf_dfx_regs;
1130
1131         while (regs->reg_name) {
1132                 val = readl(qm->io_base + regs->reg_offset);
1133                 seq_printf(s, "%s= 0x%08x\n", regs->reg_name, val);
1134                 regs++;
1135         }
1136
1137         return 0;
1138 }
1139
1140 DEFINE_SHOW_ATTRIBUTE(qm_regs);
1141
1142 static ssize_t qm_cmd_read(struct file *filp, char __user *buffer,
1143                            size_t count, loff_t *pos)
1144 {
1145         char buf[QM_DBG_READ_LEN];
1146         int len;
1147
1148         len = scnprintf(buf, QM_DBG_READ_LEN, "%s\n",
1149                         "Please echo help to cmd to get help information");
1150
1151         return simple_read_from_buffer(buffer, count, pos, buf, len);
1152 }
1153
1154 static void *qm_ctx_alloc(struct hisi_qm *qm, size_t ctx_size,
1155                           dma_addr_t *dma_addr)
1156 {
1157         struct device *dev = &qm->pdev->dev;
1158         void *ctx_addr;
1159
1160         ctx_addr = kzalloc(ctx_size, GFP_KERNEL);
1161         if (!ctx_addr)
1162                 return ERR_PTR(-ENOMEM);
1163
1164         *dma_addr = dma_map_single(dev, ctx_addr, ctx_size, DMA_FROM_DEVICE);
1165         if (dma_mapping_error(dev, *dma_addr)) {
1166                 dev_err(dev, "DMA mapping error!\n");
1167                 kfree(ctx_addr);
1168                 return ERR_PTR(-ENOMEM);
1169         }
1170
1171         return ctx_addr;
1172 }
1173
1174 static void qm_ctx_free(struct hisi_qm *qm, size_t ctx_size,
1175                         const void *ctx_addr, dma_addr_t *dma_addr)
1176 {
1177         struct device *dev = &qm->pdev->dev;
1178
1179         dma_unmap_single(dev, *dma_addr, ctx_size, DMA_FROM_DEVICE);
1180         kfree(ctx_addr);
1181 }
1182
1183 static int dump_show(struct hisi_qm *qm, void *info,
1184                      unsigned int info_size, char *info_name)
1185 {
1186         struct device *dev = &qm->pdev->dev;
1187         u8 *info_buf, *info_curr = info;
1188         u32 i;
1189 #define BYTE_PER_DW     4
1190
1191         info_buf = kzalloc(info_size, GFP_KERNEL);
1192         if (!info_buf)
1193                 return -ENOMEM;
1194
1195         for (i = 0; i < info_size; i++, info_curr++) {
1196                 if (i % BYTE_PER_DW == 0)
1197                         info_buf[i + 3UL] = *info_curr;
1198                 else if (i % BYTE_PER_DW == 1)
1199                         info_buf[i + 1UL] = *info_curr;
1200                 else if (i % BYTE_PER_DW == 2)
1201                         info_buf[i - 1] = *info_curr;
1202                 else if (i % BYTE_PER_DW == 3)
1203                         info_buf[i - 3] = *info_curr;
1204         }
1205
1206         dev_info(dev, "%s DUMP\n", info_name);
1207         for (i = 0; i < info_size; i += BYTE_PER_DW) {
1208                 pr_info("DW%u: %02X%02X %02X%02X\n", i / BYTE_PER_DW,
1209                         info_buf[i], info_buf[i + 1UL],
1210                         info_buf[i + 2UL], info_buf[i + 3UL]);
1211         }
1212
1213         kfree(info_buf);
1214
1215         return 0;
1216 }
1217
1218 static int qm_dump_sqc_raw(struct hisi_qm *qm, dma_addr_t dma_addr, u16 qp_id)
1219 {
1220         return qm_mb(qm, QM_MB_CMD_SQC, dma_addr, qp_id, 1);
1221 }
1222
1223 static int qm_dump_cqc_raw(struct hisi_qm *qm, dma_addr_t dma_addr, u16 qp_id)
1224 {
1225         return qm_mb(qm, QM_MB_CMD_CQC, dma_addr, qp_id, 1);
1226 }
1227
1228 static int qm_sqc_dump(struct hisi_qm *qm, const char *s)
1229 {
1230         struct device *dev = &qm->pdev->dev;
1231         struct qm_sqc *sqc, *sqc_curr;
1232         dma_addr_t sqc_dma;
1233         u32 qp_id;
1234         int ret;
1235
1236         if (!s)
1237                 return -EINVAL;
1238
1239         ret = kstrtou32(s, 0, &qp_id);
1240         if (ret || qp_id >= qm->qp_num) {
1241                 dev_err(dev, "Please input qp num (0-%u)", qm->qp_num - 1);
1242                 return -EINVAL;
1243         }
1244
1245         sqc = qm_ctx_alloc(qm, sizeof(*sqc), &sqc_dma);
1246         if (IS_ERR(sqc))
1247                 return PTR_ERR(sqc);
1248
1249         ret = qm_dump_sqc_raw(qm, sqc_dma, qp_id);
1250         if (ret) {
1251                 down_read(&qm->qps_lock);
1252                 if (qm->sqc) {
1253                         sqc_curr = qm->sqc + qp_id;
1254
1255                         ret = dump_show(qm, sqc_curr, sizeof(*sqc),
1256                                         "SOFT SQC");
1257                         if (ret)
1258                                 dev_info(dev, "Show soft sqc failed!\n");
1259                 }
1260                 up_read(&qm->qps_lock);
1261
1262                 goto err_free_ctx;
1263         }
1264
1265         ret = dump_show(qm, sqc, sizeof(*sqc), "SQC");
1266         if (ret)
1267                 dev_info(dev, "Show hw sqc failed!\n");
1268
1269 err_free_ctx:
1270         qm_ctx_free(qm, sizeof(*sqc), sqc, &sqc_dma);
1271         return ret;
1272 }
1273
1274 static int qm_cqc_dump(struct hisi_qm *qm, const char *s)
1275 {
1276         struct device *dev = &qm->pdev->dev;
1277         struct qm_cqc *cqc, *cqc_curr;
1278         dma_addr_t cqc_dma;
1279         u32 qp_id;
1280         int ret;
1281
1282         if (!s)
1283                 return -EINVAL;
1284
1285         ret = kstrtou32(s, 0, &qp_id);
1286         if (ret || qp_id >= qm->qp_num) {
1287                 dev_err(dev, "Please input qp num (0-%u)", qm->qp_num - 1);
1288                 return -EINVAL;
1289         }
1290
1291         cqc = qm_ctx_alloc(qm, sizeof(*cqc), &cqc_dma);
1292         if (IS_ERR(cqc))
1293                 return PTR_ERR(cqc);
1294
1295         ret = qm_dump_cqc_raw(qm, cqc_dma, qp_id);
1296         if (ret) {
1297                 down_read(&qm->qps_lock);
1298                 if (qm->cqc) {
1299                         cqc_curr = qm->cqc + qp_id;
1300
1301                         ret = dump_show(qm, cqc_curr, sizeof(*cqc),
1302                                         "SOFT CQC");
1303                         if (ret)
1304                                 dev_info(dev, "Show soft cqc failed!\n");
1305                 }
1306                 up_read(&qm->qps_lock);
1307
1308                 goto err_free_ctx;
1309         }
1310
1311         ret = dump_show(qm, cqc, sizeof(*cqc), "CQC");
1312         if (ret)
1313                 dev_info(dev, "Show hw cqc failed!\n");
1314
1315 err_free_ctx:
1316         qm_ctx_free(qm, sizeof(*cqc), cqc, &cqc_dma);
1317         return ret;
1318 }
1319
1320 static int qm_eqc_aeqc_dump(struct hisi_qm *qm, char *s, size_t size,
1321                             int cmd, char *name)
1322 {
1323         struct device *dev = &qm->pdev->dev;
1324         dma_addr_t xeqc_dma;
1325         void *xeqc;
1326         int ret;
1327
1328         if (strsep(&s, " ")) {
1329                 dev_err(dev, "Please do not input extra characters!\n");
1330                 return -EINVAL;
1331         }
1332
1333         xeqc = qm_ctx_alloc(qm, size, &xeqc_dma);
1334         if (IS_ERR(xeqc))
1335                 return PTR_ERR(xeqc);
1336
1337         ret = qm_mb(qm, cmd, xeqc_dma, 0, 1);
1338         if (ret)
1339                 goto err_free_ctx;
1340
1341         ret = dump_show(qm, xeqc, size, name);
1342         if (ret)
1343                 dev_info(dev, "Show hw %s failed!\n", name);
1344
1345 err_free_ctx:
1346         qm_ctx_free(qm, size, xeqc, &xeqc_dma);
1347         return ret;
1348 }
1349
1350 static int q_dump_param_parse(struct hisi_qm *qm, char *s,
1351                               u32 *e_id, u32 *q_id)
1352 {
1353         struct device *dev = &qm->pdev->dev;
1354         unsigned int qp_num = qm->qp_num;
1355         char *presult;
1356         int ret;
1357
1358         presult = strsep(&s, " ");
1359         if (!presult) {
1360                 dev_err(dev, "Please input qp number!\n");
1361                 return -EINVAL;
1362         }
1363
1364         ret = kstrtou32(presult, 0, q_id);
1365         if (ret || *q_id >= qp_num) {
1366                 dev_err(dev, "Please input qp num (0-%u)", qp_num - 1);
1367                 return -EINVAL;
1368         }
1369
1370         presult = strsep(&s, " ");
1371         if (!presult) {
1372                 dev_err(dev, "Please input sqe number!\n");
1373                 return -EINVAL;
1374         }
1375
1376         ret = kstrtou32(presult, 0, e_id);
1377         if (ret || *e_id >= QM_Q_DEPTH) {
1378                 dev_err(dev, "Please input sqe num (0-%d)", QM_Q_DEPTH - 1);
1379                 return -EINVAL;
1380         }
1381
1382         if (strsep(&s, " ")) {
1383                 dev_err(dev, "Please do not input extra characters!\n");
1384                 return -EINVAL;
1385         }
1386
1387         return 0;
1388 }
1389
1390 static int qm_sq_dump(struct hisi_qm *qm, char *s)
1391 {
1392         struct device *dev = &qm->pdev->dev;
1393         void *sqe, *sqe_curr;
1394         struct hisi_qp *qp;
1395         u32 qp_id, sqe_id;
1396         int ret;
1397
1398         ret = q_dump_param_parse(qm, s, &sqe_id, &qp_id);
1399         if (ret)
1400                 return ret;
1401
1402         sqe = kzalloc(qm->sqe_size * QM_Q_DEPTH, GFP_KERNEL);
1403         if (!sqe)
1404                 return -ENOMEM;
1405
1406         qp = &qm->qp_array[qp_id];
1407         memcpy(sqe, qp->sqe, qm->sqe_size * QM_Q_DEPTH);
1408         sqe_curr = sqe + (u32)(sqe_id * qm->sqe_size);
1409         memset(sqe_curr + qm->debug.sqe_mask_offset, QM_SQE_ADDR_MASK,
1410                qm->debug.sqe_mask_len);
1411
1412         ret = dump_show(qm, sqe_curr, qm->sqe_size, "SQE");
1413         if (ret)
1414                 dev_info(dev, "Show sqe failed!\n");
1415
1416         kfree(sqe);
1417
1418         return ret;
1419 }
1420
1421 static int qm_cq_dump(struct hisi_qm *qm, char *s)
1422 {
1423         struct device *dev = &qm->pdev->dev;
1424         struct qm_cqe *cqe_curr;
1425         struct hisi_qp *qp;
1426         u32 qp_id, cqe_id;
1427         int ret;
1428
1429         ret = q_dump_param_parse(qm, s, &cqe_id, &qp_id);
1430         if (ret)
1431                 return ret;
1432
1433         qp = &qm->qp_array[qp_id];
1434         cqe_curr = qp->cqe + cqe_id;
1435         ret = dump_show(qm, cqe_curr, sizeof(struct qm_cqe), "CQE");
1436         if (ret)
1437                 dev_info(dev, "Show cqe failed!\n");
1438
1439         return ret;
1440 }
1441
1442 static int qm_eq_aeq_dump(struct hisi_qm *qm, const char *s,
1443                           size_t size, char *name)
1444 {
1445         struct device *dev = &qm->pdev->dev;
1446         void *xeqe;
1447         u32 xeqe_id;
1448         int ret;
1449
1450         if (!s)
1451                 return -EINVAL;
1452
1453         ret = kstrtou32(s, 0, &xeqe_id);
1454         if (ret)
1455                 return -EINVAL;
1456
1457         if (!strcmp(name, "EQE") && xeqe_id >= QM_EQ_DEPTH) {
1458                 dev_err(dev, "Please input eqe num (0-%d)", QM_EQ_DEPTH - 1);
1459                 return -EINVAL;
1460         } else if (!strcmp(name, "AEQE") && xeqe_id >= QM_Q_DEPTH) {
1461                 dev_err(dev, "Please input aeqe num (0-%d)", QM_Q_DEPTH - 1);
1462                 return -EINVAL;
1463         }
1464
1465         down_read(&qm->qps_lock);
1466
1467         if (qm->eqe && !strcmp(name, "EQE")) {
1468                 xeqe = qm->eqe + xeqe_id;
1469         } else if (qm->aeqe && !strcmp(name, "AEQE")) {
1470                 xeqe = qm->aeqe + xeqe_id;
1471         } else {
1472                 ret = -EINVAL;
1473                 goto err_unlock;
1474         }
1475
1476         ret = dump_show(qm, xeqe, size, name);
1477         if (ret)
1478                 dev_info(dev, "Show %s failed!\n", name);
1479
1480 err_unlock:
1481         up_read(&qm->qps_lock);
1482         return ret;
1483 }
1484
1485 static int qm_dbg_help(struct hisi_qm *qm, char *s)
1486 {
1487         struct device *dev = &qm->pdev->dev;
1488
1489         if (strsep(&s, " ")) {
1490                 dev_err(dev, "Please do not input extra characters!\n");
1491                 return -EINVAL;
1492         }
1493
1494         dev_info(dev, "available commands:\n");
1495         dev_info(dev, "sqc <num>\n");
1496         dev_info(dev, "cqc <num>\n");
1497         dev_info(dev, "eqc\n");
1498         dev_info(dev, "aeqc\n");
1499         dev_info(dev, "sq <num> <e>\n");
1500         dev_info(dev, "cq <num> <e>\n");
1501         dev_info(dev, "eq <e>\n");
1502         dev_info(dev, "aeq <e>\n");
1503
1504         return 0;
1505 }
1506
1507 static int qm_cmd_write_dump(struct hisi_qm *qm, const char *cmd_buf)
1508 {
1509         struct device *dev = &qm->pdev->dev;
1510         char *presult, *s, *s_tmp;
1511         int ret;
1512
1513         s = kstrdup(cmd_buf, GFP_KERNEL);
1514         if (!s)
1515                 return -ENOMEM;
1516
1517         s_tmp = s;
1518         presult = strsep(&s, " ");
1519         if (!presult) {
1520                 ret = -EINVAL;
1521                 goto err_buffer_free;
1522         }
1523
1524         if (!strcmp(presult, "sqc"))
1525                 ret = qm_sqc_dump(qm, s);
1526         else if (!strcmp(presult, "cqc"))
1527                 ret = qm_cqc_dump(qm, s);
1528         else if (!strcmp(presult, "eqc"))
1529                 ret = qm_eqc_aeqc_dump(qm, s, sizeof(struct qm_eqc),
1530                                        QM_MB_CMD_EQC, "EQC");
1531         else if (!strcmp(presult, "aeqc"))
1532                 ret = qm_eqc_aeqc_dump(qm, s, sizeof(struct qm_aeqc),
1533                                        QM_MB_CMD_AEQC, "AEQC");
1534         else if (!strcmp(presult, "sq"))
1535                 ret = qm_sq_dump(qm, s);
1536         else if (!strcmp(presult, "cq"))
1537                 ret = qm_cq_dump(qm, s);
1538         else if (!strcmp(presult, "eq"))
1539                 ret = qm_eq_aeq_dump(qm, s, sizeof(struct qm_eqe), "EQE");
1540         else if (!strcmp(presult, "aeq"))
1541                 ret = qm_eq_aeq_dump(qm, s, sizeof(struct qm_aeqe), "AEQE");
1542         else if (!strcmp(presult, "help"))
1543                 ret = qm_dbg_help(qm, s);
1544         else
1545                 ret = -EINVAL;
1546
1547         if (ret)
1548                 dev_info(dev, "Please echo help\n");
1549
1550 err_buffer_free:
1551         kfree(s_tmp);
1552
1553         return ret;
1554 }
1555
1556 static ssize_t qm_cmd_write(struct file *filp, const char __user *buffer,
1557                             size_t count, loff_t *pos)
1558 {
1559         struct hisi_qm *qm = filp->private_data;
1560         char *cmd_buf, *cmd_buf_tmp;
1561         int ret;
1562
1563         if (*pos)
1564                 return 0;
1565
1566         /* Judge if the instance is being reset. */
1567         if (unlikely(atomic_read(&qm->status.flags) == QM_STOP))
1568                 return 0;
1569
1570         if (count > QM_DBG_WRITE_LEN)
1571                 return -ENOSPC;
1572
1573         cmd_buf = kzalloc(count + 1, GFP_KERNEL);
1574         if (!cmd_buf)
1575                 return -ENOMEM;
1576
1577         if (copy_from_user(cmd_buf, buffer, count)) {
1578                 kfree(cmd_buf);
1579                 return -EFAULT;
1580         }
1581
1582         cmd_buf[count] = '\0';
1583
1584         cmd_buf_tmp = strchr(cmd_buf, '\n');
1585         if (cmd_buf_tmp) {
1586                 *cmd_buf_tmp = '\0';
1587                 count = cmd_buf_tmp - cmd_buf + 1;
1588         }
1589
1590         ret = qm_cmd_write_dump(qm, cmd_buf);
1591         if (ret) {
1592                 kfree(cmd_buf);
1593                 return ret;
1594         }
1595
1596         kfree(cmd_buf);
1597
1598         return count;
1599 }
1600
1601 static const struct file_operations qm_cmd_fops = {
1602         .owner = THIS_MODULE,
1603         .open = simple_open,
1604         .read = qm_cmd_read,
1605         .write = qm_cmd_write,
1606 };
1607
1608 static void qm_create_debugfs_file(struct hisi_qm *qm, struct dentry *dir,
1609                                    enum qm_debug_file index)
1610 {
1611         struct debugfs_file *file = qm->debug.files + index;
1612
1613         debugfs_create_file(qm_debug_file_name[index], 0600, dir, file,
1614                             &qm_debug_fops);
1615
1616         file->index = index;
1617         mutex_init(&file->lock);
1618         file->debug = &qm->debug;
1619 }
1620
1621 static void qm_hw_error_init_v1(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe)
1622 {
1623         writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK);
1624 }
1625
1626 static void qm_hw_error_init_v2(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe)
1627 {
1628         u32 irq_enable = ce | nfe | fe;
1629         u32 irq_unmask = ~irq_enable;
1630
1631         qm->error_mask = ce | nfe | fe;
1632
1633         /* clear QM hw residual error source */
1634         writel(QM_ABNORMAL_INT_SOURCE_CLR,
1635                qm->io_base + QM_ABNORMAL_INT_SOURCE);
1636
1637         /* configure error type */
1638         writel(ce, qm->io_base + QM_RAS_CE_ENABLE);
1639         writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD);
1640         writel(nfe, qm->io_base + QM_RAS_NFE_ENABLE);
1641         writel(fe, qm->io_base + QM_RAS_FE_ENABLE);
1642
1643         irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1644         writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK);
1645 }
1646
1647 static void qm_hw_error_uninit_v2(struct hisi_qm *qm)
1648 {
1649         writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK);
1650 }
1651
1652 static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status)
1653 {
1654         const struct hisi_qm_hw_error *err;
1655         struct device *dev = &qm->pdev->dev;
1656         u32 reg_val, type, vf_num;
1657         int i;
1658
1659         for (i = 0; i < ARRAY_SIZE(qm_hw_error); i++) {
1660                 err = &qm_hw_error[i];
1661                 if (!(err->int_msk & error_status))
1662                         continue;
1663
1664                 dev_err(dev, "%s [error status=0x%x] found\n",
1665                         err->msg, err->int_msk);
1666
1667                 if (err->int_msk & QM_DB_TIMEOUT) {
1668                         reg_val = readl(qm->io_base + QM_ABNORMAL_INF01);
1669                         type = (reg_val & QM_DB_TIMEOUT_TYPE) >>
1670                                QM_DB_TIMEOUT_TYPE_SHIFT;
1671                         vf_num = reg_val & QM_DB_TIMEOUT_VF;
1672                         dev_err(dev, "qm %s doorbell timeout in function %u\n",
1673                                 qm_db_timeout[type], vf_num);
1674                 } else if (err->int_msk & QM_OF_FIFO_OF) {
1675                         reg_val = readl(qm->io_base + QM_ABNORMAL_INF00);
1676                         type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >>
1677                                QM_FIFO_OVERFLOW_TYPE_SHIFT;
1678                         vf_num = reg_val & QM_FIFO_OVERFLOW_VF;
1679
1680                         if (type < ARRAY_SIZE(qm_fifo_overflow))
1681                                 dev_err(dev, "qm %s fifo overflow in function %u\n",
1682                                         qm_fifo_overflow[type], vf_num);
1683                         else
1684                                 dev_err(dev, "unknown error type\n");
1685                 }
1686         }
1687 }
1688
1689 static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm)
1690 {
1691         u32 error_status, tmp, val;
1692
1693         /* read err sts */
1694         tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
1695         error_status = qm->error_mask & tmp;
1696
1697         if (error_status) {
1698                 if (error_status & QM_ECC_MBIT)
1699                         qm->err_status.is_qm_ecc_mbit = true;
1700
1701                 qm_log_hw_error(qm, error_status);
1702                 val = error_status | QM_DB_RANDOM_INVALID | QM_BASE_CE;
1703                 /* ce error does not need to be reset */
1704                 if (val == (QM_DB_RANDOM_INVALID | QM_BASE_CE)) {
1705                         writel(error_status, qm->io_base +
1706                                QM_ABNORMAL_INT_SOURCE);
1707                         writel(qm->err_info.nfe,
1708                                qm->io_base + QM_RAS_NFE_ENABLE);
1709                         return ACC_ERR_RECOVERED;
1710                 }
1711
1712                 return ACC_ERR_NEED_RESET;
1713         }
1714
1715         return ACC_ERR_RECOVERED;
1716 }
1717
1718 static int qm_stop_qp(struct hisi_qp *qp)
1719 {
1720         return qm_mb(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0);
1721 }
1722
1723 static const struct hisi_qm_hw_ops qm_hw_ops_v1 = {
1724         .qm_db = qm_db_v1,
1725         .get_irq_num = qm_get_irq_num_v1,
1726         .hw_error_init = qm_hw_error_init_v1,
1727 };
1728
1729 static const struct hisi_qm_hw_ops qm_hw_ops_v2 = {
1730         .get_vft = qm_get_vft_v2,
1731         .qm_db = qm_db_v2,
1732         .get_irq_num = qm_get_irq_num_v2,
1733         .hw_error_init = qm_hw_error_init_v2,
1734         .hw_error_uninit = qm_hw_error_uninit_v2,
1735         .hw_error_handle = qm_hw_error_handle_v2,
1736 };
1737
1738 static const struct hisi_qm_hw_ops qm_hw_ops_v3 = {
1739         .get_vft = qm_get_vft_v2,
1740         .qm_db = qm_db_v2,
1741         .get_irq_num = qm_get_irq_num_v2,
1742         .hw_error_init = qm_hw_error_init_v2,
1743         .hw_error_uninit = qm_hw_error_uninit_v2,
1744         .hw_error_handle = qm_hw_error_handle_v2,
1745         .stop_qp = qm_stop_qp,
1746 };
1747
1748 static void *qm_get_avail_sqe(struct hisi_qp *qp)
1749 {
1750         struct hisi_qp_status *qp_status = &qp->qp_status;
1751         u16 sq_tail = qp_status->sq_tail;
1752
1753         if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH - 1))
1754                 return NULL;
1755
1756         return qp->sqe + sq_tail * qp->qm->sqe_size;
1757 }
1758
1759 static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type)
1760 {
1761         struct device *dev = &qm->pdev->dev;
1762         struct hisi_qp *qp;
1763         int qp_id;
1764
1765         if (!qm_qp_avail_state(qm, NULL, QP_INIT))
1766                 return ERR_PTR(-EPERM);
1767
1768         if (qm->qp_in_used == qm->qp_num) {
1769                 dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",
1770                                      qm->qp_num);
1771                 atomic64_inc(&qm->debug.dfx.create_qp_err_cnt);
1772                 return ERR_PTR(-EBUSY);
1773         }
1774
1775         qp_id = idr_alloc_cyclic(&qm->qp_idr, NULL, 0, qm->qp_num, GFP_ATOMIC);
1776         if (qp_id < 0) {
1777                 dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",
1778                                     qm->qp_num);
1779                 atomic64_inc(&qm->debug.dfx.create_qp_err_cnt);
1780                 return ERR_PTR(-EBUSY);
1781         }
1782
1783         qp = &qm->qp_array[qp_id];
1784
1785         memset(qp->cqe, 0, sizeof(struct qm_cqe) * QM_Q_DEPTH);
1786
1787         qp->event_cb = NULL;
1788         qp->req_cb = NULL;
1789         qp->qp_id = qp_id;
1790         qp->alg_type = alg_type;
1791         qp->is_in_kernel = true;
1792         qm->qp_in_used++;
1793         atomic_set(&qp->qp_status.flags, QP_INIT);
1794
1795         return qp;
1796 }
1797
1798 /**
1799  * hisi_qm_create_qp() - Create a queue pair from qm.
1800  * @qm: The qm we create a qp from.
1801  * @alg_type: Accelerator specific algorithm type in sqc.
1802  *
1803  * return created qp, -EBUSY if all qps in qm allocated, -ENOMEM if allocating
1804  * qp memory fails.
1805  */
1806 struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type)
1807 {
1808         struct hisi_qp *qp;
1809
1810         down_write(&qm->qps_lock);
1811         qp = qm_create_qp_nolock(qm, alg_type);
1812         up_write(&qm->qps_lock);
1813
1814         return qp;
1815 }
1816 EXPORT_SYMBOL_GPL(hisi_qm_create_qp);
1817
1818 /**
1819  * hisi_qm_release_qp() - Release a qp back to its qm.
1820  * @qp: The qp we want to release.
1821  *
1822  * This function releases the resource of a qp.
1823  */
1824 void hisi_qm_release_qp(struct hisi_qp *qp)
1825 {
1826         struct hisi_qm *qm = qp->qm;
1827
1828         down_write(&qm->qps_lock);
1829
1830         if (!qm_qp_avail_state(qm, qp, QP_CLOSE)) {
1831                 up_write(&qm->qps_lock);
1832                 return;
1833         }
1834
1835         qm->qp_in_used--;
1836         idr_remove(&qm->qp_idr, qp->qp_id);
1837
1838         up_write(&qm->qps_lock);
1839 }
1840 EXPORT_SYMBOL_GPL(hisi_qm_release_qp);
1841
1842 static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
1843 {
1844         struct hisi_qm *qm = qp->qm;
1845         struct device *dev = &qm->pdev->dev;
1846         enum qm_hw_ver ver = qm->ver;
1847         struct qm_sqc *sqc;
1848         dma_addr_t sqc_dma;
1849         int ret;
1850
1851         sqc = kzalloc(sizeof(struct qm_sqc), GFP_KERNEL);
1852         if (!sqc)
1853                 return -ENOMEM;
1854
1855         INIT_QC_COMMON(sqc, qp->sqe_dma, pasid);
1856         if (ver == QM_HW_V1) {
1857                 sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size));
1858                 sqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1);
1859         } else {
1860                 sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size));
1861                 sqc->w8 = 0; /* rand_qc */
1862         }
1863         sqc->cq_num = cpu_to_le16(qp_id);
1864         sqc->w13 = cpu_to_le16(QM_MK_SQC_W13(0, 1, qp->alg_type));
1865
1866         if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel)
1867                 sqc->w11 = cpu_to_le16(QM_QC_PASID_ENABLE <<
1868                                        QM_QC_PASID_ENABLE_SHIFT);
1869
1870         sqc_dma = dma_map_single(dev, sqc, sizeof(struct qm_sqc),
1871                                  DMA_TO_DEVICE);
1872         if (dma_mapping_error(dev, sqc_dma)) {
1873                 kfree(sqc);
1874                 return -ENOMEM;
1875         }
1876
1877         ret = qm_mb(qm, QM_MB_CMD_SQC, sqc_dma, qp_id, 0);
1878         dma_unmap_single(dev, sqc_dma, sizeof(struct qm_sqc), DMA_TO_DEVICE);
1879         kfree(sqc);
1880
1881         return ret;
1882 }
1883
1884 static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
1885 {
1886         struct hisi_qm *qm = qp->qm;
1887         struct device *dev = &qm->pdev->dev;
1888         enum qm_hw_ver ver = qm->ver;
1889         struct qm_cqc *cqc;
1890         dma_addr_t cqc_dma;
1891         int ret;
1892
1893         cqc = kzalloc(sizeof(struct qm_cqc), GFP_KERNEL);
1894         if (!cqc)
1895                 return -ENOMEM;
1896
1897         INIT_QC_COMMON(cqc, qp->cqe_dma, pasid);
1898         if (ver == QM_HW_V1) {
1899                 cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0,
1900                                                         QM_QC_CQE_SIZE));
1901                 cqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1);
1902         } else {
1903                 cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE));
1904                 cqc->w8 = 0; /* rand_qc */
1905         }
1906         cqc->dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT);
1907
1908         if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel)
1909                 cqc->w11 = cpu_to_le16(QM_QC_PASID_ENABLE);
1910
1911         cqc_dma = dma_map_single(dev, cqc, sizeof(struct qm_cqc),
1912                                  DMA_TO_DEVICE);
1913         if (dma_mapping_error(dev, cqc_dma)) {
1914                 kfree(cqc);
1915                 return -ENOMEM;
1916         }
1917
1918         ret = qm_mb(qm, QM_MB_CMD_CQC, cqc_dma, qp_id, 0);
1919         dma_unmap_single(dev, cqc_dma, sizeof(struct qm_cqc), DMA_TO_DEVICE);
1920         kfree(cqc);
1921
1922         return ret;
1923 }
1924
1925 static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
1926 {
1927         int ret;
1928
1929         qm_init_qp_status(qp);
1930
1931         ret = qm_sq_ctx_cfg(qp, qp_id, pasid);
1932         if (ret)
1933                 return ret;
1934
1935         return qm_cq_ctx_cfg(qp, qp_id, pasid);
1936 }
1937
1938 static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg)
1939 {
1940         struct hisi_qm *qm = qp->qm;
1941         struct device *dev = &qm->pdev->dev;
1942         int qp_id = qp->qp_id;
1943         u32 pasid = arg;
1944         int ret;
1945
1946         if (!qm_qp_avail_state(qm, qp, QP_START))
1947                 return -EPERM;
1948
1949         ret = qm_qp_ctx_cfg(qp, qp_id, pasid);
1950         if (ret)
1951                 return ret;
1952
1953         atomic_set(&qp->qp_status.flags, QP_START);
1954         dev_dbg(dev, "queue %d started\n", qp_id);
1955
1956         return 0;
1957 }
1958
1959 /**
1960  * hisi_qm_start_qp() - Start a qp into running.
1961  * @qp: The qp we want to start to run.
1962  * @arg: Accelerator specific argument.
1963  *
1964  * After this function, qp can receive request from user. Return 0 if
1965  * successful, Return -EBUSY if failed.
1966  */
1967 int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg)
1968 {
1969         struct hisi_qm *qm = qp->qm;
1970         int ret;
1971
1972         down_write(&qm->qps_lock);
1973         ret = qm_start_qp_nolock(qp, arg);
1974         up_write(&qm->qps_lock);
1975
1976         return ret;
1977 }
1978 EXPORT_SYMBOL_GPL(hisi_qm_start_qp);
1979
1980 /**
1981  * qp_stop_fail_cb() - call request cb.
1982  * @qp: stopped failed qp.
1983  *
1984  * Callback function should be called whether task completed or not.
1985  */
1986 static void qp_stop_fail_cb(struct hisi_qp *qp)
1987 {
1988         int qp_used = atomic_read(&qp->qp_status.used);
1989         u16 cur_tail = qp->qp_status.sq_tail;
1990         u16 cur_head = (cur_tail + QM_Q_DEPTH - qp_used) % QM_Q_DEPTH;
1991         struct hisi_qm *qm = qp->qm;
1992         u16 pos;
1993         int i;
1994
1995         for (i = 0; i < qp_used; i++) {
1996                 pos = (i + cur_head) % QM_Q_DEPTH;
1997                 qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos));
1998                 atomic_dec(&qp->qp_status.used);
1999         }
2000 }
2001
2002 /**
2003  * qm_drain_qp() - Drain a qp.
2004  * @qp: The qp we want to drain.
2005  *
2006  * Determine whether the queue is cleared by judging the tail pointers of
2007  * sq and cq.
2008  */
2009 static int qm_drain_qp(struct hisi_qp *qp)
2010 {
2011         size_t size = sizeof(struct qm_sqc) + sizeof(struct qm_cqc);
2012         struct hisi_qm *qm = qp->qm;
2013         struct device *dev = &qm->pdev->dev;
2014         struct qm_sqc *sqc;
2015         struct qm_cqc *cqc;
2016         dma_addr_t dma_addr;
2017         int ret = 0, i = 0;
2018         void *addr;
2019
2020         /*
2021          * No need to judge if ECC multi-bit error occurs because the
2022          * master OOO will be blocked.
2023          */
2024         if (qm->err_status.is_qm_ecc_mbit || qm->err_status.is_dev_ecc_mbit)
2025                 return 0;
2026
2027         /* Kunpeng930 supports drain qp by device */
2028         if (qm->ops->stop_qp) {
2029                 ret = qm->ops->stop_qp(qp);
2030                 if (ret)
2031                         dev_err(dev, "Failed to stop qp(%u)!\n", qp->qp_id);
2032                 return ret;
2033         }
2034
2035         addr = qm_ctx_alloc(qm, size, &dma_addr);
2036         if (IS_ERR(addr)) {
2037                 dev_err(dev, "Failed to alloc ctx for sqc and cqc!\n");
2038                 return -ENOMEM;
2039         }
2040
2041         while (++i) {
2042                 ret = qm_dump_sqc_raw(qm, dma_addr, qp->qp_id);
2043                 if (ret) {
2044                         dev_err_ratelimited(dev, "Failed to dump sqc!\n");
2045                         break;
2046                 }
2047                 sqc = addr;
2048
2049                 ret = qm_dump_cqc_raw(qm, (dma_addr + sizeof(struct qm_sqc)),
2050                                       qp->qp_id);
2051                 if (ret) {
2052                         dev_err_ratelimited(dev, "Failed to dump cqc!\n");
2053                         break;
2054                 }
2055                 cqc = addr + sizeof(struct qm_sqc);
2056
2057                 if ((sqc->tail == cqc->tail) &&
2058                     (QM_SQ_TAIL_IDX(sqc) == QM_CQ_TAIL_IDX(cqc)))
2059                         break;
2060
2061                 if (i == MAX_WAIT_COUNTS) {
2062                         dev_err(dev, "Fail to empty queue %u!\n", qp->qp_id);
2063                         ret = -EBUSY;
2064                         break;
2065                 }
2066
2067                 usleep_range(WAIT_PERIOD_US_MIN, WAIT_PERIOD_US_MAX);
2068         }
2069
2070         qm_ctx_free(qm, size, addr, &dma_addr);
2071
2072         return ret;
2073 }
2074
2075 static int qm_stop_qp_nolock(struct hisi_qp *qp)
2076 {
2077         struct device *dev = &qp->qm->pdev->dev;
2078         int ret;
2079
2080         /*
2081          * It is allowed to stop and release qp when reset, If the qp is
2082          * stopped when reset but still want to be released then, the
2083          * is_resetting flag should be set negative so that this qp will not
2084          * be restarted after reset.
2085          */
2086         if (atomic_read(&qp->qp_status.flags) == QP_STOP) {
2087                 qp->is_resetting = false;
2088                 return 0;
2089         }
2090
2091         if (!qm_qp_avail_state(qp->qm, qp, QP_STOP))
2092                 return -EPERM;
2093
2094         atomic_set(&qp->qp_status.flags, QP_STOP);
2095
2096         ret = qm_drain_qp(qp);
2097         if (ret)
2098                 dev_err(dev, "Failed to drain out data for stopping!\n");
2099
2100         if (qp->qm->wq)
2101                 flush_workqueue(qp->qm->wq);
2102         else
2103                 flush_work(&qp->qm->work);
2104
2105         if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used)))
2106                 qp_stop_fail_cb(qp);
2107
2108         dev_dbg(dev, "stop queue %u!", qp->qp_id);
2109
2110         return 0;
2111 }
2112
2113 /**
2114  * hisi_qm_stop_qp() - Stop a qp in qm.
2115  * @qp: The qp we want to stop.
2116  *
2117  * This function is reverse of hisi_qm_start_qp. Return 0 if successful.
2118  */
2119 int hisi_qm_stop_qp(struct hisi_qp *qp)
2120 {
2121         int ret;
2122
2123         down_write(&qp->qm->qps_lock);
2124         ret = qm_stop_qp_nolock(qp);
2125         up_write(&qp->qm->qps_lock);
2126
2127         return ret;
2128 }
2129 EXPORT_SYMBOL_GPL(hisi_qm_stop_qp);
2130
2131 /**
2132  * hisi_qp_send() - Queue up a task in the hardware queue.
2133  * @qp: The qp in which to put the message.
2134  * @msg: The message.
2135  *
2136  * This function will return -EBUSY if qp is currently full, and -EAGAIN
2137  * if qp related qm is resetting.
2138  *
2139  * Note: This function may run with qm_irq_thread and ACC reset at same time.
2140  *       It has no race with qm_irq_thread. However, during hisi_qp_send, ACC
2141  *       reset may happen, we have no lock here considering performance. This
2142  *       causes current qm_db sending fail or can not receive sended sqe. QM
2143  *       sync/async receive function should handle the error sqe. ACC reset
2144  *       done function should clear used sqe to 0.
2145  */
2146 int hisi_qp_send(struct hisi_qp *qp, const void *msg)
2147 {
2148         struct hisi_qp_status *qp_status = &qp->qp_status;
2149         u16 sq_tail = qp_status->sq_tail;
2150         u16 sq_tail_next = (sq_tail + 1) % QM_Q_DEPTH;
2151         void *sqe = qm_get_avail_sqe(qp);
2152
2153         if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP ||
2154                      atomic_read(&qp->qm->status.flags) == QM_STOP ||
2155                      qp->is_resetting)) {
2156                 dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n");
2157                 return -EAGAIN;
2158         }
2159
2160         if (!sqe)
2161                 return -EBUSY;
2162
2163         memcpy(sqe, msg, qp->qm->sqe_size);
2164
2165         qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0);
2166         atomic_inc(&qp->qp_status.used);
2167         qp_status->sq_tail = sq_tail_next;
2168
2169         return 0;
2170 }
2171 EXPORT_SYMBOL_GPL(hisi_qp_send);
2172
2173 static void hisi_qm_cache_wb(struct hisi_qm *qm)
2174 {
2175         unsigned int val;
2176
2177         if (qm->ver == QM_HW_V1)
2178                 return;
2179
2180         writel(0x1, qm->io_base + QM_CACHE_WB_START);
2181         if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE,
2182                                        val, val & BIT(0), POLL_PERIOD,
2183                                        POLL_TIMEOUT))
2184                 dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n");
2185 }
2186
2187 static void qm_qp_event_notifier(struct hisi_qp *qp)
2188 {
2189         wake_up_interruptible(&qp->uacce_q->wait);
2190 }
2191
2192 static int hisi_qm_get_available_instances(struct uacce_device *uacce)
2193 {
2194         return hisi_qm_get_free_qp_num(uacce->priv);
2195 }
2196
2197 static int hisi_qm_uacce_get_queue(struct uacce_device *uacce,
2198                                    unsigned long arg,
2199                                    struct uacce_queue *q)
2200 {
2201         struct hisi_qm *qm = uacce->priv;
2202         struct hisi_qp *qp;
2203         u8 alg_type = 0;
2204
2205         qp = hisi_qm_create_qp(qm, alg_type);
2206         if (IS_ERR(qp))
2207                 return PTR_ERR(qp);
2208
2209         q->priv = qp;
2210         q->uacce = uacce;
2211         qp->uacce_q = q;
2212         qp->event_cb = qm_qp_event_notifier;
2213         qp->pasid = arg;
2214         qp->is_in_kernel = false;
2215
2216         return 0;
2217 }
2218
2219 static void hisi_qm_uacce_put_queue(struct uacce_queue *q)
2220 {
2221         struct hisi_qp *qp = q->priv;
2222
2223         hisi_qm_cache_wb(qp->qm);
2224         hisi_qm_release_qp(qp);
2225 }
2226
2227 /* map sq/cq/doorbell to user space */
2228 static int hisi_qm_uacce_mmap(struct uacce_queue *q,
2229                               struct vm_area_struct *vma,
2230                               struct uacce_qfile_region *qfr)
2231 {
2232         struct hisi_qp *qp = q->priv;
2233         struct hisi_qm *qm = qp->qm;
2234         resource_size_t phys_base = qm->db_phys_base +
2235                                     qp->qp_id * qm->db_interval;
2236         size_t sz = vma->vm_end - vma->vm_start;
2237         struct pci_dev *pdev = qm->pdev;
2238         struct device *dev = &pdev->dev;
2239         unsigned long vm_pgoff;
2240         int ret;
2241
2242         switch (qfr->type) {
2243         case UACCE_QFRT_MMIO:
2244                 if (qm->ver == QM_HW_V1) {
2245                         if (sz > PAGE_SIZE * QM_DOORBELL_PAGE_NR)
2246                                 return -EINVAL;
2247                 } else if (qm->ver == QM_HW_V2 || !qm->use_db_isolation) {
2248                         if (sz > PAGE_SIZE * (QM_DOORBELL_PAGE_NR +
2249                             QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE))
2250                                 return -EINVAL;
2251                 } else {
2252                         if (sz > qm->db_interval)
2253                                 return -EINVAL;
2254                 }
2255
2256                 vma->vm_flags |= VM_IO;
2257
2258                 return remap_pfn_range(vma, vma->vm_start,
2259                                        phys_base >> PAGE_SHIFT,
2260                                        sz, pgprot_noncached(vma->vm_page_prot));
2261         case UACCE_QFRT_DUS:
2262                 if (sz != qp->qdma.size)
2263                         return -EINVAL;
2264
2265                 /*
2266                  * dma_mmap_coherent() requires vm_pgoff as 0
2267                  * restore vm_pfoff to initial value for mmap()
2268                  */
2269                 vm_pgoff = vma->vm_pgoff;
2270                 vma->vm_pgoff = 0;
2271                 ret = dma_mmap_coherent(dev, vma, qp->qdma.va,
2272                                         qp->qdma.dma, sz);
2273                 vma->vm_pgoff = vm_pgoff;
2274                 return ret;
2275
2276         default:
2277                 return -EINVAL;
2278         }
2279 }
2280
2281 static int hisi_qm_uacce_start_queue(struct uacce_queue *q)
2282 {
2283         struct hisi_qp *qp = q->priv;
2284
2285         return hisi_qm_start_qp(qp, qp->pasid);
2286 }
2287
2288 static void hisi_qm_uacce_stop_queue(struct uacce_queue *q)
2289 {
2290         hisi_qm_stop_qp(q->priv);
2291 }
2292
2293 static void qm_set_sqctype(struct uacce_queue *q, u16 type)
2294 {
2295         struct hisi_qm *qm = q->uacce->priv;
2296         struct hisi_qp *qp = q->priv;
2297
2298         down_write(&qm->qps_lock);
2299         qp->alg_type = type;
2300         up_write(&qm->qps_lock);
2301 }
2302
2303 static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd,
2304                                 unsigned long arg)
2305 {
2306         struct hisi_qp *qp = q->priv;
2307         struct hisi_qp_ctx qp_ctx;
2308
2309         if (cmd == UACCE_CMD_QM_SET_QP_CTX) {
2310                 if (copy_from_user(&qp_ctx, (void __user *)arg,
2311                                    sizeof(struct hisi_qp_ctx)))
2312                         return -EFAULT;
2313
2314                 if (qp_ctx.qc_type != 0 && qp_ctx.qc_type != 1)
2315                         return -EINVAL;
2316
2317                 qm_set_sqctype(q, qp_ctx.qc_type);
2318                 qp_ctx.id = qp->qp_id;
2319
2320                 if (copy_to_user((void __user *)arg, &qp_ctx,
2321                                  sizeof(struct hisi_qp_ctx)))
2322                         return -EFAULT;
2323         } else {
2324                 return -EINVAL;
2325         }
2326
2327         return 0;
2328 }
2329
2330 static const struct uacce_ops uacce_qm_ops = {
2331         .get_available_instances = hisi_qm_get_available_instances,
2332         .get_queue = hisi_qm_uacce_get_queue,
2333         .put_queue = hisi_qm_uacce_put_queue,
2334         .start_queue = hisi_qm_uacce_start_queue,
2335         .stop_queue = hisi_qm_uacce_stop_queue,
2336         .mmap = hisi_qm_uacce_mmap,
2337         .ioctl = hisi_qm_uacce_ioctl,
2338 };
2339
2340 static int qm_alloc_uacce(struct hisi_qm *qm)
2341 {
2342         struct pci_dev *pdev = qm->pdev;
2343         struct uacce_device *uacce;
2344         unsigned long mmio_page_nr;
2345         unsigned long dus_page_nr;
2346         struct uacce_interface interface = {
2347                 .flags = UACCE_DEV_SVA,
2348                 .ops = &uacce_qm_ops,
2349         };
2350         int ret;
2351
2352         ret = strscpy(interface.name, pdev->driver->name,
2353                       sizeof(interface.name));
2354         if (ret < 0)
2355                 return -ENAMETOOLONG;
2356
2357         uacce = uacce_alloc(&pdev->dev, &interface);
2358         if (IS_ERR(uacce))
2359                 return PTR_ERR(uacce);
2360
2361         if (uacce->flags & UACCE_DEV_SVA && qm->mode == UACCE_MODE_SVA) {
2362                 qm->use_sva = true;
2363         } else {
2364                 /* only consider sva case */
2365                 uacce_remove(uacce);
2366                 qm->uacce = NULL;
2367                 return -EINVAL;
2368         }
2369
2370         uacce->is_vf = pdev->is_virtfn;
2371         uacce->priv = qm;
2372         uacce->algs = qm->algs;
2373
2374         if (qm->ver == QM_HW_V1)
2375                 uacce->api_ver = HISI_QM_API_VER_BASE;
2376         else if (qm->ver == QM_HW_V2)
2377                 uacce->api_ver = HISI_QM_API_VER2_BASE;
2378         else
2379                 uacce->api_ver = HISI_QM_API_VER3_BASE;
2380
2381         if (qm->ver == QM_HW_V1)
2382                 mmio_page_nr = QM_DOORBELL_PAGE_NR;
2383         else if (qm->ver == QM_HW_V2 || !qm->use_db_isolation)
2384                 mmio_page_nr = QM_DOORBELL_PAGE_NR +
2385                         QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE;
2386         else
2387                 mmio_page_nr = qm->db_interval / PAGE_SIZE;
2388
2389         dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * QM_Q_DEPTH +
2390                        sizeof(struct qm_cqe) * QM_Q_DEPTH) >> PAGE_SHIFT;
2391
2392         uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr;
2393         uacce->qf_pg_num[UACCE_QFRT_DUS]  = dus_page_nr;
2394
2395         qm->uacce = uacce;
2396
2397         return 0;
2398 }
2399
2400 /**
2401  * qm_frozen() - Try to froze QM to cut continuous queue request. If
2402  * there is user on the QM, return failure without doing anything.
2403  * @qm: The qm needed to be fronzen.
2404  *
2405  * This function frozes QM, then we can do SRIOV disabling.
2406  */
2407 static int qm_frozen(struct hisi_qm *qm)
2408 {
2409         if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl))
2410                 return 0;
2411
2412         down_write(&qm->qps_lock);
2413
2414         if (!qm->qp_in_used) {
2415                 qm->qp_in_used = qm->qp_num;
2416                 up_write(&qm->qps_lock);
2417                 set_bit(QM_DRIVER_REMOVING, &qm->misc_ctl);
2418                 return 0;
2419         }
2420
2421         up_write(&qm->qps_lock);
2422
2423         return -EBUSY;
2424 }
2425
2426 static int qm_try_frozen_vfs(struct pci_dev *pdev,
2427                              struct hisi_qm_list *qm_list)
2428 {
2429         struct hisi_qm *qm, *vf_qm;
2430         struct pci_dev *dev;
2431         int ret = 0;
2432
2433         if (!qm_list || !pdev)
2434                 return -EINVAL;
2435
2436         /* Try to frozen all the VFs as disable SRIOV */
2437         mutex_lock(&qm_list->lock);
2438         list_for_each_entry(qm, &qm_list->list, list) {
2439                 dev = qm->pdev;
2440                 if (dev == pdev)
2441                         continue;
2442                 if (pci_physfn(dev) == pdev) {
2443                         vf_qm = pci_get_drvdata(dev);
2444                         ret = qm_frozen(vf_qm);
2445                         if (ret)
2446                                 goto frozen_fail;
2447                 }
2448         }
2449
2450 frozen_fail:
2451         mutex_unlock(&qm_list->lock);
2452
2453         return ret;
2454 }
2455
2456 /**
2457  * hisi_qm_wait_task_finish() - Wait until the task is finished
2458  * when removing the driver.
2459  * @qm: The qm needed to wait for the task to finish.
2460  * @qm_list: The list of all available devices.
2461  */
2462 void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
2463 {
2464         while (qm_frozen(qm) ||
2465                ((qm->fun_type == QM_HW_PF) &&
2466                qm_try_frozen_vfs(qm->pdev, qm_list))) {
2467                 msleep(WAIT_PERIOD);
2468         }
2469
2470         while (test_bit(QM_RST_SCHED, &qm->misc_ctl) ||
2471                test_bit(QM_RESETTING, &qm->misc_ctl))
2472                 msleep(WAIT_PERIOD);
2473
2474         udelay(REMOVE_WAIT_DELAY);
2475 }
2476 EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish);
2477
2478 /**
2479  * hisi_qm_get_free_qp_num() - Get free number of qp in qm.
2480  * @qm: The qm which want to get free qp.
2481  *
2482  * This function return free number of qp in qm.
2483  */
2484 int hisi_qm_get_free_qp_num(struct hisi_qm *qm)
2485 {
2486         int ret;
2487
2488         down_read(&qm->qps_lock);
2489         ret = qm->qp_num - qm->qp_in_used;
2490         up_read(&qm->qps_lock);
2491
2492         return ret;
2493 }
2494 EXPORT_SYMBOL_GPL(hisi_qm_get_free_qp_num);
2495
2496 static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num)
2497 {
2498         struct device *dev = &qm->pdev->dev;
2499         struct qm_dma *qdma;
2500         int i;
2501
2502         for (i = num - 1; i >= 0; i--) {
2503                 qdma = &qm->qp_array[i].qdma;
2504                 dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma);
2505         }
2506
2507         kfree(qm->qp_array);
2508 }
2509
2510 static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id)
2511 {
2512         struct device *dev = &qm->pdev->dev;
2513         size_t off = qm->sqe_size * QM_Q_DEPTH;
2514         struct hisi_qp *qp;
2515
2516         qp = &qm->qp_array[id];
2517         qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma,
2518                                          GFP_KERNEL);
2519         if (!qp->qdma.va)
2520                 return -ENOMEM;
2521
2522         qp->sqe = qp->qdma.va;
2523         qp->sqe_dma = qp->qdma.dma;
2524         qp->cqe = qp->qdma.va + off;
2525         qp->cqe_dma = qp->qdma.dma + off;
2526         qp->qdma.size = dma_size;
2527         qp->qm = qm;
2528         qp->qp_id = id;
2529
2530         return 0;
2531 }
2532
2533 static int hisi_qm_memory_init(struct hisi_qm *qm)
2534 {
2535         struct device *dev = &qm->pdev->dev;
2536         size_t qp_dma_size, off = 0;
2537         int i, ret = 0;
2538
2539 #define QM_INIT_BUF(qm, type, num) do { \
2540         (qm)->type = ((qm)->qdma.va + (off)); \
2541         (qm)->type##_dma = (qm)->qdma.dma + (off); \
2542         off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \
2543 } while (0)
2544
2545         idr_init(&qm->qp_idr);
2546         qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) +
2547                         QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) +
2548                         QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) +
2549                         QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num);
2550         qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma,
2551                                          GFP_ATOMIC);
2552         dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size);
2553         if (!qm->qdma.va)
2554                 return -ENOMEM;
2555
2556         QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH);
2557         QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH);
2558         QM_INIT_BUF(qm, sqc, qm->qp_num);
2559         QM_INIT_BUF(qm, cqc, qm->qp_num);
2560
2561         qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp), GFP_KERNEL);
2562         if (!qm->qp_array) {
2563                 ret = -ENOMEM;
2564                 goto err_alloc_qp_array;
2565         }
2566
2567         /* one more page for device or qp statuses */
2568         qp_dma_size = qm->sqe_size * QM_Q_DEPTH +
2569                       sizeof(struct qm_cqe) * QM_Q_DEPTH;
2570         qp_dma_size = PAGE_ALIGN(qp_dma_size);
2571         for (i = 0; i < qm->qp_num; i++) {
2572                 ret = hisi_qp_memory_init(qm, qp_dma_size, i);
2573                 if (ret)
2574                         goto err_init_qp_mem;
2575
2576                 dev_dbg(dev, "allocate qp dma buf size=%zx)\n", qp_dma_size);
2577         }
2578
2579         return ret;
2580
2581 err_init_qp_mem:
2582         hisi_qp_memory_uninit(qm, i);
2583 err_alloc_qp_array:
2584         dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma);
2585
2586         return ret;
2587 }
2588
2589 static void hisi_qm_pre_init(struct hisi_qm *qm)
2590 {
2591         struct pci_dev *pdev = qm->pdev;
2592
2593         if (qm->ver == QM_HW_V1)
2594                 qm->ops = &qm_hw_ops_v1;
2595         else if (qm->ver == QM_HW_V2)
2596                 qm->ops = &qm_hw_ops_v2;
2597         else
2598                 qm->ops = &qm_hw_ops_v3;
2599
2600         pci_set_drvdata(pdev, qm);
2601         mutex_init(&qm->mailbox_lock);
2602         init_rwsem(&qm->qps_lock);
2603         qm->qp_in_used = 0;
2604         qm->misc_ctl = false;
2605 }
2606
2607 static void qm_put_pci_res(struct hisi_qm *qm)
2608 {
2609         struct pci_dev *pdev = qm->pdev;
2610
2611         if (qm->use_db_isolation)
2612                 iounmap(qm->db_io_base);
2613
2614         iounmap(qm->io_base);
2615         pci_release_mem_regions(pdev);
2616 }
2617
2618 static void hisi_qm_pci_uninit(struct hisi_qm *qm)
2619 {
2620         struct pci_dev *pdev = qm->pdev;
2621
2622         pci_free_irq_vectors(pdev);
2623         qm_put_pci_res(qm);
2624         pci_disable_device(pdev);
2625 }
2626
2627 /**
2628  * hisi_qm_uninit() - Uninitialize qm.
2629  * @qm: The qm needed uninit.
2630  *
2631  * This function uninits qm related device resources.
2632  */
2633 void hisi_qm_uninit(struct hisi_qm *qm)
2634 {
2635         struct pci_dev *pdev = qm->pdev;
2636         struct device *dev = &pdev->dev;
2637
2638         down_write(&qm->qps_lock);
2639
2640         if (!qm_avail_state(qm, QM_CLOSE)) {
2641                 up_write(&qm->qps_lock);
2642                 return;
2643         }
2644
2645         hisi_qp_memory_uninit(qm, qm->qp_num);
2646         idr_destroy(&qm->qp_idr);
2647
2648         if (qm->qdma.va) {
2649                 hisi_qm_cache_wb(qm);
2650                 dma_free_coherent(dev, qm->qdma.size,
2651                                   qm->qdma.va, qm->qdma.dma);
2652         }
2653
2654         qm_irq_unregister(qm);
2655         hisi_qm_pci_uninit(qm);
2656         uacce_remove(qm->uacce);
2657         qm->uacce = NULL;
2658
2659         up_write(&qm->qps_lock);
2660 }
2661 EXPORT_SYMBOL_GPL(hisi_qm_uninit);
2662
2663 /**
2664  * hisi_qm_get_vft() - Get vft from a qm.
2665  * @qm: The qm we want to get its vft.
2666  * @base: The base number of queue in vft.
2667  * @number: The number of queues in vft.
2668  *
2669  * We can allocate multiple queues to a qm by configuring virtual function
2670  * table. We get related configures by this function. Normally, we call this
2671  * function in VF driver to get the queue information.
2672  *
2673  * qm hw v1 does not support this interface.
2674  */
2675 int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number)
2676 {
2677         if (!base || !number)
2678                 return -EINVAL;
2679
2680         if (!qm->ops->get_vft) {
2681                 dev_err(&qm->pdev->dev, "Don't support vft read!\n");
2682                 return -EINVAL;
2683         }
2684
2685         return qm->ops->get_vft(qm, base, number);
2686 }
2687 EXPORT_SYMBOL_GPL(hisi_qm_get_vft);
2688
2689 /**
2690  * hisi_qm_set_vft() - Set vft to a qm.
2691  * @qm: The qm we want to set its vft.
2692  * @fun_num: The function number.
2693  * @base: The base number of queue in vft.
2694  * @number: The number of queues in vft.
2695  *
2696  * This function is alway called in PF driver, it is used to assign queues
2697  * among PF and VFs.
2698  *
2699  * Assign queues A~B to PF: hisi_qm_set_vft(qm, 0, A, B - A + 1)
2700  * Assign queues A~B to VF: hisi_qm_set_vft(qm, 2, A, B - A + 1)
2701  * (VF function number 0x2)
2702  */
2703 static int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
2704                     u32 number)
2705 {
2706         u32 max_q_num = qm->ctrl_qp_num;
2707
2708         if (base >= max_q_num || number > max_q_num ||
2709             (base + number) > max_q_num)
2710                 return -EINVAL;
2711
2712         return qm_set_sqc_cqc_vft(qm, fun_num, base, number);
2713 }
2714
2715 static void qm_init_eq_aeq_status(struct hisi_qm *qm)
2716 {
2717         struct hisi_qm_status *status = &qm->status;
2718
2719         status->eq_head = 0;
2720         status->aeq_head = 0;
2721         status->eqc_phase = true;
2722         status->aeqc_phase = true;
2723 }
2724
2725 static int qm_eq_ctx_cfg(struct hisi_qm *qm)
2726 {
2727         struct device *dev = &qm->pdev->dev;
2728         struct qm_eqc *eqc;
2729         dma_addr_t eqc_dma;
2730         int ret;
2731
2732         eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL);
2733         if (!eqc)
2734                 return -ENOMEM;
2735
2736         eqc->base_l = cpu_to_le32(lower_32_bits(qm->eqe_dma));
2737         eqc->base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma));
2738         if (qm->ver == QM_HW_V1)
2739                 eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE);
2740         eqc->dw6 = cpu_to_le32((QM_EQ_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT));
2741
2742         eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc),
2743                                  DMA_TO_DEVICE);
2744         if (dma_mapping_error(dev, eqc_dma)) {
2745                 kfree(eqc);
2746                 return -ENOMEM;
2747         }
2748
2749         ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0);
2750         dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE);
2751         kfree(eqc);
2752
2753         return ret;
2754 }
2755
2756 static int qm_aeq_ctx_cfg(struct hisi_qm *qm)
2757 {
2758         struct device *dev = &qm->pdev->dev;
2759         struct qm_aeqc *aeqc;
2760         dma_addr_t aeqc_dma;
2761         int ret;
2762
2763         aeqc = kzalloc(sizeof(struct qm_aeqc), GFP_KERNEL);
2764         if (!aeqc)
2765                 return -ENOMEM;
2766
2767         aeqc->base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma));
2768         aeqc->base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma));
2769         aeqc->dw6 = cpu_to_le32((QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT));
2770
2771         aeqc_dma = dma_map_single(dev, aeqc, sizeof(struct qm_aeqc),
2772                                   DMA_TO_DEVICE);
2773         if (dma_mapping_error(dev, aeqc_dma)) {
2774                 kfree(aeqc);
2775                 return -ENOMEM;
2776         }
2777
2778         ret = qm_mb(qm, QM_MB_CMD_AEQC, aeqc_dma, 0, 0);
2779         dma_unmap_single(dev, aeqc_dma, sizeof(struct qm_aeqc), DMA_TO_DEVICE);
2780         kfree(aeqc);
2781
2782         return ret;
2783 }
2784
2785 static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm)
2786 {
2787         struct device *dev = &qm->pdev->dev;
2788         int ret;
2789
2790         qm_init_eq_aeq_status(qm);
2791
2792         ret = qm_eq_ctx_cfg(qm);
2793         if (ret) {
2794                 dev_err(dev, "Set eqc failed!\n");
2795                 return ret;
2796         }
2797
2798         return qm_aeq_ctx_cfg(qm);
2799 }
2800
2801 static int __hisi_qm_start(struct hisi_qm *qm)
2802 {
2803         int ret;
2804
2805         WARN_ON(!qm->qdma.va);
2806
2807         if (qm->fun_type == QM_HW_PF) {
2808                 ret = qm_dev_mem_reset(qm);
2809                 if (ret)
2810                         return ret;
2811
2812                 ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num);
2813                 if (ret)
2814                         return ret;
2815         }
2816
2817         ret = qm_eq_aeq_ctx_cfg(qm);
2818         if (ret)
2819                 return ret;
2820
2821         ret = qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);
2822         if (ret)
2823                 return ret;
2824
2825         ret = qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0);
2826         if (ret)
2827                 return ret;
2828
2829         writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK);
2830         writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK);
2831
2832         return 0;
2833 }
2834
2835 /**
2836  * hisi_qm_start() - start qm
2837  * @qm: The qm to be started.
2838  *
2839  * This function starts a qm, then we can allocate qp from this qm.
2840  */
2841 int hisi_qm_start(struct hisi_qm *qm)
2842 {
2843         struct device *dev = &qm->pdev->dev;
2844         int ret = 0;
2845
2846         down_write(&qm->qps_lock);
2847
2848         if (!qm_avail_state(qm, QM_START)) {
2849                 up_write(&qm->qps_lock);
2850                 return -EPERM;
2851         }
2852
2853         dev_dbg(dev, "qm start with %u queue pairs\n", qm->qp_num);
2854
2855         if (!qm->qp_num) {
2856                 dev_err(dev, "qp_num should not be 0\n");
2857                 ret = -EINVAL;
2858                 goto err_unlock;
2859         }
2860
2861         ret = __hisi_qm_start(qm);
2862         if (!ret)
2863                 atomic_set(&qm->status.flags, QM_START);
2864
2865 err_unlock:
2866         up_write(&qm->qps_lock);
2867         return ret;
2868 }
2869 EXPORT_SYMBOL_GPL(hisi_qm_start);
2870
2871 static int qm_restart(struct hisi_qm *qm)
2872 {
2873         struct device *dev = &qm->pdev->dev;
2874         struct hisi_qp *qp;
2875         int ret, i;
2876
2877         ret = hisi_qm_start(qm);
2878         if (ret < 0)
2879                 return ret;
2880
2881         down_write(&qm->qps_lock);
2882         for (i = 0; i < qm->qp_num; i++) {
2883                 qp = &qm->qp_array[i];
2884                 if (atomic_read(&qp->qp_status.flags) == QP_STOP &&
2885                     qp->is_resetting == true) {
2886                         ret = qm_start_qp_nolock(qp, 0);
2887                         if (ret < 0) {
2888                                 dev_err(dev, "Failed to start qp%d!\n", i);
2889
2890                                 up_write(&qm->qps_lock);
2891                                 return ret;
2892                         }
2893                         qp->is_resetting = false;
2894                 }
2895         }
2896         up_write(&qm->qps_lock);
2897
2898         return 0;
2899 }
2900
2901 /* Stop started qps in reset flow */
2902 static int qm_stop_started_qp(struct hisi_qm *qm)
2903 {
2904         struct device *dev = &qm->pdev->dev;
2905         struct hisi_qp *qp;
2906         int i, ret;
2907
2908         for (i = 0; i < qm->qp_num; i++) {
2909                 qp = &qm->qp_array[i];
2910                 if (qp && atomic_read(&qp->qp_status.flags) == QP_START) {
2911                         qp->is_resetting = true;
2912                         ret = qm_stop_qp_nolock(qp);
2913                         if (ret < 0) {
2914                                 dev_err(dev, "Failed to stop qp%d!\n", i);
2915                                 return ret;
2916                         }
2917                 }
2918         }
2919
2920         return 0;
2921 }
2922
2923
2924 /**
2925  * qm_clear_queues() - Clear all queues memory in a qm.
2926  * @qm: The qm in which the queues will be cleared.
2927  *
2928  * This function clears all queues memory in a qm. Reset of accelerator can
2929  * use this to clear queues.
2930  */
2931 static void qm_clear_queues(struct hisi_qm *qm)
2932 {
2933         struct hisi_qp *qp;
2934         int i;
2935
2936         for (i = 0; i < qm->qp_num; i++) {
2937                 qp = &qm->qp_array[i];
2938                 if (qp->is_resetting)
2939                         memset(qp->qdma.va, 0, qp->qdma.size);
2940         }
2941
2942         memset(qm->qdma.va, 0, qm->qdma.size);
2943 }
2944
2945 /**
2946  * hisi_qm_stop() - Stop a qm.
2947  * @qm: The qm which will be stopped.
2948  * @r: The reason to stop qm.
2949  *
2950  * This function stops qm and its qps, then qm can not accept request.
2951  * Related resources are not released at this state, we can use hisi_qm_start
2952  * to let qm start again.
2953  */
2954 int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r)
2955 {
2956         struct device *dev = &qm->pdev->dev;
2957         int ret = 0;
2958
2959         down_write(&qm->qps_lock);
2960
2961         qm->status.stop_reason = r;
2962         if (!qm_avail_state(qm, QM_STOP)) {
2963                 ret = -EPERM;
2964                 goto err_unlock;
2965         }
2966
2967         if (qm->status.stop_reason == QM_SOFT_RESET ||
2968             qm->status.stop_reason == QM_FLR) {
2969                 ret = qm_stop_started_qp(qm);
2970                 if (ret < 0) {
2971                         dev_err(dev, "Failed to stop started qp!\n");
2972                         goto err_unlock;
2973                 }
2974         }
2975
2976         /* Mask eq and aeq irq */
2977         writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);
2978         writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);
2979
2980         if (qm->fun_type == QM_HW_PF) {
2981                 ret = hisi_qm_set_vft(qm, 0, 0, 0);
2982                 if (ret < 0) {
2983                         dev_err(dev, "Failed to set vft!\n");
2984                         ret = -EBUSY;
2985                         goto err_unlock;
2986                 }
2987         }
2988
2989         qm_clear_queues(qm);
2990         atomic_set(&qm->status.flags, QM_STOP);
2991
2992 err_unlock:
2993         up_write(&qm->qps_lock);
2994         return ret;
2995 }
2996 EXPORT_SYMBOL_GPL(hisi_qm_stop);
2997
2998 static ssize_t qm_status_read(struct file *filp, char __user *buffer,
2999                               size_t count, loff_t *pos)
3000 {
3001         struct hisi_qm *qm = filp->private_data;
3002         char buf[QM_DBG_READ_LEN];
3003         int val, len;
3004
3005         val = atomic_read(&qm->status.flags);
3006         len = scnprintf(buf, QM_DBG_READ_LEN, "%s\n", qm_s[val]);
3007
3008         return simple_read_from_buffer(buffer, count, pos, buf, len);
3009 }
3010
3011 static const struct file_operations qm_status_fops = {
3012         .owner = THIS_MODULE,
3013         .open = simple_open,
3014         .read = qm_status_read,
3015 };
3016
3017 static int qm_debugfs_atomic64_set(void *data, u64 val)
3018 {
3019         if (val)
3020                 return -EINVAL;
3021
3022         atomic64_set((atomic64_t *)data, 0);
3023
3024         return 0;
3025 }
3026
3027 static int qm_debugfs_atomic64_get(void *data, u64 *val)
3028 {
3029         *val = atomic64_read((atomic64_t *)data);
3030
3031         return 0;
3032 }
3033
3034 DEFINE_DEBUGFS_ATTRIBUTE(qm_atomic64_ops, qm_debugfs_atomic64_get,
3035                          qm_debugfs_atomic64_set, "%llu\n");
3036
3037 /**
3038  * hisi_qm_debug_init() - Initialize qm related debugfs files.
3039  * @qm: The qm for which we want to add debugfs files.
3040  *
3041  * Create qm related debugfs files.
3042  */
3043 void hisi_qm_debug_init(struct hisi_qm *qm)
3044 {
3045         struct qm_dfx *dfx = &qm->debug.dfx;
3046         struct dentry *qm_d;
3047         void *data;
3048         int i;
3049
3050         qm_d = debugfs_create_dir("qm", qm->debug.debug_root);
3051         qm->debug.qm_d = qm_d;
3052
3053         /* only show this in PF */
3054         if (qm->fun_type == QM_HW_PF) {
3055                 qm_create_debugfs_file(qm, qm->debug.debug_root, CURRENT_QM);
3056                 for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++)
3057                         qm_create_debugfs_file(qm, qm_d, i);
3058         }
3059
3060         debugfs_create_file("regs", 0444, qm->debug.qm_d, qm, &qm_regs_fops);
3061
3062         debugfs_create_file("cmd", 0444, qm->debug.qm_d, qm, &qm_cmd_fops);
3063
3064         debugfs_create_file("status", 0444, qm->debug.qm_d, qm,
3065                         &qm_status_fops);
3066         for (i = 0; i < ARRAY_SIZE(qm_dfx_files); i++) {
3067                 data = (atomic64_t *)((uintptr_t)dfx + qm_dfx_files[i].offset);
3068                 debugfs_create_file(qm_dfx_files[i].name,
3069                         0644,
3070                         qm_d,
3071                         data,
3072                         &qm_atomic64_ops);
3073         }
3074 }
3075 EXPORT_SYMBOL_GPL(hisi_qm_debug_init);
3076
3077 /**
3078  * hisi_qm_debug_regs_clear() - clear qm debug related registers.
3079  * @qm: The qm for which we want to clear its debug registers.
3080  */
3081 void hisi_qm_debug_regs_clear(struct hisi_qm *qm)
3082 {
3083         struct qm_dfx_registers *regs;
3084         int i;
3085
3086         /* clear current_qm */
3087         writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF);
3088         writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF);
3089
3090         /* clear current_q */
3091         writel(0x0, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
3092         writel(0x0, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
3093
3094         /*
3095          * these registers are reading and clearing, so clear them after
3096          * reading them.
3097          */
3098         writel(0x1, qm->io_base + QM_DFX_CNT_CLR_CE);
3099
3100         regs = qm_dfx_regs;
3101         for (i = 0; i < CNT_CYC_REGS_NUM; i++) {
3102                 readl(qm->io_base + regs->reg_offset);
3103                 regs++;
3104         }
3105
3106         writel(0x0, qm->io_base + QM_DFX_CNT_CLR_CE);
3107 }
3108 EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear);
3109
3110 static void qm_hw_error_init(struct hisi_qm *qm)
3111 {
3112         struct hisi_qm_err_info *err_info = &qm->err_info;
3113
3114         if (!qm->ops->hw_error_init) {
3115                 dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n");
3116                 return;
3117         }
3118
3119         qm->ops->hw_error_init(qm, err_info->ce, err_info->nfe, err_info->fe);
3120 }
3121
3122 static void qm_hw_error_uninit(struct hisi_qm *qm)
3123 {
3124         if (!qm->ops->hw_error_uninit) {
3125                 dev_err(&qm->pdev->dev, "Unexpected QM hw error uninit!\n");
3126                 return;
3127         }
3128
3129         qm->ops->hw_error_uninit(qm);
3130 }
3131
3132 static enum acc_err_result qm_hw_error_handle(struct hisi_qm *qm)
3133 {
3134         if (!qm->ops->hw_error_handle) {
3135                 dev_err(&qm->pdev->dev, "QM doesn't support hw error report!\n");
3136                 return ACC_ERR_NONE;
3137         }
3138
3139         return qm->ops->hw_error_handle(qm);
3140 }
3141
3142 /**
3143  * hisi_qm_dev_err_init() - Initialize device error configuration.
3144  * @qm: The qm for which we want to do error initialization.
3145  *
3146  * Initialize QM and device error related configuration.
3147  */
3148 void hisi_qm_dev_err_init(struct hisi_qm *qm)
3149 {
3150         if (qm->fun_type == QM_HW_VF)
3151                 return;
3152
3153         qm_hw_error_init(qm);
3154
3155         if (!qm->err_ini->hw_err_enable) {
3156                 dev_err(&qm->pdev->dev, "Device doesn't support hw error init!\n");
3157                 return;
3158         }
3159         qm->err_ini->hw_err_enable(qm);
3160 }
3161 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_init);
3162
3163 /**
3164  * hisi_qm_dev_err_uninit() - Uninitialize device error configuration.
3165  * @qm: The qm for which we want to do error uninitialization.
3166  *
3167  * Uninitialize QM and device error related configuration.
3168  */
3169 void hisi_qm_dev_err_uninit(struct hisi_qm *qm)
3170 {
3171         if (qm->fun_type == QM_HW_VF)
3172                 return;
3173
3174         qm_hw_error_uninit(qm);
3175
3176         if (!qm->err_ini->hw_err_disable) {
3177                 dev_err(&qm->pdev->dev, "Unexpected device hw error uninit!\n");
3178                 return;
3179         }
3180         qm->err_ini->hw_err_disable(qm);
3181 }
3182 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit);
3183
3184 /**
3185  * hisi_qm_free_qps() - free multiple queue pairs.
3186  * @qps: The queue pairs need to be freed.
3187  * @qp_num: The num of queue pairs.
3188  */
3189 void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num)
3190 {
3191         int i;
3192
3193         if (!qps || qp_num <= 0)
3194                 return;
3195
3196         for (i = qp_num - 1; i >= 0; i--)
3197                 hisi_qm_release_qp(qps[i]);
3198 }
3199 EXPORT_SYMBOL_GPL(hisi_qm_free_qps);
3200
3201 static void free_list(struct list_head *head)
3202 {
3203         struct hisi_qm_resource *res, *tmp;
3204
3205         list_for_each_entry_safe(res, tmp, head, list) {
3206                 list_del(&res->list);
3207                 kfree(res);
3208         }
3209 }
3210
3211 static int hisi_qm_sort_devices(int node, struct list_head *head,
3212                                 struct hisi_qm_list *qm_list)
3213 {
3214         struct hisi_qm_resource *res, *tmp;
3215         struct hisi_qm *qm;
3216         struct list_head *n;
3217         struct device *dev;
3218         int dev_node = 0;
3219
3220         list_for_each_entry(qm, &qm_list->list, list) {
3221                 dev = &qm->pdev->dev;
3222
3223                 if (IS_ENABLED(CONFIG_NUMA)) {
3224                         dev_node = dev_to_node(dev);
3225                         if (dev_node < 0)
3226                                 dev_node = 0;
3227                 }
3228
3229                 res = kzalloc(sizeof(*res), GFP_KERNEL);
3230                 if (!res)
3231                         return -ENOMEM;
3232
3233                 res->qm = qm;
3234                 res->distance = node_distance(dev_node, node);
3235                 n = head;
3236                 list_for_each_entry(tmp, head, list) {
3237                         if (res->distance < tmp->distance) {
3238                                 n = &tmp->list;
3239                                 break;
3240                         }
3241                 }
3242                 list_add_tail(&res->list, n);
3243         }
3244
3245         return 0;
3246 }
3247
3248 /**
3249  * hisi_qm_alloc_qps_node() - Create multiple queue pairs.
3250  * @qm_list: The list of all available devices.
3251  * @qp_num: The number of queue pairs need created.
3252  * @alg_type: The algorithm type.
3253  * @node: The numa node.
3254  * @qps: The queue pairs need created.
3255  *
3256  * This function will sort all available device according to numa distance.
3257  * Then try to create all queue pairs from one device, if all devices do
3258  * not meet the requirements will return error.
3259  */
3260 int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num,
3261                            u8 alg_type, int node, struct hisi_qp **qps)
3262 {
3263         struct hisi_qm_resource *tmp;
3264         int ret = -ENODEV;
3265         LIST_HEAD(head);
3266         int i;
3267
3268         if (!qps || !qm_list || qp_num <= 0)
3269                 return -EINVAL;
3270
3271         mutex_lock(&qm_list->lock);
3272         if (hisi_qm_sort_devices(node, &head, qm_list)) {
3273                 mutex_unlock(&qm_list->lock);
3274                 goto err;
3275         }
3276
3277         list_for_each_entry(tmp, &head, list) {
3278                 for (i = 0; i < qp_num; i++) {
3279                         qps[i] = hisi_qm_create_qp(tmp->qm, alg_type);
3280                         if (IS_ERR(qps[i])) {
3281                                 hisi_qm_free_qps(qps, i);
3282                                 break;
3283                         }
3284                 }
3285
3286                 if (i == qp_num) {
3287                         ret = 0;
3288                         break;
3289                 }
3290         }
3291
3292         mutex_unlock(&qm_list->lock);
3293         if (ret)
3294                 pr_info("Failed to create qps, node[%d], alg[%u], qp[%d]!\n",
3295                         node, alg_type, qp_num);
3296
3297 err:
3298         free_list(&head);
3299         return ret;
3300 }
3301 EXPORT_SYMBOL_GPL(hisi_qm_alloc_qps_node);
3302
3303 static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs)
3304 {
3305         u32 remain_q_num, vfs_q_num, act_q_num, q_num, i, j;
3306         u32 max_qp_num = qm->max_qp_num;
3307         u32 q_base = qm->qp_num;
3308         int ret;
3309
3310         if (!num_vfs)
3311                 return -EINVAL;
3312
3313         vfs_q_num = qm->ctrl_qp_num - qm->qp_num;
3314
3315         /* If vfs_q_num is less than num_vfs, return error. */
3316         if (vfs_q_num < num_vfs)
3317                 return -EINVAL;
3318
3319         q_num = vfs_q_num / num_vfs;
3320         remain_q_num = vfs_q_num % num_vfs;
3321
3322         for (i = num_vfs; i > 0; i--) {
3323                 /*
3324                  * if q_num + remain_q_num > max_qp_num in last vf, divide the
3325                  * remaining queues equally.
3326                  */
3327                 if (i == num_vfs && q_num + remain_q_num <= max_qp_num) {
3328                         act_q_num = q_num + remain_q_num;
3329                         remain_q_num = 0;
3330                 } else if (remain_q_num > 0) {
3331                         act_q_num = q_num + 1;
3332                         remain_q_num--;
3333                 } else {
3334                         act_q_num = q_num;
3335                 }
3336
3337                 act_q_num = min_t(int, act_q_num, max_qp_num);
3338                 ret = hisi_qm_set_vft(qm, i, q_base, act_q_num);
3339                 if (ret) {
3340                         for (j = num_vfs; j > i; j--)
3341                                 hisi_qm_set_vft(qm, j, 0, 0);
3342                         return ret;
3343                 }
3344                 q_base += act_q_num;
3345         }
3346
3347         return 0;
3348 }
3349
3350 static int qm_clear_vft_config(struct hisi_qm *qm)
3351 {
3352         int ret;
3353         u32 i;
3354
3355         for (i = 1; i <= qm->vfs_num; i++) {
3356                 ret = hisi_qm_set_vft(qm, i, 0, 0);
3357                 if (ret)
3358                         return ret;
3359         }
3360         qm->vfs_num = 0;
3361
3362         return 0;
3363 }
3364
3365 /**
3366  * hisi_qm_sriov_enable() - enable virtual functions
3367  * @pdev: the PCIe device
3368  * @max_vfs: the number of virtual functions to enable
3369  *
3370  * Returns the number of enabled VFs. If there are VFs enabled already or
3371  * max_vfs is more than the total number of device can be enabled, returns
3372  * failure.
3373  */
3374 int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs)
3375 {
3376         struct hisi_qm *qm = pci_get_drvdata(pdev);
3377         int pre_existing_vfs, num_vfs, total_vfs, ret;
3378
3379         total_vfs = pci_sriov_get_totalvfs(pdev);
3380         pre_existing_vfs = pci_num_vf(pdev);
3381         if (pre_existing_vfs) {
3382                 pci_err(pdev, "%d VFs already enabled. Please disable pre-enabled VFs!\n",
3383                         pre_existing_vfs);
3384                 return 0;
3385         }
3386
3387         num_vfs = min_t(int, max_vfs, total_vfs);
3388         ret = qm_vf_q_assign(qm, num_vfs);
3389         if (ret) {
3390                 pci_err(pdev, "Can't assign queues for VF!\n");
3391                 return ret;
3392         }
3393
3394         qm->vfs_num = num_vfs;
3395
3396         ret = pci_enable_sriov(pdev, num_vfs);
3397         if (ret) {
3398                 pci_err(pdev, "Can't enable VF!\n");
3399                 qm_clear_vft_config(qm);
3400                 return ret;
3401         }
3402
3403         pci_info(pdev, "VF enabled, vfs_num(=%d)!\n", num_vfs);
3404
3405         return num_vfs;
3406 }
3407 EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable);
3408
3409 /**
3410  * hisi_qm_sriov_disable - disable virtual functions
3411  * @pdev: the PCI device.
3412  * @is_frozen: true when all the VFs are frozen.
3413  *
3414  * Return failure if there are VFs assigned already or VF is in used.
3415  */
3416 int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen)
3417 {
3418         struct hisi_qm *qm = pci_get_drvdata(pdev);
3419
3420         if (pci_vfs_assigned(pdev)) {
3421                 pci_err(pdev, "Failed to disable VFs as VFs are assigned!\n");
3422                 return -EPERM;
3423         }
3424
3425         /* While VF is in used, SRIOV cannot be disabled. */
3426         if (!is_frozen && qm_try_frozen_vfs(pdev, qm->qm_list)) {
3427                 pci_err(pdev, "Task is using its VF!\n");
3428                 return -EBUSY;
3429         }
3430
3431         pci_disable_sriov(pdev);
3432         return qm_clear_vft_config(qm);
3433 }
3434 EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable);
3435
3436 /**
3437  * hisi_qm_sriov_configure - configure the number of VFs
3438  * @pdev: The PCI device
3439  * @num_vfs: The number of VFs need enabled
3440  *
3441  * Enable SR-IOV according to num_vfs, 0 means disable.
3442  */
3443 int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs)
3444 {
3445         if (num_vfs == 0)
3446                 return hisi_qm_sriov_disable(pdev, false);
3447         else
3448                 return hisi_qm_sriov_enable(pdev, num_vfs);
3449 }
3450 EXPORT_SYMBOL_GPL(hisi_qm_sriov_configure);
3451
3452 static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm)
3453 {
3454         u32 err_sts;
3455
3456         if (!qm->err_ini->get_dev_hw_err_status) {
3457                 dev_err(&qm->pdev->dev, "Device doesn't support get hw error status!\n");
3458                 return ACC_ERR_NONE;
3459         }
3460
3461         /* get device hardware error status */
3462         err_sts = qm->err_ini->get_dev_hw_err_status(qm);
3463         if (err_sts) {
3464                 if (err_sts & qm->err_info.ecc_2bits_mask)
3465                         qm->err_status.is_dev_ecc_mbit = true;
3466
3467                 if (qm->err_ini->log_dev_hw_err)
3468                         qm->err_ini->log_dev_hw_err(qm, err_sts);
3469
3470                 /* ce error does not need to be reset */
3471                 if ((err_sts | qm->err_info.dev_ce_mask) ==
3472                      qm->err_info.dev_ce_mask) {
3473                         if (qm->err_ini->clear_dev_hw_err_status)
3474                                 qm->err_ini->clear_dev_hw_err_status(qm,
3475                                                                 err_sts);
3476
3477                         return ACC_ERR_RECOVERED;
3478                 }
3479
3480                 return ACC_ERR_NEED_RESET;
3481         }
3482
3483         return ACC_ERR_RECOVERED;
3484 }
3485
3486 static enum acc_err_result qm_process_dev_error(struct hisi_qm *qm)
3487 {
3488         enum acc_err_result qm_ret, dev_ret;
3489
3490         /* log qm error */
3491         qm_ret = qm_hw_error_handle(qm);
3492
3493         /* log device error */
3494         dev_ret = qm_dev_err_handle(qm);
3495
3496         return (qm_ret == ACC_ERR_NEED_RESET ||
3497                 dev_ret == ACC_ERR_NEED_RESET) ?
3498                 ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED;
3499 }
3500
3501 /**
3502  * hisi_qm_dev_err_detected() - Get device and qm error status then log it.
3503  * @pdev: The PCI device which need report error.
3504  * @state: The connectivity between CPU and device.
3505  *
3506  * We register this function into PCIe AER handlers, It will report device or
3507  * qm hardware error status when error occur.
3508  */
3509 pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev,
3510                                           pci_channel_state_t state)
3511 {
3512         struct hisi_qm *qm = pci_get_drvdata(pdev);
3513         enum acc_err_result ret;
3514
3515         if (pdev->is_virtfn)
3516                 return PCI_ERS_RESULT_NONE;
3517
3518         pci_info(pdev, "PCI error detected, state(=%u)!!\n", state);
3519         if (state == pci_channel_io_perm_failure)
3520                 return PCI_ERS_RESULT_DISCONNECT;
3521
3522         ret = qm_process_dev_error(qm);
3523         if (ret == ACC_ERR_NEED_RESET)
3524                 return PCI_ERS_RESULT_NEED_RESET;
3525
3526         return PCI_ERS_RESULT_RECOVERED;
3527 }
3528 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected);
3529
3530 static u32 qm_get_hw_error_status(struct hisi_qm *qm)
3531 {
3532         return readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
3533 }
3534
3535 static int qm_check_req_recv(struct hisi_qm *qm)
3536 {
3537         struct pci_dev *pdev = qm->pdev;
3538         int ret;
3539         u32 val;
3540
3541         writel(ACC_VENDOR_ID_VALUE, qm->io_base + QM_PEH_VENDOR_ID);
3542         ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val,
3543                                          (val == ACC_VENDOR_ID_VALUE),
3544                                          POLL_PERIOD, POLL_TIMEOUT);
3545         if (ret) {
3546                 dev_err(&pdev->dev, "Fails to read QM reg!\n");
3547                 return ret;
3548         }
3549
3550         writel(PCI_VENDOR_ID_HUAWEI, qm->io_base + QM_PEH_VENDOR_ID);
3551         ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val,
3552                                          (val == PCI_VENDOR_ID_HUAWEI),
3553                                          POLL_PERIOD, POLL_TIMEOUT);
3554         if (ret)
3555                 dev_err(&pdev->dev, "Fails to read QM reg in the second time!\n");
3556
3557         return ret;
3558 }
3559
3560 static int qm_set_pf_mse(struct hisi_qm *qm, bool set)
3561 {
3562         struct pci_dev *pdev = qm->pdev;
3563         u16 cmd;
3564         int i;
3565
3566         pci_read_config_word(pdev, PCI_COMMAND, &cmd);
3567         if (set)
3568                 cmd |= PCI_COMMAND_MEMORY;
3569         else
3570                 cmd &= ~PCI_COMMAND_MEMORY;
3571
3572         pci_write_config_word(pdev, PCI_COMMAND, cmd);
3573         for (i = 0; i < MAX_WAIT_COUNTS; i++) {
3574                 pci_read_config_word(pdev, PCI_COMMAND, &cmd);
3575                 if (set == ((cmd & PCI_COMMAND_MEMORY) >> 1))
3576                         return 0;
3577
3578                 udelay(1);
3579         }
3580
3581         return -ETIMEDOUT;
3582 }
3583
3584 static int qm_set_vf_mse(struct hisi_qm *qm, bool set)
3585 {
3586         struct pci_dev *pdev = qm->pdev;
3587         u16 sriov_ctrl;
3588         int pos;
3589         int i;
3590
3591         pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
3592         pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);
3593         if (set)
3594                 sriov_ctrl |= PCI_SRIOV_CTRL_MSE;
3595         else
3596                 sriov_ctrl &= ~PCI_SRIOV_CTRL_MSE;
3597         pci_write_config_word(pdev, pos + PCI_SRIOV_CTRL, sriov_ctrl);
3598
3599         for (i = 0; i < MAX_WAIT_COUNTS; i++) {
3600                 pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);
3601                 if (set == (sriov_ctrl & PCI_SRIOV_CTRL_MSE) >>
3602                     ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT)
3603                         return 0;
3604
3605                 udelay(1);
3606         }
3607
3608         return -ETIMEDOUT;
3609 }
3610
3611 static int qm_set_msi(struct hisi_qm *qm, bool set)
3612 {
3613         struct pci_dev *pdev = qm->pdev;
3614
3615         if (set) {
3616                 pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64,
3617                                        0);
3618         } else {
3619                 pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64,
3620                                        ACC_PEH_MSI_DISABLE);
3621                 if (qm->err_status.is_qm_ecc_mbit ||
3622                     qm->err_status.is_dev_ecc_mbit)
3623                         return 0;
3624
3625                 mdelay(1);
3626                 if (readl(qm->io_base + QM_PEH_DFX_INFO0))
3627                         return -EFAULT;
3628         }
3629
3630         return 0;
3631 }
3632
3633 static int qm_vf_reset_prepare(struct hisi_qm *qm,
3634                                enum qm_stop_reason stop_reason)
3635 {
3636         struct hisi_qm_list *qm_list = qm->qm_list;
3637         struct pci_dev *pdev = qm->pdev;
3638         struct pci_dev *virtfn;
3639         struct hisi_qm *vf_qm;
3640         int ret = 0;
3641
3642         mutex_lock(&qm_list->lock);
3643         list_for_each_entry(vf_qm, &qm_list->list, list) {
3644                 virtfn = vf_qm->pdev;
3645                 if (virtfn == pdev)
3646                         continue;
3647
3648                 if (pci_physfn(virtfn) == pdev) {
3649                         /* save VFs PCIE BAR configuration */
3650                         pci_save_state(virtfn);
3651
3652                         ret = hisi_qm_stop(vf_qm, stop_reason);
3653                         if (ret)
3654                                 goto stop_fail;
3655                 }
3656         }
3657
3658 stop_fail:
3659         mutex_unlock(&qm_list->lock);
3660         return ret;
3661 }
3662
3663 static int qm_reset_prepare_ready(struct hisi_qm *qm)
3664 {
3665         struct pci_dev *pdev = qm->pdev;
3666         struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
3667         int delay = 0;
3668
3669         /* All reset requests need to be queued for processing */
3670         while (test_and_set_bit(QM_RESETTING, &pf_qm->misc_ctl)) {
3671                 msleep(++delay);
3672                 if (delay > QM_RESET_WAIT_TIMEOUT)
3673                         return -EBUSY;
3674         }
3675
3676         return 0;
3677 }
3678
3679 static int qm_controller_reset_prepare(struct hisi_qm *qm)
3680 {
3681         struct pci_dev *pdev = qm->pdev;
3682         int ret;
3683
3684         ret = qm_reset_prepare_ready(qm);
3685         if (ret) {
3686                 pci_err(pdev, "Controller reset not ready!\n");
3687                 return ret;
3688         }
3689
3690         if (qm->vfs_num) {
3691                 ret = qm_vf_reset_prepare(qm, QM_SOFT_RESET);
3692                 if (ret) {
3693                         pci_err(pdev, "Fails to stop VFs!\n");
3694                         clear_bit(QM_RESETTING, &qm->misc_ctl);
3695                         return ret;
3696                 }
3697         }
3698
3699         ret = hisi_qm_stop(qm, QM_SOFT_RESET);
3700         if (ret) {
3701                 pci_err(pdev, "Fails to stop QM!\n");
3702                 clear_bit(QM_RESETTING, &qm->misc_ctl);
3703                 return ret;
3704         }
3705
3706         clear_bit(QM_RST_SCHED, &qm->misc_ctl);
3707
3708         return 0;
3709 }
3710
3711 static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm)
3712 {
3713         u32 nfe_enb = 0;
3714
3715         if (!qm->err_status.is_dev_ecc_mbit &&
3716             qm->err_status.is_qm_ecc_mbit &&
3717             qm->err_ini->close_axi_master_ooo) {
3718
3719                 qm->err_ini->close_axi_master_ooo(qm);
3720
3721         } else if (qm->err_status.is_dev_ecc_mbit &&
3722                    !qm->err_status.is_qm_ecc_mbit &&
3723                    !qm->err_ini->close_axi_master_ooo) {
3724
3725                 nfe_enb = readl(qm->io_base + QM_RAS_NFE_ENABLE);
3726                 writel(nfe_enb & QM_RAS_NFE_MBIT_DISABLE,
3727                        qm->io_base + QM_RAS_NFE_ENABLE);
3728                 writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SET);
3729         }
3730 }
3731
3732 static int qm_soft_reset(struct hisi_qm *qm)
3733 {
3734         struct pci_dev *pdev = qm->pdev;
3735         int ret;
3736         u32 val;
3737
3738         /* Ensure all doorbells and mailboxes received by QM */
3739         ret = qm_check_req_recv(qm);
3740         if (ret)
3741                 return ret;
3742
3743         if (qm->vfs_num) {
3744                 ret = qm_set_vf_mse(qm, false);
3745                 if (ret) {
3746                         pci_err(pdev, "Fails to disable vf MSE bit.\n");
3747                         return ret;
3748                 }
3749         }
3750
3751         ret = qm_set_msi(qm, false);
3752         if (ret) {
3753                 pci_err(pdev, "Fails to disable PEH MSI bit.\n");
3754                 return ret;
3755         }
3756
3757         qm_dev_ecc_mbit_handle(qm);
3758
3759         /* OOO register set and check */
3760         writel(ACC_MASTER_GLOBAL_CTRL_SHUTDOWN,
3761                qm->io_base + ACC_MASTER_GLOBAL_CTRL);
3762
3763         /* If bus lock, reset chip */
3764         ret = readl_relaxed_poll_timeout(qm->io_base + ACC_MASTER_TRANS_RETURN,
3765                                          val,
3766                                          (val == ACC_MASTER_TRANS_RETURN_RW),
3767                                          POLL_PERIOD, POLL_TIMEOUT);
3768         if (ret) {
3769                 pci_emerg(pdev, "Bus lock! Please reset system.\n");
3770                 return ret;
3771         }
3772
3773         ret = qm_set_pf_mse(qm, false);
3774         if (ret) {
3775                 pci_err(pdev, "Fails to disable pf MSE bit.\n");
3776                 return ret;
3777         }
3778
3779         /* The reset related sub-control registers are not in PCI BAR */
3780         if (ACPI_HANDLE(&pdev->dev)) {
3781                 unsigned long long value = 0;
3782                 acpi_status s;
3783
3784                 s = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev),
3785                                           qm->err_info.acpi_rst,
3786                                           NULL, &value);
3787                 if (ACPI_FAILURE(s)) {
3788                         pci_err(pdev, "NO controller reset method!\n");
3789                         return -EIO;
3790                 }
3791
3792                 if (value) {
3793                         pci_err(pdev, "Reset step %llu failed!\n", value);
3794                         return -EIO;
3795                 }
3796         } else {
3797                 pci_err(pdev, "No reset method!\n");
3798                 return -EINVAL;
3799         }
3800
3801         return 0;
3802 }
3803
3804 static int qm_vf_reset_done(struct hisi_qm *qm)
3805 {
3806         struct hisi_qm_list *qm_list = qm->qm_list;
3807         struct pci_dev *pdev = qm->pdev;
3808         struct pci_dev *virtfn;
3809         struct hisi_qm *vf_qm;
3810         int ret = 0;
3811
3812         mutex_lock(&qm_list->lock);
3813         list_for_each_entry(vf_qm, &qm_list->list, list) {
3814                 virtfn = vf_qm->pdev;
3815                 if (virtfn == pdev)
3816                         continue;
3817
3818                 if (pci_physfn(virtfn) == pdev) {
3819                         /* enable VFs PCIE BAR configuration */
3820                         pci_restore_state(virtfn);
3821
3822                         ret = qm_restart(vf_qm);
3823                         if (ret)
3824                                 goto restart_fail;
3825                 }
3826         }
3827
3828 restart_fail:
3829         mutex_unlock(&qm_list->lock);
3830         return ret;
3831 }
3832
3833 static u32 qm_get_dev_err_status(struct hisi_qm *qm)
3834 {
3835         return qm->err_ini->get_dev_hw_err_status(qm);
3836 }
3837
3838 static int qm_dev_hw_init(struct hisi_qm *qm)
3839 {
3840         return qm->err_ini->hw_init(qm);
3841 }
3842
3843 static void qm_restart_prepare(struct hisi_qm *qm)
3844 {
3845         u32 value;
3846
3847         if (!qm->err_status.is_qm_ecc_mbit &&
3848             !qm->err_status.is_dev_ecc_mbit)
3849                 return;
3850
3851         /* temporarily close the OOO port used for PEH to write out MSI */
3852         value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);
3853         writel(value & ~qm->err_info.msi_wr_port,
3854                qm->io_base + ACC_AM_CFG_PORT_WR_EN);
3855
3856         /* clear dev ecc 2bit error source if having */
3857         value = qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask;
3858         if (value && qm->err_ini->clear_dev_hw_err_status)
3859                 qm->err_ini->clear_dev_hw_err_status(qm, value);
3860
3861         /* clear QM ecc mbit error source */
3862         writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SOURCE);
3863
3864         /* clear AM Reorder Buffer ecc mbit source */
3865         writel(ACC_ROB_ECC_ERR_MULTPL, qm->io_base + ACC_AM_ROB_ECC_INT_STS);
3866
3867         if (qm->err_ini->open_axi_master_ooo)
3868                 qm->err_ini->open_axi_master_ooo(qm);
3869 }
3870
3871 static void qm_restart_done(struct hisi_qm *qm)
3872 {
3873         u32 value;
3874
3875         if (!qm->err_status.is_qm_ecc_mbit &&
3876             !qm->err_status.is_dev_ecc_mbit)
3877                 return;
3878
3879         /* open the OOO port for PEH to write out MSI */
3880         value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);
3881         value |= qm->err_info.msi_wr_port;
3882         writel(value, qm->io_base + ACC_AM_CFG_PORT_WR_EN);
3883
3884         qm->err_status.is_qm_ecc_mbit = false;
3885         qm->err_status.is_dev_ecc_mbit = false;
3886 }
3887
3888 static int qm_controller_reset_done(struct hisi_qm *qm)
3889 {
3890         struct pci_dev *pdev = qm->pdev;
3891         int ret;
3892
3893         ret = qm_set_msi(qm, true);
3894         if (ret) {
3895                 pci_err(pdev, "Fails to enable PEH MSI bit!\n");
3896                 return ret;
3897         }
3898
3899         ret = qm_set_pf_mse(qm, true);
3900         if (ret) {
3901                 pci_err(pdev, "Fails to enable pf MSE bit!\n");
3902                 return ret;
3903         }
3904
3905         if (qm->vfs_num) {
3906                 ret = qm_set_vf_mse(qm, true);
3907                 if (ret) {
3908                         pci_err(pdev, "Fails to enable vf MSE bit!\n");
3909                         return ret;
3910                 }
3911         }
3912
3913         ret = qm_dev_hw_init(qm);
3914         if (ret) {
3915                 pci_err(pdev, "Failed to init device\n");
3916                 return ret;
3917         }
3918
3919         qm_restart_prepare(qm);
3920
3921         ret = qm_restart(qm);
3922         if (ret) {
3923                 pci_err(pdev, "Failed to start QM!\n");
3924                 return ret;
3925         }
3926
3927         if (qm->vfs_num) {
3928                 ret = qm_vf_q_assign(qm, qm->vfs_num);
3929                 if (ret) {
3930                         pci_err(pdev, "Failed to assign queue!\n");
3931                         return ret;
3932                 }
3933         }
3934
3935         ret = qm_vf_reset_done(qm);
3936         if (ret) {
3937                 pci_err(pdev, "Failed to start VFs!\n");
3938                 return -EPERM;
3939         }
3940
3941         hisi_qm_dev_err_init(qm);
3942         qm_restart_done(qm);
3943
3944         clear_bit(QM_RESETTING, &qm->misc_ctl);
3945
3946         return 0;
3947 }
3948
3949 static int qm_controller_reset(struct hisi_qm *qm)
3950 {
3951         struct pci_dev *pdev = qm->pdev;
3952         int ret;
3953
3954         pci_info(pdev, "Controller resetting...\n");
3955
3956         ret = qm_controller_reset_prepare(qm);
3957         if (ret) {
3958                 clear_bit(QM_RST_SCHED, &qm->misc_ctl);
3959                 return ret;
3960         }
3961
3962         ret = qm_soft_reset(qm);
3963         if (ret) {
3964                 pci_err(pdev, "Controller reset failed (%d)\n", ret);
3965                 clear_bit(QM_RESETTING, &qm->misc_ctl);
3966                 return ret;
3967         }
3968
3969         ret = qm_controller_reset_done(qm);
3970         if (ret) {
3971                 clear_bit(QM_RESETTING, &qm->misc_ctl);
3972                 return ret;
3973         }
3974
3975         pci_info(pdev, "Controller reset complete\n");
3976
3977         return 0;
3978 }
3979
3980 /**
3981  * hisi_qm_dev_slot_reset() - slot reset
3982  * @pdev: the PCIe device
3983  *
3984  * This function offers QM relate PCIe device reset interface. Drivers which
3985  * use QM can use this function as slot_reset in its struct pci_error_handlers.
3986  */
3987 pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev)
3988 {
3989         struct hisi_qm *qm = pci_get_drvdata(pdev);
3990         int ret;
3991
3992         if (pdev->is_virtfn)
3993                 return PCI_ERS_RESULT_RECOVERED;
3994
3995         pci_aer_clear_nonfatal_status(pdev);
3996
3997         /* reset pcie device controller */
3998         ret = qm_controller_reset(qm);
3999         if (ret) {
4000                 pci_err(pdev, "Controller reset failed (%d)\n", ret);
4001                 return PCI_ERS_RESULT_DISCONNECT;
4002         }
4003
4004         return PCI_ERS_RESULT_RECOVERED;
4005 }
4006 EXPORT_SYMBOL_GPL(hisi_qm_dev_slot_reset);
4007
4008 /* check the interrupt is ecc-mbit error or not */
4009 static int qm_check_dev_error(struct hisi_qm *qm)
4010 {
4011         int ret;
4012
4013         if (qm->fun_type == QM_HW_VF)
4014                 return 0;
4015
4016         ret = qm_get_hw_error_status(qm) & QM_ECC_MBIT;
4017         if (ret)
4018                 return ret;
4019
4020         return (qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask);
4021 }
4022
4023 void hisi_qm_reset_prepare(struct pci_dev *pdev)
4024 {
4025         struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
4026         struct hisi_qm *qm = pci_get_drvdata(pdev);
4027         u32 delay = 0;
4028         int ret;
4029
4030         hisi_qm_dev_err_uninit(pf_qm);
4031
4032         /*
4033          * Check whether there is an ECC mbit error, If it occurs, need to
4034          * wait for soft reset to fix it.
4035          */
4036         while (qm_check_dev_error(pf_qm)) {
4037                 msleep(++delay);
4038                 if (delay > QM_RESET_WAIT_TIMEOUT)
4039                         return;
4040         }
4041
4042         ret = qm_reset_prepare_ready(qm);
4043         if (ret) {
4044                 pci_err(pdev, "FLR not ready!\n");
4045                 return;
4046         }
4047
4048         if (qm->vfs_num) {
4049                 ret = qm_vf_reset_prepare(qm, QM_FLR);
4050                 if (ret) {
4051                         pci_err(pdev, "Failed to prepare reset, ret = %d.\n",
4052                                 ret);
4053                         return;
4054                 }
4055         }
4056
4057         ret = hisi_qm_stop(qm, QM_FLR);
4058         if (ret) {
4059                 pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret);
4060                 return;
4061         }
4062
4063         pci_info(pdev, "FLR resetting...\n");
4064 }
4065 EXPORT_SYMBOL_GPL(hisi_qm_reset_prepare);
4066
4067 static bool qm_flr_reset_complete(struct pci_dev *pdev)
4068 {
4069         struct pci_dev *pf_pdev = pci_physfn(pdev);
4070         struct hisi_qm *qm = pci_get_drvdata(pf_pdev);
4071         u32 id;
4072
4073         pci_read_config_dword(qm->pdev, PCI_COMMAND, &id);
4074         if (id == QM_PCI_COMMAND_INVALID) {
4075                 pci_err(pdev, "Device can not be used!\n");
4076                 return false;
4077         }
4078
4079         return true;
4080 }
4081
4082 void hisi_qm_reset_done(struct pci_dev *pdev)
4083 {
4084         struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
4085         struct hisi_qm *qm = pci_get_drvdata(pdev);
4086         int ret;
4087
4088         hisi_qm_dev_err_init(pf_qm);
4089
4090         ret = qm_restart(qm);
4091         if (ret) {
4092                 pci_err(pdev, "Failed to start QM, ret = %d.\n", ret);
4093                 goto flr_done;
4094         }
4095
4096         if (qm->fun_type == QM_HW_PF) {
4097                 ret = qm_dev_hw_init(qm);
4098                 if (ret) {
4099                         pci_err(pdev, "Failed to init PF, ret = %d.\n", ret);
4100                         goto flr_done;
4101                 }
4102
4103                 if (!qm->vfs_num)
4104                         goto flr_done;
4105
4106                 ret = qm_vf_q_assign(qm, qm->vfs_num);
4107                 if (ret) {
4108                         pci_err(pdev, "Failed to assign VFs, ret = %d.\n", ret);
4109                         goto flr_done;
4110                 }
4111
4112                 ret = qm_vf_reset_done(qm);
4113                 if (ret) {
4114                         pci_err(pdev, "Failed to start VFs, ret = %d.\n", ret);
4115                         goto flr_done;
4116                 }
4117         }
4118
4119 flr_done:
4120         if (qm_flr_reset_complete(pdev))
4121                 pci_info(pdev, "FLR reset complete\n");
4122
4123         clear_bit(QM_RESETTING, &qm->misc_ctl);
4124 }
4125 EXPORT_SYMBOL_GPL(hisi_qm_reset_done);
4126
4127 static irqreturn_t qm_abnormal_irq(int irq, void *data)
4128 {
4129         struct hisi_qm *qm = data;
4130         enum acc_err_result ret;
4131
4132         atomic64_inc(&qm->debug.dfx.abnormal_irq_cnt);
4133         ret = qm_process_dev_error(qm);
4134         if (ret == ACC_ERR_NEED_RESET &&
4135             !test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl) &&
4136             !test_and_set_bit(QM_RST_SCHED, &qm->misc_ctl))
4137                 schedule_work(&qm->rst_work);
4138
4139         return IRQ_HANDLED;
4140 }
4141
4142 static int qm_irq_register(struct hisi_qm *qm)
4143 {
4144         struct pci_dev *pdev = qm->pdev;
4145         int ret;
4146
4147         ret = request_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR),
4148                           qm_irq, 0, qm->dev_name, qm);
4149         if (ret)
4150                 return ret;
4151
4152         if (qm->ver != QM_HW_V1) {
4153                 ret = request_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR),
4154                                   qm_aeq_irq, 0, qm->dev_name, qm);
4155                 if (ret)
4156                         goto err_aeq_irq;
4157
4158                 if (qm->fun_type == QM_HW_PF) {
4159                         ret = request_irq(pci_irq_vector(pdev,
4160                                           QM_ABNORMAL_EVENT_IRQ_VECTOR),
4161                                           qm_abnormal_irq, 0, qm->dev_name, qm);
4162                         if (ret)
4163                                 goto err_abonormal_irq;
4164                 }
4165         }
4166
4167         return 0;
4168
4169 err_abonormal_irq:
4170         free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm);
4171 err_aeq_irq:
4172         free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm);
4173         return ret;
4174 }
4175
4176 /**
4177  * hisi_qm_dev_shutdown() - Shutdown device.
4178  * @pdev: The device will be shutdown.
4179  *
4180  * This function will stop qm when OS shutdown or rebooting.
4181  */
4182 void hisi_qm_dev_shutdown(struct pci_dev *pdev)
4183 {
4184         struct hisi_qm *qm = pci_get_drvdata(pdev);
4185         int ret;
4186
4187         ret = hisi_qm_stop(qm, QM_NORMAL);
4188         if (ret)
4189                 dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n");
4190 }
4191 EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown);
4192
4193 static void hisi_qm_controller_reset(struct work_struct *rst_work)
4194 {
4195         struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work);
4196         int ret;
4197
4198         /* reset pcie device controller */
4199         ret = qm_controller_reset(qm);
4200         if (ret)
4201                 dev_err(&qm->pdev->dev, "controller reset failed (%d)\n", ret);
4202
4203 }
4204
4205 /**
4206  * hisi_qm_alg_register() - Register alg to crypto and add qm to qm_list.
4207  * @qm: The qm needs add.
4208  * @qm_list: The qm list.
4209  *
4210  * This function adds qm to qm list, and will register algorithm to
4211  * crypto when the qm list is empty.
4212  */
4213 int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
4214 {
4215         int flag = 0;
4216         int ret = 0;
4217         /* HW V2 not support both use uacce sva mode and hardware crypto algs */
4218         if (qm->ver <= QM_HW_V2 && qm->use_sva)
4219                 return 0;
4220
4221         mutex_lock(&qm_list->lock);
4222         if (list_empty(&qm_list->list))
4223                 flag = 1;
4224         list_add_tail(&qm->list, &qm_list->list);
4225         mutex_unlock(&qm_list->lock);
4226
4227         if (flag) {
4228                 ret = qm_list->register_to_crypto(qm);
4229                 if (ret) {
4230                         mutex_lock(&qm_list->lock);
4231                         list_del(&qm->list);
4232                         mutex_unlock(&qm_list->lock);
4233                 }
4234         }
4235
4236         return ret;
4237 }
4238 EXPORT_SYMBOL_GPL(hisi_qm_alg_register);
4239
4240 /**
4241  * hisi_qm_alg_unregister() - Unregister alg from crypto and delete qm from
4242  * qm list.
4243  * @qm: The qm needs delete.
4244  * @qm_list: The qm list.
4245  *
4246  * This function deletes qm from qm list, and will unregister algorithm
4247  * from crypto when the qm list is empty.
4248  */
4249 void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
4250 {
4251         if (qm->ver <= QM_HW_V2 && qm->use_sva)
4252                 return;
4253
4254         mutex_lock(&qm_list->lock);
4255         list_del(&qm->list);
4256         mutex_unlock(&qm_list->lock);
4257
4258         if (list_empty(&qm_list->list))
4259                 qm_list->unregister_from_crypto(qm);
4260 }
4261 EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister);
4262
4263 static int qm_get_qp_num(struct hisi_qm *qm)
4264 {
4265         if (qm->ver == QM_HW_V1)
4266                 qm->ctrl_qp_num = QM_QNUM_V1;
4267         else if (qm->ver == QM_HW_V2)
4268                 qm->ctrl_qp_num = QM_QNUM_V2;
4269         else
4270                 qm->ctrl_qp_num = readl(qm->io_base + QM_CAPBILITY) &
4271                                         QM_QP_NUN_MASK;
4272
4273         if (qm->use_db_isolation)
4274                 qm->max_qp_num = (readl(qm->io_base + QM_CAPBILITY) >>
4275                                   QM_QP_MAX_NUM_SHIFT) & QM_QP_NUN_MASK;
4276         else
4277                 qm->max_qp_num = qm->ctrl_qp_num;
4278
4279         /* check if qp number is valid */
4280         if (qm->qp_num > qm->max_qp_num) {
4281                 dev_err(&qm->pdev->dev, "qp num(%u) is more than max qp num(%u)!\n",
4282                         qm->qp_num, qm->max_qp_num);
4283                 return -EINVAL;
4284         }
4285
4286         return 0;
4287 }
4288
4289 static int qm_get_pci_res(struct hisi_qm *qm)
4290 {
4291         struct pci_dev *pdev = qm->pdev;
4292         struct device *dev = &pdev->dev;
4293         int ret;
4294
4295         ret = pci_request_mem_regions(pdev, qm->dev_name);
4296         if (ret < 0) {
4297                 dev_err(dev, "Failed to request mem regions!\n");
4298                 return ret;
4299         }
4300
4301         qm->phys_base = pci_resource_start(pdev, PCI_BAR_2);
4302         qm->io_base = ioremap(qm->phys_base, pci_resource_len(pdev, PCI_BAR_2));
4303         if (!qm->io_base) {
4304                 ret = -EIO;
4305                 goto err_request_mem_regions;
4306         }
4307
4308         if (qm->ver > QM_HW_V2) {
4309                 if (qm->fun_type == QM_HW_PF)
4310                         qm->use_db_isolation = readl(qm->io_base +
4311                                                      QM_QUE_ISO_EN) & BIT(0);
4312                 else
4313                         qm->use_db_isolation = readl(qm->io_base +
4314                                                      QM_QUE_ISO_CFG_V) & BIT(0);
4315         }
4316
4317         if (qm->use_db_isolation) {
4318                 qm->db_interval = QM_QP_DB_INTERVAL;
4319                 qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4);
4320                 qm->db_io_base = ioremap(qm->db_phys_base,
4321                                          pci_resource_len(pdev, PCI_BAR_4));
4322                 if (!qm->db_io_base) {
4323                         ret = -EIO;
4324                         goto err_ioremap;
4325                 }
4326         } else {
4327                 qm->db_phys_base = qm->phys_base;
4328                 qm->db_io_base = qm->io_base;
4329                 qm->db_interval = 0;
4330         }
4331
4332         if (qm->fun_type == QM_HW_PF) {
4333                 ret = qm_get_qp_num(qm);
4334                 if (ret)
4335                         goto err_db_ioremap;
4336         }
4337
4338         return 0;
4339
4340 err_db_ioremap:
4341         if (qm->use_db_isolation)
4342                 iounmap(qm->db_io_base);
4343 err_ioremap:
4344         iounmap(qm->io_base);
4345 err_request_mem_regions:
4346         pci_release_mem_regions(pdev);
4347         return ret;
4348 }
4349
4350 static int hisi_qm_pci_init(struct hisi_qm *qm)
4351 {
4352         struct pci_dev *pdev = qm->pdev;
4353         struct device *dev = &pdev->dev;
4354         unsigned int num_vec;
4355         int ret;
4356
4357         ret = pci_enable_device_mem(pdev);
4358         if (ret < 0) {
4359                 dev_err(dev, "Failed to enable device mem!\n");
4360                 return ret;
4361         }
4362
4363         ret = qm_get_pci_res(qm);
4364         if (ret)
4365                 goto err_disable_pcidev;
4366
4367         ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
4368         if (ret < 0)
4369                 goto err_get_pci_res;
4370         pci_set_master(pdev);
4371
4372         if (!qm->ops->get_irq_num) {
4373                 ret = -EOPNOTSUPP;
4374                 goto err_get_pci_res;
4375         }
4376         num_vec = qm->ops->get_irq_num(qm);
4377         ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI);
4378         if (ret < 0) {
4379                 dev_err(dev, "Failed to enable MSI vectors!\n");
4380                 goto err_get_pci_res;
4381         }
4382
4383         return 0;
4384
4385 err_get_pci_res:
4386         qm_put_pci_res(qm);
4387 err_disable_pcidev:
4388         pci_disable_device(pdev);
4389         return ret;
4390 }
4391
4392 /**
4393  * hisi_qm_init() - Initialize configures about qm.
4394  * @qm: The qm needing init.
4395  *
4396  * This function init qm, then we can call hisi_qm_start to put qm into work.
4397  */
4398 int hisi_qm_init(struct hisi_qm *qm)
4399 {
4400         struct pci_dev *pdev = qm->pdev;
4401         struct device *dev = &pdev->dev;
4402         int ret;
4403
4404         hisi_qm_pre_init(qm);
4405
4406         ret = hisi_qm_pci_init(qm);
4407         if (ret)
4408                 return ret;
4409
4410         ret = qm_irq_register(qm);
4411         if (ret)
4412                 goto err_pci_init;
4413
4414         if (qm->fun_type == QM_HW_VF && qm->ver != QM_HW_V1) {
4415                 /* v2 starts to support get vft by mailbox */
4416                 ret = hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num);
4417                 if (ret)
4418                         goto err_irq_register;
4419         }
4420
4421         ret = qm_alloc_uacce(qm);
4422         if (ret < 0)
4423                 dev_warn(dev, "fail to alloc uacce (%d)\n", ret);
4424
4425         ret = hisi_qm_memory_init(qm);
4426         if (ret)
4427                 goto err_alloc_uacce;
4428
4429         INIT_WORK(&qm->work, qm_work_process);
4430         if (qm->fun_type == QM_HW_PF)
4431                 INIT_WORK(&qm->rst_work, hisi_qm_controller_reset);
4432
4433         atomic_set(&qm->status.flags, QM_INIT);
4434
4435         return 0;
4436
4437 err_alloc_uacce:
4438         uacce_remove(qm->uacce);
4439         qm->uacce = NULL;
4440 err_irq_register:
4441         qm_irq_unregister(qm);
4442 err_pci_init:
4443         hisi_qm_pci_uninit(qm);
4444         return ret;
4445 }
4446 EXPORT_SYMBOL_GPL(hisi_qm_init);
4447
4448 MODULE_LICENSE("GPL v2");
4449 MODULE_AUTHOR("Zhou Wang <[email protected]>");
4450 MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver");