drivers/crypto/hisilicon/sec2/sec_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2019 HiSilicon Limited. */
   3
   4 #include <linux/acpi.h>
   5 #include <linux/bitops.h>
   6 #include <linux/debugfs.h>
   7 #include <linux/init.h>
   8 #include <linux/io.h>
   9 #include <linux/iommu.h>
  10 #include <linux/kernel.h>
  11 #include <linux/module.h>
  12 #include <linux/pci.h>
  13 #include <linux/pm_runtime.h>
  14 #include <linux/seq_file.h>
  15 #include <linux/topology.h>
  16 #include <linux/uacce.h>
  17
  18 #include "sec.h"
  19
  20 #define SEC_VF_NUM                      63
  21 #define SEC_QUEUE_NUM_V1                4096
  22 #define PCI_DEVICE_ID_HUAWEI_SEC_PF     0xa255
  23
  24 #define SEC_BD_ERR_CHK_EN0              0xEFFFFFFF
  25 #define SEC_BD_ERR_CHK_EN1              0x7ffff7fd
  26 #define SEC_BD_ERR_CHK_EN3              0xffffbfff
  27
  28 #define SEC_SQE_SIZE                    128
  29 #define SEC_PF_DEF_Q_NUM                256
  30 #define SEC_PF_DEF_Q_BASE               0
  31 #define SEC_CTX_Q_NUM_DEF               2
  32 #define SEC_CTX_Q_NUM_MAX               32
  33
  34 #define SEC_CTRL_CNT_CLR_CE             0x301120
  35 #define SEC_CTRL_CNT_CLR_CE_BIT BIT(0)
  36 #define SEC_CORE_INT_SOURCE             0x301010
  37 #define SEC_CORE_INT_MASK               0x301000
  38 #define SEC_CORE_INT_STATUS             0x301008
  39 #define SEC_CORE_SRAM_ECC_ERR_INFO      0x301C14
  40 #define SEC_ECC_NUM                     16
  41 #define SEC_ECC_MASH                    0xFF
  42 #define SEC_CORE_INT_DISABLE            0x0
  43
  44 #define SEC_RAS_CE_REG                  0x301050
  45 #define SEC_RAS_FE_REG                  0x301054
  46 #define SEC_RAS_NFE_REG                 0x301058
  47 #define SEC_RAS_FE_ENB_MSK              0x0
  48 #define SEC_OOO_SHUTDOWN_SEL            0x301014
  49 #define SEC_RAS_DISABLE         0x0
  50 #define SEC_MEM_START_INIT_REG  0x301100
  51 #define SEC_MEM_INIT_DONE_REG           0x301104
  52
  53 /* clock gating */
  54 #define SEC_CONTROL_REG         0x301200
  55 #define SEC_DYNAMIC_GATE_REG            0x30121c
  56 #define SEC_CORE_AUTO_GATE              0x30212c
  57 #define SEC_DYNAMIC_GATE_EN             0x7fff
  58 #define SEC_CORE_AUTO_GATE_EN           GENMASK(3, 0)
  59 #define SEC_CLK_GATE_ENABLE             BIT(3)
  60 #define SEC_CLK_GATE_DISABLE            (~BIT(3))
  61
  62 #define SEC_TRNG_EN_SHIFT               8
  63 #define SEC_AXI_SHUTDOWN_ENABLE BIT(12)
  64 #define SEC_AXI_SHUTDOWN_DISABLE        0xFFFFEFFF
  65
  66 #define SEC_INTERFACE_USER_CTRL0_REG    0x301220
  67 #define SEC_INTERFACE_USER_CTRL1_REG    0x301224
  68 #define SEC_SAA_EN_REG                  0x301270
  69 #define SEC_BD_ERR_CHK_EN_REG0          0x301380
  70 #define SEC_BD_ERR_CHK_EN_REG1          0x301384
  71 #define SEC_BD_ERR_CHK_EN_REG3          0x30138c
  72
  73 #define SEC_USER0_SMMU_NORMAL           (BIT(23) | BIT(15))
  74 #define SEC_USER1_SMMU_NORMAL           (BIT(31) | BIT(23) | BIT(15) | BIT(7))
  75 #define SEC_USER1_ENABLE_CONTEXT_SSV    BIT(24)
  76 #define SEC_USER1_ENABLE_DATA_SSV       BIT(16)
  77 #define SEC_USER1_WB_CONTEXT_SSV        BIT(8)
  78 #define SEC_USER1_WB_DATA_SSV           BIT(0)
  79 #define SEC_USER1_SVA_SET               (SEC_USER1_ENABLE_CONTEXT_SSV | \
  80                                         SEC_USER1_ENABLE_DATA_SSV | \
  81                                         SEC_USER1_WB_CONTEXT_SSV |  \
  82                                         SEC_USER1_WB_DATA_SSV)
  83 #define SEC_USER1_SMMU_SVA              (SEC_USER1_SMMU_NORMAL | SEC_USER1_SVA_SET)
  84 #define SEC_USER1_SMMU_MASK             (~SEC_USER1_SVA_SET)
  85 #define SEC_INTERFACE_USER_CTRL0_REG_V3 0x302220
  86 #define SEC_INTERFACE_USER_CTRL1_REG_V3 0x302224
  87 #define SEC_USER1_SMMU_NORMAL_V3        (BIT(23) | BIT(17) | BIT(11) | BIT(5))
  88 #define SEC_USER1_SMMU_MASK_V3          0xFF79E79E
  89 #define SEC_CORE_INT_STATUS_M_ECC       BIT(2)
  90
  91 #define SEC_PREFETCH_CFG                0x301130
  92 #define SEC_SVA_TRANS                   0x301EC4
  93 #define SEC_PREFETCH_ENABLE             (~(BIT(0) | BIT(1) | BIT(11)))
  94 #define SEC_PREFETCH_DISABLE            BIT(1)
  95 #define SEC_SVA_DISABLE_READY           (BIT(7) | BIT(11))
  96
  97 #define SEC_DELAY_10_US                 10
  98 #define SEC_POLL_TIMEOUT_US             1000
  99 #define SEC_DBGFS_VAL_MAX_LEN           20
 100 #define SEC_SINGLE_PORT_MAX_TRANS       0x2060
 101
 102 #define SEC_SQE_MASK_OFFSET             16
 103 #define SEC_SQE_MASK_LEN                108
 104 #define SEC_SHAPER_TYPE_RATE            400
 105
 106 #define SEC_DFX_BASE            0x301000
 107 #define SEC_DFX_CORE            0x302100
 108 #define SEC_DFX_COMMON1         0x301600
 109 #define SEC_DFX_COMMON2         0x301C00
 110 #define SEC_DFX_BASE_LEN                0x9D
 111 #define SEC_DFX_CORE_LEN                0x32B
 112 #define SEC_DFX_COMMON1_LEN             0x45
 113 #define SEC_DFX_COMMON2_LEN             0xBA
 114
 115 #define SEC_ALG_BITMAP_SHIFT            32
 116
 117 #define SEC_CIPHER_BITMAP               (GENMASK_ULL(5, 0) | GENMASK_ULL(16, 12) | \
 118                                         GENMASK(24, 21))
 119 #define SEC_DIGEST_BITMAP               (GENMASK_ULL(11, 8) | GENMASK_ULL(20, 19) | \
 120                                         GENMASK_ULL(42, 25))
 121 #define SEC_AEAD_BITMAP                 (GENMASK_ULL(7, 6) | GENMASK_ULL(18, 17) | \
 122                                         GENMASK_ULL(45, 43))
 123
 124 struct sec_hw_error {
 125         u32 int_msk;
 126         const char *msg;
 127 };
 128
 129 struct sec_dfx_item {
 130         const char *name;
 131         u32 offset;
 132 };
 133
 134 static const char sec_name[] = "hisi_sec2";
 135 static struct dentry *sec_debugfs_root;
 136
 137 static struct hisi_qm_list sec_devices = {
 138         .register_to_crypto     = sec_register_to_crypto,
 139         .unregister_from_crypto = sec_unregister_from_crypto,
 140 };
 141
 142 static const struct hisi_qm_cap_info sec_basic_info[] = {
 143         {SEC_QM_NFE_MASK_CAP,   0x3124, 0, GENMASK(31, 0), 0x0, 0x1C77, 0x7C77},
 144         {SEC_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC77, 0x6C77},
 145         {SEC_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C77},
 146         {SEC_QM_CE_MASK_CAP,    0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8},
 147         {SEC_NFE_MASK_CAP,      0x3130, 0, GENMASK(31, 0), 0x0, 0x177, 0x60177},
 148         {SEC_RESET_MASK_CAP,    0x3134, 0, GENMASK(31, 0), 0x0, 0x177, 0x177},
 149         {SEC_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x4, 0x177},
 150         {SEC_CE_MASK_CAP,       0x3138, 0, GENMASK(31, 0), 0x0, 0x88, 0xC088},
 151         {SEC_CLUSTER_NUM_CAP, 0x313c, 20, GENMASK(3, 0), 0x1, 0x1, 0x1},
 152         {SEC_CORE_TYPE_NUM_CAP, 0x313c, 16, GENMASK(3, 0), 0x1, 0x1, 0x1},
 153         {SEC_CORE_NUM_CAP, 0x313c, 8, GENMASK(7, 0), 0x4, 0x4, 0x4},
 154         {SEC_CORES_PER_CLUSTER_NUM_CAP, 0x313c, 0, GENMASK(7, 0), 0x4, 0x4, 0x4},
 155         {SEC_CORE_ENABLE_BITMAP, 0x3140, 0, GENMASK(31, 0), 0x17F, 0x17F, 0xF},
 156         {SEC_DRV_ALG_BITMAP_LOW, 0x3144, 0, GENMASK(31, 0), 0x18050CB, 0x18050CB, 0x18670CF},
 157         {SEC_DRV_ALG_BITMAP_HIGH, 0x3148, 0, GENMASK(31, 0), 0x395C, 0x395C, 0x395C},
 158         {SEC_DEV_ALG_BITMAP_LOW, 0x314c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 159         {SEC_DEV_ALG_BITMAP_HIGH, 0x3150, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF},
 160         {SEC_CORE1_ALG_BITMAP_LOW, 0x3154, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 161         {SEC_CORE1_ALG_BITMAP_HIGH, 0x3158, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF},
 162         {SEC_CORE2_ALG_BITMAP_LOW, 0x315c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 163         {SEC_CORE2_ALG_BITMAP_HIGH, 0x3160, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF},
 164         {SEC_CORE3_ALG_BITMAP_LOW, 0x3164, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 165         {SEC_CORE3_ALG_BITMAP_HIGH, 0x3168, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF},
 166         {SEC_CORE4_ALG_BITMAP_LOW, 0x316c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
 167         {SEC_CORE4_ALG_BITMAP_HIGH, 0x3170, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF},
 168 };
 169
 170 static const u32 sec_pre_store_caps[] = {
 171         SEC_DRV_ALG_BITMAP_LOW,
 172         SEC_DRV_ALG_BITMAP_HIGH,
 173         SEC_DEV_ALG_BITMAP_LOW,
 174         SEC_DEV_ALG_BITMAP_HIGH,
 175 };
 176
 177 static const struct qm_dev_alg sec_dev_algs[] = { {
 178                 .alg_msk = SEC_CIPHER_BITMAP,
 179                 .alg = "cipher\n",
 180         }, {
 181                 .alg_msk = SEC_DIGEST_BITMAP,
 182                 .alg = "digest\n",
 183         }, {
 184                 .alg_msk = SEC_AEAD_BITMAP,
 185                 .alg = "aead\n",
 186         },
 187 };
 188
 189 static const struct sec_hw_error sec_hw_errors[] = {
 190         {
 191                 .int_msk = BIT(0),
 192                 .msg = "sec_axi_rresp_err_rint"
 193         },
 194         {
 195                 .int_msk = BIT(1),
 196                 .msg = "sec_axi_bresp_err_rint"
 197         },
 198         {
 199                 .int_msk = BIT(2),
 200                 .msg = "sec_ecc_2bit_err_rint"
 201         },
 202         {
 203                 .int_msk = BIT(3),
 204                 .msg = "sec_ecc_1bit_err_rint"
 205         },
 206         {
 207                 .int_msk = BIT(4),
 208                 .msg = "sec_req_trng_timeout_rint"
 209         },
 210         {
 211                 .int_msk = BIT(5),
 212                 .msg = "sec_fsm_hbeat_rint"
 213         },
 214         {
 215                 .int_msk = BIT(6),
 216                 .msg = "sec_channel_req_rng_timeout_rint"
 217         },
 218         {
 219                 .int_msk = BIT(7),
 220                 .msg = "sec_bd_err_rint"
 221         },
 222         {
 223                 .int_msk = BIT(8),
 224                 .msg = "sec_chain_buff_err_rint"
 225         },
 226         {
 227                 .int_msk = BIT(14),
 228                 .msg = "sec_no_secure_access"
 229         },
 230         {
 231                 .int_msk = BIT(15),
 232                 .msg = "sec_wrapping_key_auth_err"
 233         },
 234         {
 235                 .int_msk = BIT(16),
 236                 .msg = "sec_km_key_crc_fail"
 237         },
 238         {
 239                 .int_msk = BIT(17),
 240                 .msg = "sec_axi_poison_err"
 241         },
 242         {
 243                 .int_msk = BIT(18),
 244                 .msg = "sec_sva_err"
 245         },
 246         {}
 247 };
 248
 249 static const char * const sec_dbg_file_name[] = {
 250         [SEC_CLEAR_ENABLE] = "clear_enable",
 251 };
 252
 253 static struct sec_dfx_item sec_dfx_labels[] = {
 254         {"send_cnt", offsetof(struct sec_dfx, send_cnt)},
 255         {"recv_cnt", offsetof(struct sec_dfx, recv_cnt)},
 256         {"send_busy_cnt", offsetof(struct sec_dfx, send_busy_cnt)},
 257         {"recv_busy_cnt", offsetof(struct sec_dfx, recv_busy_cnt)},
 258         {"err_bd_cnt", offsetof(struct sec_dfx, err_bd_cnt)},
 259         {"invalid_req_cnt", offsetof(struct sec_dfx, invalid_req_cnt)},
 260         {"done_flag_cnt", offsetof(struct sec_dfx, done_flag_cnt)},
 261 };
 262
 263 static const struct debugfs_reg32 sec_dfx_regs[] = {
 264         {"SEC_PF_ABNORMAL_INT_SOURCE    ",  0x301010},
 265         {"SEC_SAA_EN                    ",  0x301270},
 266         {"SEC_BD_LATENCY_MIN            ",  0x301600},
 267         {"SEC_BD_LATENCY_MAX            ",  0x301608},
 268         {"SEC_BD_LATENCY_AVG            ",  0x30160C},
 269         {"SEC_BD_NUM_IN_SAA0            ",  0x301670},
 270         {"SEC_BD_NUM_IN_SAA1            ",  0x301674},
 271         {"SEC_BD_NUM_IN_SEC             ",  0x301680},
 272         {"SEC_ECC_1BIT_CNT              ",  0x301C00},
 273         {"SEC_ECC_1BIT_INFO             ",  0x301C04},
 274         {"SEC_ECC_2BIT_CNT              ",  0x301C10},
 275         {"SEC_ECC_2BIT_INFO             ",  0x301C14},
 276         {"SEC_BD_SAA0                   ",  0x301C20},
 277         {"SEC_BD_SAA1                   ",  0x301C24},
 278         {"SEC_BD_SAA2                   ",  0x301C28},
 279         {"SEC_BD_SAA3                   ",  0x301C2C},
 280         {"SEC_BD_SAA4                   ",  0x301C30},
 281         {"SEC_BD_SAA5                   ",  0x301C34},
 282         {"SEC_BD_SAA6                   ",  0x301C38},
 283         {"SEC_BD_SAA7                   ",  0x301C3C},
 284         {"SEC_BD_SAA8                   ",  0x301C40},
 285         {"SEC_RAS_CE_ENABLE             ",  0x301050},
 286         {"SEC_RAS_FE_ENABLE             ",  0x301054},
 287         {"SEC_RAS_NFE_ENABLE            ",  0x301058},
 288         {"SEC_REQ_TRNG_TIME_TH          ",  0x30112C},
 289         {"SEC_CHANNEL_RNG_REQ_THLD      ",  0x302110},
 290 };
 291
 292 /* define the SEC's dfx regs region and region length */
 293 static struct dfx_diff_registers sec_diff_regs[] = {
 294         {
 295                 .reg_offset = SEC_DFX_BASE,
 296                 .reg_len = SEC_DFX_BASE_LEN,
 297         }, {
 298                 .reg_offset = SEC_DFX_COMMON1,
 299                 .reg_len = SEC_DFX_COMMON1_LEN,
 300         }, {
 301                 .reg_offset = SEC_DFX_COMMON2,
 302                 .reg_len = SEC_DFX_COMMON2_LEN,
 303         }, {
 304                 .reg_offset = SEC_DFX_CORE,
 305                 .reg_len = SEC_DFX_CORE_LEN,
 306         },
 307 };
 308
 309 static int sec_diff_regs_show(struct seq_file *s, void *unused)
 310 {
 311         struct hisi_qm *qm = s->private;
 312
 313         hisi_qm_acc_diff_regs_dump(qm, s, qm->debug.acc_diff_regs,
 314                                         ARRAY_SIZE(sec_diff_regs));
 315
 316         return 0;
 317 }
 318 DEFINE_SHOW_ATTRIBUTE(sec_diff_regs);
 319
 320 static bool pf_q_num_flag;
 321 static int sec_pf_q_num_set(const char *val, const struct kernel_param *kp)
 322 {
 323         pf_q_num_flag = true;
 324
 325         return q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_SEC_PF);
 326 }
 327
 328 static const struct kernel_param_ops sec_pf_q_num_ops = {
 329         .set = sec_pf_q_num_set,
 330         .get = param_get_int,
 331 };
 332
 333 static u32 pf_q_num = SEC_PF_DEF_Q_NUM;
 334 module_param_cb(pf_q_num, &sec_pf_q_num_ops, &pf_q_num, 0444);
 335 MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)");
 336
 337 static int sec_ctx_q_num_set(const char *val, const struct kernel_param *kp)
 338 {
 339         u32 ctx_q_num;
 340         int ret;
 341
 342         if (!val)
 343                 return -EINVAL;
 344
 345         ret = kstrtou32(val, 10, &ctx_q_num);
 346         if (ret)
 347                 return -EINVAL;
 348
 349         if (!ctx_q_num || ctx_q_num > SEC_CTX_Q_NUM_MAX || ctx_q_num & 0x1) {
 350                 pr_err("ctx queue num[%u] is invalid!\n", ctx_q_num);
 351                 return -EINVAL;
 352         }
 353
 354         return param_set_int(val, kp);
 355 }
 356
 357 static const struct kernel_param_ops sec_ctx_q_num_ops = {
 358         .set = sec_ctx_q_num_set,
 359         .get = param_get_int,
 360 };
 361 static u32 ctx_q_num = SEC_CTX_Q_NUM_DEF;
 362 module_param_cb(ctx_q_num, &sec_ctx_q_num_ops, &ctx_q_num, 0444);
 363 MODULE_PARM_DESC(ctx_q_num, "Queue num in ctx (2 default, 2, 4, ..., 32)");
 364
 365 static const struct kernel_param_ops vfs_num_ops = {
 366         .set = vfs_num_set,
 367         .get = param_get_int,
 368 };
 369
 370 static u32 vfs_num;
 371 module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444);
 372 MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");
 373
 374 void sec_destroy_qps(struct hisi_qp **qps, int qp_num)
 375 {
 376         hisi_qm_free_qps(qps, qp_num);
 377         kfree(qps);
 378 }
 379
 380 struct hisi_qp **sec_create_qps(void)
 381 {
 382         int node = cpu_to_node(raw_smp_processor_id());
 383         u32 ctx_num = ctx_q_num;
 384         struct hisi_qp **qps;
 385         int ret;
 386
 387         qps = kcalloc(ctx_num, sizeof(struct hisi_qp *), GFP_KERNEL);
 388         if (!qps)
 389                 return NULL;
 390
 391         ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, 0, node, qps);
 392         if (!ret)
 393                 return qps;
 394
 395         kfree(qps);
 396         return NULL;
 397 }
 398
 399 u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low)
 400 {
 401         u32 cap_val_h, cap_val_l;
 402
 403         cap_val_h = qm->cap_tables.dev_cap_table[high].cap_val;
 404         cap_val_l = qm->cap_tables.dev_cap_table[low].cap_val;
 405
 406         return ((u64)cap_val_h << SEC_ALG_BITMAP_SHIFT) | (u64)cap_val_l;
 407 }
 408
 409 static const struct kernel_param_ops sec_uacce_mode_ops = {
 410         .set = uacce_mode_set,
 411         .get = param_get_int,
 412 };
 413
 414 /*
 415  * uacce_mode = 0 means sec only register to crypto,
 416  * uacce_mode = 1 means sec both register to crypto and uacce.
 417  */
 418 static u32 uacce_mode = UACCE_MODE_NOUACCE;
 419 module_param_cb(uacce_mode, &sec_uacce_mode_ops, &uacce_mode, 0444);
 420 MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC);
 421
 422 static const struct pci_device_id sec_dev_ids[] = {
 423         { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_PF) },
 424         { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_VF) },
 425         { 0, }
 426 };
 427 MODULE_DEVICE_TABLE(pci, sec_dev_ids);
 428
 429 static void sec_set_endian(struct hisi_qm *qm)
 430 {
 431         u32 reg;
 432
 433         reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG);
 434         reg &= ~(BIT(1) | BIT(0));
 435         if (!IS_ENABLED(CONFIG_64BIT))
 436                 reg |= BIT(1);
 437
 438         if (!IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN))
 439                 reg |= BIT(0);
 440
 441         writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG);
 442 }
 443
 444 static void sec_engine_sva_config(struct hisi_qm *qm)
 445 {
 446         u32 reg;
 447
 448         if (qm->ver > QM_HW_V2) {
 449                 reg = readl_relaxed(qm->io_base +
 450                                 SEC_INTERFACE_USER_CTRL0_REG_V3);
 451                 reg |= SEC_USER0_SMMU_NORMAL;
 452                 writel_relaxed(reg, qm->io_base +
 453                                 SEC_INTERFACE_USER_CTRL0_REG_V3);
 454
 455                 reg = readl_relaxed(qm->io_base +
 456                                 SEC_INTERFACE_USER_CTRL1_REG_V3);
 457                 reg &= SEC_USER1_SMMU_MASK_V3;
 458                 reg |= SEC_USER1_SMMU_NORMAL_V3;
 459                 writel_relaxed(reg, qm->io_base +
 460                                 SEC_INTERFACE_USER_CTRL1_REG_V3);
 461         } else {
 462                 reg = readl_relaxed(qm->io_base +
 463                                 SEC_INTERFACE_USER_CTRL0_REG);
 464                 reg |= SEC_USER0_SMMU_NORMAL;
 465                 writel_relaxed(reg, qm->io_base +
 466                                 SEC_INTERFACE_USER_CTRL0_REG);
 467                 reg = readl_relaxed(qm->io_base +
 468                                 SEC_INTERFACE_USER_CTRL1_REG);
 469                 reg &= SEC_USER1_SMMU_MASK;
 470                 if (qm->use_sva)
 471                         reg |= SEC_USER1_SMMU_SVA;
 472                 else
 473                         reg |= SEC_USER1_SMMU_NORMAL;
 474                 writel_relaxed(reg, qm->io_base +
 475                                 SEC_INTERFACE_USER_CTRL1_REG);
 476         }
 477 }
 478
 479 static void sec_open_sva_prefetch(struct hisi_qm *qm)
 480 {
 481         u32 val;
 482         int ret;
 483
 484         if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))
 485                 return;
 486
 487         /* Enable prefetch */
 488         val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG);
 489         val &= SEC_PREFETCH_ENABLE;
 490         writel(val, qm->io_base + SEC_PREFETCH_CFG);
 491
 492         ret = readl_relaxed_poll_timeout(qm->io_base + SEC_PREFETCH_CFG,
 493                                          val, !(val & SEC_PREFETCH_DISABLE),
 494                                          SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US);
 495         if (ret)
 496                 pci_err(qm->pdev, "failed to open sva prefetch\n");
 497 }
 498
 499 static void sec_close_sva_prefetch(struct hisi_qm *qm)
 500 {
 501         u32 val;
 502         int ret;
 503
 504         if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))
 505                 return;
 506
 507         val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG);
 508         val |= SEC_PREFETCH_DISABLE;
 509         writel(val, qm->io_base + SEC_PREFETCH_CFG);
 510
 511         ret = readl_relaxed_poll_timeout(qm->io_base + SEC_SVA_TRANS,
 512                                          val, !(val & SEC_SVA_DISABLE_READY),
 513                                          SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US);
 514         if (ret)
 515                 pci_err(qm->pdev, "failed to close sva prefetch\n");
 516 }
 517
 518 static void sec_enable_clock_gate(struct hisi_qm *qm)
 519 {
 520         u32 val;
 521
 522         if (qm->ver < QM_HW_V3)
 523                 return;
 524
 525         val = readl_relaxed(qm->io_base + SEC_CONTROL_REG);
 526         val |= SEC_CLK_GATE_ENABLE;
 527         writel_relaxed(val, qm->io_base + SEC_CONTROL_REG);
 528
 529         val = readl(qm->io_base + SEC_DYNAMIC_GATE_REG);
 530         val |= SEC_DYNAMIC_GATE_EN;
 531         writel(val, qm->io_base + SEC_DYNAMIC_GATE_REG);
 532
 533         val = readl(qm->io_base + SEC_CORE_AUTO_GATE);
 534         val |= SEC_CORE_AUTO_GATE_EN;
 535         writel(val, qm->io_base + SEC_CORE_AUTO_GATE);
 536 }
 537
 538 static void sec_disable_clock_gate(struct hisi_qm *qm)
 539 {
 540         u32 val;
 541
 542         /* Kunpeng920 needs to close clock gating */
 543         val = readl_relaxed(qm->io_base + SEC_CONTROL_REG);
 544         val &= SEC_CLK_GATE_DISABLE;
 545         writel_relaxed(val, qm->io_base + SEC_CONTROL_REG);
 546 }
 547
 548 static int sec_engine_init(struct hisi_qm *qm)
 549 {
 550         int ret;
 551         u32 reg;
 552
 553         /* disable clock gate control before mem init */
 554         sec_disable_clock_gate(qm);
 555
 556         writel_relaxed(0x1, qm->io_base + SEC_MEM_START_INIT_REG);
 557
 558         ret = readl_relaxed_poll_timeout(qm->io_base + SEC_MEM_INIT_DONE_REG,
 559                                          reg, reg & 0x1, SEC_DELAY_10_US,
 560                                          SEC_POLL_TIMEOUT_US);
 561         if (ret) {
 562                 pci_err(qm->pdev, "fail to init sec mem\n");
 563                 return ret;
 564         }
 565
 566         reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG);
 567         reg |= (0x1 << SEC_TRNG_EN_SHIFT);
 568         writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG);
 569
 570         sec_engine_sva_config(qm);
 571
 572         writel(SEC_SINGLE_PORT_MAX_TRANS,
 573                qm->io_base + AM_CFG_SINGLE_PORT_MAX_TRANS);
 574
 575         reg = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CORE_ENABLE_BITMAP, qm->cap_ver);
 576         writel(reg, qm->io_base + SEC_SAA_EN_REG);
 577
 578         if (qm->ver < QM_HW_V3) {
 579                 /* HW V2 enable sm4 extra mode, as ctr/ecb */
 580                 writel_relaxed(SEC_BD_ERR_CHK_EN0,
 581                                qm->io_base + SEC_BD_ERR_CHK_EN_REG0);
 582
 583                 /* HW V2 enable sm4 xts mode multiple iv */
 584                 writel_relaxed(SEC_BD_ERR_CHK_EN1,
 585                                qm->io_base + SEC_BD_ERR_CHK_EN_REG1);
 586                 writel_relaxed(SEC_BD_ERR_CHK_EN3,
 587                                qm->io_base + SEC_BD_ERR_CHK_EN_REG3);
 588         }
 589
 590         /* config endian */
 591         sec_set_endian(qm);
 592
 593         sec_enable_clock_gate(qm);
 594
 595         return 0;
 596 }
 597
 598 static int sec_set_user_domain_and_cache(struct hisi_qm *qm)
 599 {
 600         /* qm user domain */
 601         writel(AXUSER_BASE, qm->io_base + QM_ARUSER_M_CFG_1);
 602         writel(ARUSER_M_CFG_ENABLE, qm->io_base + QM_ARUSER_M_CFG_ENABLE);
 603         writel(AXUSER_BASE, qm->io_base + QM_AWUSER_M_CFG_1);
 604         writel(AWUSER_M_CFG_ENABLE, qm->io_base + QM_AWUSER_M_CFG_ENABLE);
 605         writel(WUSER_M_CFG_ENABLE, qm->io_base + QM_WUSER_M_CFG_ENABLE);
 606
 607         /* qm cache */
 608         writel(AXI_M_CFG, qm->io_base + QM_AXI_M_CFG);
 609         writel(AXI_M_CFG_ENABLE, qm->io_base + QM_AXI_M_CFG_ENABLE);
 610
 611         /* disable FLR triggered by BME(bus master enable) */
 612         writel(PEH_AXUSER_CFG, qm->io_base + QM_PEH_AXUSER_CFG);
 613         writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE);
 614
 615         /* enable sqc,cqc writeback */
 616         writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE |
 617                CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) |
 618                FIELD_PREP(CQC_CACHE_WB_THRD, 1), qm->io_base + QM_CACHE_CTL);
 619
 620         return sec_engine_init(qm);
 621 }
 622
 623 /* sec_debug_regs_clear() - clear the sec debug regs */
 624 static void sec_debug_regs_clear(struct hisi_qm *qm)
 625 {
 626         int i;
 627
 628         /* clear sec dfx regs */
 629         writel(0x1, qm->io_base + SEC_CTRL_CNT_CLR_CE);
 630         for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++)
 631                 readl(qm->io_base + sec_dfx_regs[i].offset);
 632
 633         /* clear rdclr_en */
 634         writel(0x0, qm->io_base + SEC_CTRL_CNT_CLR_CE);
 635
 636         hisi_qm_debug_regs_clear(qm);
 637 }
 638
 639 static void sec_master_ooo_ctrl(struct hisi_qm *qm, bool enable)
 640 {
 641         u32 val1, val2;
 642
 643         val1 = readl(qm->io_base + SEC_CONTROL_REG);
 644         if (enable) {
 645                 val1 |= SEC_AXI_SHUTDOWN_ENABLE;
 646                 val2 = hisi_qm_get_hw_info(qm, sec_basic_info,
 647                                            SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);
 648         } else {
 649                 val1 &= SEC_AXI_SHUTDOWN_DISABLE;
 650                 val2 = 0x0;
 651         }
 652
 653         if (qm->ver > QM_HW_V2)
 654                 writel(val2, qm->io_base + SEC_OOO_SHUTDOWN_SEL);
 655
 656         writel(val1, qm->io_base + SEC_CONTROL_REG);
 657 }
 658
 659 static void sec_hw_error_enable(struct hisi_qm *qm)
 660 {
 661         u32 ce, nfe;
 662
 663         if (qm->ver == QM_HW_V1) {
 664                 writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK);
 665                 pci_info(qm->pdev, "V1 not support hw error handle\n");
 666                 return;
 667         }
 668
 669         ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CE_MASK_CAP, qm->cap_ver);
 670         nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver);
 671
 672         /* clear SEC hw error source if having */
 673         writel(ce | nfe | SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_CORE_INT_SOURCE);
 674
 675         /* enable RAS int */
 676         writel(ce, qm->io_base + SEC_RAS_CE_REG);
 677         writel(SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_RAS_FE_REG);
 678         writel(nfe, qm->io_base + SEC_RAS_NFE_REG);
 679
 680         /* enable SEC block master OOO when nfe occurs on Kunpeng930 */
 681         sec_master_ooo_ctrl(qm, true);
 682
 683         /* enable SEC hw error interrupts */
 684         writel(ce | nfe | SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_CORE_INT_MASK);
 685 }
 686
 687 static void sec_hw_error_disable(struct hisi_qm *qm)
 688 {
 689         /* disable SEC hw error interrupts */
 690         writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK);
 691
 692         /* disable SEC block master OOO when nfe occurs on Kunpeng930 */
 693         sec_master_ooo_ctrl(qm, false);
 694
 695         /* disable RAS int */
 696         writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_CE_REG);
 697         writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_FE_REG);
 698         writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_NFE_REG);
 699 }
 700
 701 static u32 sec_clear_enable_read(struct hisi_qm *qm)
 702 {
 703         return readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) &
 704                         SEC_CTRL_CNT_CLR_CE_BIT;
 705 }
 706
 707 static int sec_clear_enable_write(struct hisi_qm *qm, u32 val)
 708 {
 709         u32 tmp;
 710
 711         if (val != 1 && val)
 712                 return -EINVAL;
 713
 714         tmp = (readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) &
 715                ~SEC_CTRL_CNT_CLR_CE_BIT) | val;
 716         writel(tmp, qm->io_base + SEC_CTRL_CNT_CLR_CE);
 717
 718         return 0;
 719 }
 720
 721 static ssize_t sec_debug_read(struct file *filp, char __user *buf,
 722                                size_t count, loff_t *pos)
 723 {
 724         struct sec_debug_file *file = filp->private_data;
 725         char tbuf[SEC_DBGFS_VAL_MAX_LEN];
 726         struct hisi_qm *qm = file->qm;
 727         u32 val;
 728         int ret;
 729
 730         ret = hisi_qm_get_dfx_access(qm);
 731         if (ret)
 732                 return ret;
 733
 734         spin_lock_irq(&file->lock);
 735
 736         switch (file->index) {
 737         case SEC_CLEAR_ENABLE:
 738                 val = sec_clear_enable_read(qm);
 739                 break;
 740         default:
 741                 goto err_input;
 742         }
 743
 744         spin_unlock_irq(&file->lock);
 745
 746         hisi_qm_put_dfx_access(qm);
 747         ret = snprintf(tbuf, SEC_DBGFS_VAL_MAX_LEN, "%u\n", val);
 748         return simple_read_from_buffer(buf, count, pos, tbuf, ret);
 749
 750 err_input:
 751         spin_unlock_irq(&file->lock);
 752         hisi_qm_put_dfx_access(qm);
 753         return -EINVAL;
 754 }
 755
 756 static ssize_t sec_debug_write(struct file *filp, const char __user *buf,
 757                                size_t count, loff_t *pos)
 758 {
 759         struct sec_debug_file *file = filp->private_data;
 760         char tbuf[SEC_DBGFS_VAL_MAX_LEN];
 761         struct hisi_qm *qm = file->qm;
 762         unsigned long val;
 763         int len, ret;
 764
 765         if (*pos != 0)
 766                 return 0;
 767
 768         if (count >= SEC_DBGFS_VAL_MAX_LEN)
 769                 return -ENOSPC;
 770
 771         len = simple_write_to_buffer(tbuf, SEC_DBGFS_VAL_MAX_LEN - 1,
 772                                      pos, buf, count);
 773         if (len < 0)
 774                 return len;
 775
 776         tbuf[len] = '\0';
 777         if (kstrtoul(tbuf, 0, &val))
 778                 return -EFAULT;
 779
 780         ret = hisi_qm_get_dfx_access(qm);
 781         if (ret)
 782                 return ret;
 783
 784         spin_lock_irq(&file->lock);
 785
 786         switch (file->index) {
 787         case SEC_CLEAR_ENABLE:
 788                 ret = sec_clear_enable_write(qm, val);
 789                 if (ret)
 790                         goto err_input;
 791                 break;
 792         default:
 793                 ret = -EINVAL;
 794                 goto err_input;
 795         }
 796
 797         ret = count;
 798
 799  err_input:
 800         spin_unlock_irq(&file->lock);
 801         hisi_qm_put_dfx_access(qm);
 802         return ret;
 803 }
 804
 805 static const struct file_operations sec_dbg_fops = {
 806         .owner = THIS_MODULE,
 807         .open = simple_open,
 808         .read = sec_debug_read,
 809         .write = sec_debug_write,
 810 };
 811
 812 static int sec_debugfs_atomic64_get(void *data, u64 *val)
 813 {
 814         *val = atomic64_read((atomic64_t *)data);
 815
 816         return 0;
 817 }
 818
 819 static int sec_debugfs_atomic64_set(void *data, u64 val)
 820 {
 821         if (val)
 822                 return -EINVAL;
 823
 824         atomic64_set((atomic64_t *)data, 0);
 825
 826         return 0;
 827 }
 828
 829 DEFINE_DEBUGFS_ATTRIBUTE(sec_atomic64_ops, sec_debugfs_atomic64_get,
 830                          sec_debugfs_atomic64_set, "%lld\n");
 831
 832 static int sec_regs_show(struct seq_file *s, void *unused)
 833 {
 834         hisi_qm_regs_dump(s, s->private);
 835
 836         return 0;
 837 }
 838
 839 DEFINE_SHOW_ATTRIBUTE(sec_regs);
 840
 841 static int sec_core_debug_init(struct hisi_qm *qm)
 842 {
 843         struct dfx_diff_registers *sec_regs = qm->debug.acc_diff_regs;
 844         struct sec_dev *sec = container_of(qm, struct sec_dev, qm);
 845         struct device *dev = &qm->pdev->dev;
 846         struct sec_dfx *dfx = &sec->debug.dfx;
 847         struct debugfs_regset32 *regset;
 848         struct dentry *tmp_d;
 849         int i;
 850
 851         tmp_d = debugfs_create_dir("sec_dfx", qm->debug.debug_root);
 852
 853         regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
 854         if (!regset)
 855                 return -ENOMEM;
 856
 857         regset->regs = sec_dfx_regs;
 858         regset->nregs = ARRAY_SIZE(sec_dfx_regs);
 859         regset->base = qm->io_base;
 860         regset->dev = dev;
 861
 862         if (qm->pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF)
 863                 debugfs_create_file("regs", 0444, tmp_d, regset, &sec_regs_fops);
 864         if (qm->fun_type == QM_HW_PF && sec_regs)
 865                 debugfs_create_file("diff_regs", 0444, tmp_d,
 866                                       qm, &sec_diff_regs_fops);
 867
 868         for (i = 0; i < ARRAY_SIZE(sec_dfx_labels); i++) {
 869                 atomic64_t *data = (atomic64_t *)((uintptr_t)dfx +
 870                                         sec_dfx_labels[i].offset);
 871                 debugfs_create_file(sec_dfx_labels[i].name, 0644,
 872                                    tmp_d, data, &sec_atomic64_ops);
 873         }
 874
 875         return 0;
 876 }
 877
 878 static int sec_debug_init(struct hisi_qm *qm)
 879 {
 880         struct sec_dev *sec = container_of(qm, struct sec_dev, qm);
 881         int i;
 882
 883         if (qm->pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF) {
 884                 for (i = SEC_CLEAR_ENABLE; i < SEC_DEBUG_FILE_NUM; i++) {
 885                         spin_lock_init(&sec->debug.files[i].lock);
 886                         sec->debug.files[i].index = i;
 887                         sec->debug.files[i].qm = qm;
 888
 889                         debugfs_create_file(sec_dbg_file_name[i], 0600,
 890                                                   qm->debug.debug_root,
 891                                                   sec->debug.files + i,
 892                                                   &sec_dbg_fops);
 893                 }
 894         }
 895
 896         return sec_core_debug_init(qm);
 897 }
 898
 899 static int sec_debugfs_init(struct hisi_qm *qm)
 900 {
 901         struct device *dev = &qm->pdev->dev;
 902         int ret;
 903
 904         ret = hisi_qm_regs_debugfs_init(qm, sec_diff_regs, ARRAY_SIZE(sec_diff_regs));
 905         if (ret) {
 906                 dev_warn(dev, "Failed to init SEC diff regs!\n");
 907                 return ret;
 908         }
 909
 910         qm->debug.debug_root = debugfs_create_dir(dev_name(dev),
 911                                                         sec_debugfs_root);
 912         qm->debug.sqe_mask_offset = SEC_SQE_MASK_OFFSET;
 913         qm->debug.sqe_mask_len = SEC_SQE_MASK_LEN;
 914
 915         hisi_qm_debug_init(qm);
 916
 917         ret = sec_debug_init(qm);
 918         if (ret)
 919                 goto debugfs_remove;
 920
 921         return 0;
 922
 923 debugfs_remove:
 924         debugfs_remove_recursive(qm->debug.debug_root);
 925         hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(sec_diff_regs));
 926         return ret;
 927 }
 928
 929 static void sec_debugfs_exit(struct hisi_qm *qm)
 930 {
 931         debugfs_remove_recursive(qm->debug.debug_root);
 932
 933         hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(sec_diff_regs));
 934 }
 935
 936 static int sec_show_last_regs_init(struct hisi_qm *qm)
 937 {
 938         struct qm_debug *debug = &qm->debug;
 939         int i;
 940
 941         debug->last_words = kcalloc(ARRAY_SIZE(sec_dfx_regs),
 942                                         sizeof(unsigned int), GFP_KERNEL);
 943         if (!debug->last_words)
 944                 return -ENOMEM;
 945
 946         for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++)
 947                 debug->last_words[i] = readl_relaxed(qm->io_base +
 948                                                         sec_dfx_regs[i].offset);
 949
 950         return 0;
 951 }
 952
 953 static void sec_show_last_regs_uninit(struct hisi_qm *qm)
 954 {
 955         struct qm_debug *debug = &qm->debug;
 956
 957         if (qm->fun_type == QM_HW_VF || !debug->last_words)
 958                 return;
 959
 960         kfree(debug->last_words);
 961         debug->last_words = NULL;
 962 }
 963
 964 static void sec_show_last_dfx_regs(struct hisi_qm *qm)
 965 {
 966         struct qm_debug *debug = &qm->debug;
 967         struct pci_dev *pdev = qm->pdev;
 968         u32 val;
 969         int i;
 970
 971         if (qm->fun_type == QM_HW_VF || !debug->last_words)
 972                 return;
 973
 974         /* dumps last word of the debugging registers during controller reset */
 975         for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++) {
 976                 val = readl_relaxed(qm->io_base + sec_dfx_regs[i].offset);
 977                 if (val != debug->last_words[i])
 978                         pci_info(pdev, "%s \t= 0x%08x => 0x%08x\n",
 979                                 sec_dfx_regs[i].name, debug->last_words[i], val);
 980         }
 981 }
 982
 983 static void sec_log_hw_error(struct hisi_qm *qm, u32 err_sts)
 984 {
 985         const struct sec_hw_error *errs = sec_hw_errors;
 986         struct device *dev = &qm->pdev->dev;
 987         u32 err_val;
 988
 989         while (errs->msg) {
 990                 if (errs->int_msk & err_sts) {
 991                         dev_err(dev, "%s [error status=0x%x] found\n",
 992                                         errs->msg, errs->int_msk);
 993
 994                         if (SEC_CORE_INT_STATUS_M_ECC & errs->int_msk) {
 995                                 err_val = readl(qm->io_base +
 996                                                 SEC_CORE_SRAM_ECC_ERR_INFO);
 997                                 dev_err(dev, "multi ecc sram num=0x%x\n",
 998                                                 ((err_val) >> SEC_ECC_NUM) &
 999                                                 SEC_ECC_MASH);
1000                         }
1001                 }
1002                 errs++;
1003         }
1004 }
1005
1006 static u32 sec_get_hw_err_status(struct hisi_qm *qm)
1007 {
1008         return readl(qm->io_base + SEC_CORE_INT_STATUS);
1009 }
1010
1011 static void sec_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts)
1012 {
1013         u32 nfe;
1014
1015         writel(err_sts, qm->io_base + SEC_CORE_INT_SOURCE);
1016         nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver);
1017         writel(nfe, qm->io_base + SEC_RAS_NFE_REG);
1018 }
1019
1020 static void sec_open_axi_master_ooo(struct hisi_qm *qm)
1021 {
1022         u32 val;
1023
1024         val = readl(qm->io_base + SEC_CONTROL_REG);
1025         writel(val & SEC_AXI_SHUTDOWN_DISABLE, qm->io_base + SEC_CONTROL_REG);
1026         writel(val | SEC_AXI_SHUTDOWN_ENABLE, qm->io_base + SEC_CONTROL_REG);
1027 }
1028
1029 static void sec_err_info_init(struct hisi_qm *qm)
1030 {
1031         struct hisi_qm_err_info *err_info = &qm->err_info;
1032
1033         err_info->fe = SEC_RAS_FE_ENB_MSK;
1034         err_info->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_CE_MASK_CAP, qm->cap_ver);
1035         err_info->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_NFE_MASK_CAP, qm->cap_ver);
1036         err_info->ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC;
1037         err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info,
1038                                      SEC_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);
1039         err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info,
1040                         SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);
1041         err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info,
1042                         SEC_QM_RESET_MASK_CAP, qm->cap_ver);
1043         err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info,
1044                         SEC_RESET_MASK_CAP, qm->cap_ver);
1045         err_info->msi_wr_port = BIT(0);
1046         err_info->acpi_rst = "SRST";
1047 }
1048
1049 static const struct hisi_qm_err_ini sec_err_ini = {
1050         .hw_init                = sec_set_user_domain_and_cache,
1051         .hw_err_enable          = sec_hw_error_enable,
1052         .hw_err_disable         = sec_hw_error_disable,
1053         .get_dev_hw_err_status  = sec_get_hw_err_status,
1054         .clear_dev_hw_err_status = sec_clear_hw_err_status,
1055         .log_dev_hw_err         = sec_log_hw_error,
1056         .open_axi_master_ooo    = sec_open_axi_master_ooo,
1057         .open_sva_prefetch      = sec_open_sva_prefetch,
1058         .close_sva_prefetch     = sec_close_sva_prefetch,
1059         .show_last_dfx_regs     = sec_show_last_dfx_regs,
1060         .err_info_init          = sec_err_info_init,
1061 };
1062
1063 static int sec_pf_probe_init(struct sec_dev *sec)
1064 {
1065         struct hisi_qm *qm = &sec->qm;
1066         int ret;
1067
1068         qm->err_ini = &sec_err_ini;
1069         qm->err_ini->err_info_init(qm);
1070
1071         ret = sec_set_user_domain_and_cache(qm);
1072         if (ret)
1073                 return ret;
1074
1075         sec_open_sva_prefetch(qm);
1076         hisi_qm_dev_err_init(qm);
1077         sec_debug_regs_clear(qm);
1078         ret = sec_show_last_regs_init(qm);
1079         if (ret)
1080                 pci_err(qm->pdev, "Failed to init last word regs!\n");
1081
1082         return ret;
1083 }
1084
1085 static int sec_pre_store_cap_reg(struct hisi_qm *qm)
1086 {
1087         struct hisi_qm_cap_record *sec_cap;
1088         struct pci_dev *pdev = qm->pdev;
1089         size_t i, size;
1090
1091         size = ARRAY_SIZE(sec_pre_store_caps);
1092         sec_cap = devm_kzalloc(&pdev->dev, sizeof(*sec_cap) * size, GFP_KERNEL);
1093         if (!sec_cap)
1094                 return -ENOMEM;
1095
1096         for (i = 0; i < size; i++) {
1097                 sec_cap[i].type = sec_pre_store_caps[i];
1098                 sec_cap[i].cap_val = hisi_qm_get_hw_info(qm, sec_basic_info,
1099                                      sec_pre_store_caps[i], qm->cap_ver);
1100         }
1101
1102         qm->cap_tables.dev_cap_table = sec_cap;
1103
1104         return 0;
1105 }
1106
1107 static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
1108 {
1109         u64 alg_msk;
1110         int ret;
1111
1112         qm->pdev = pdev;
1113         qm->ver = pdev->revision;
1114         qm->mode = uacce_mode;
1115         qm->sqe_size = SEC_SQE_SIZE;
1116         qm->dev_name = sec_name;
1117
1118         qm->fun_type = (pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF) ?
1119                         QM_HW_PF : QM_HW_VF;
1120         if (qm->fun_type == QM_HW_PF) {
1121                 qm->qp_base = SEC_PF_DEF_Q_BASE;
1122                 qm->qp_num = pf_q_num;
1123                 qm->debug.curr_qm_qp_num = pf_q_num;
1124                 qm->qm_list = &sec_devices;
1125                 if (pf_q_num_flag)
1126                         set_bit(QM_MODULE_PARAM, &qm->misc_ctl);
1127         } else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) {
1128                 /*
1129                  * have no way to get qm configure in VM in v1 hardware,
1130                  * so currently force PF to uses SEC_PF_DEF_Q_NUM, and force
1131                  * to trigger only one VF in v1 hardware.
1132                  * v2 hardware has no such problem.
1133                  */
1134                 qm->qp_base = SEC_PF_DEF_Q_NUM;
1135                 qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM;
1136         }
1137
1138         ret = hisi_qm_init(qm);
1139         if (ret) {
1140                 pci_err(qm->pdev, "Failed to init sec qm configures!\n");
1141                 return ret;
1142         }
1143
1144         /* Fetch and save the value of capability registers */
1145         ret = sec_pre_store_cap_reg(qm);
1146         if (ret) {
1147                 pci_err(qm->pdev, "Failed to pre-store capability registers!\n");
1148                 hisi_qm_uninit(qm);
1149                 return ret;
1150         }
1151
1152         alg_msk = sec_get_alg_bitmap(qm, SEC_DEV_ALG_BITMAP_HIGH_IDX, SEC_DEV_ALG_BITMAP_LOW_IDX);
1153         ret = hisi_qm_set_algs(qm, alg_msk, sec_dev_algs, ARRAY_SIZE(sec_dev_algs));
1154         if (ret) {
1155                 pci_err(qm->pdev, "Failed to set sec algs!\n");
1156                 hisi_qm_uninit(qm);
1157         }
1158
1159         return ret;
1160 }
1161
1162 static void sec_qm_uninit(struct hisi_qm *qm)
1163 {
1164         hisi_qm_uninit(qm);
1165 }
1166
1167 static int sec_probe_init(struct sec_dev *sec)
1168 {
1169         u32 type_rate = SEC_SHAPER_TYPE_RATE;
1170         struct hisi_qm *qm = &sec->qm;
1171         int ret;
1172
1173         if (qm->fun_type == QM_HW_PF) {
1174                 ret = sec_pf_probe_init(sec);
1175                 if (ret)
1176                         return ret;
1177                 /* enable shaper type 0 */
1178                 if (qm->ver >= QM_HW_V3) {
1179                         type_rate |= QM_SHAPER_ENABLE;
1180                         qm->type_rate = type_rate;
1181                 }
1182         }
1183
1184         return 0;
1185 }
1186
1187 static void sec_probe_uninit(struct hisi_qm *qm)
1188 {
1189         hisi_qm_dev_err_uninit(qm);
1190 }
1191
1192 static void sec_iommu_used_check(struct sec_dev *sec)
1193 {
1194         struct iommu_domain *domain;
1195         struct device *dev = &sec->qm.pdev->dev;
1196
1197         domain = iommu_get_domain_for_dev(dev);
1198
1199         /* Check if iommu is used */
1200         sec->iommu_used = false;
1201         if (domain) {
1202                 if (domain->type & __IOMMU_DOMAIN_PAGING)
1203                         sec->iommu_used = true;
1204                 dev_info(dev, "SMMU Opened, the iommu type = %u\n",
1205                         domain->type);
1206         }
1207 }
1208
1209 static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1210 {
1211         struct sec_dev *sec;
1212         struct hisi_qm *qm;
1213         int ret;
1214
1215         sec = devm_kzalloc(&pdev->dev, sizeof(*sec), GFP_KERNEL);
1216         if (!sec)
1217                 return -ENOMEM;
1218
1219         qm = &sec->qm;
1220         ret = sec_qm_init(qm, pdev);
1221         if (ret) {
1222                 pci_err(pdev, "Failed to init SEC QM (%d)!\n", ret);
1223                 return ret;
1224         }
1225
1226         sec->ctx_q_num = ctx_q_num;
1227         sec_iommu_used_check(sec);
1228
1229         ret = sec_probe_init(sec);
1230         if (ret) {
1231                 pci_err(pdev, "Failed to probe!\n");
1232                 goto err_qm_uninit;
1233         }
1234
1235         ret = hisi_qm_start(qm);
1236         if (ret) {
1237                 pci_err(pdev, "Failed to start sec qm!\n");
1238                 goto err_probe_uninit;
1239         }
1240
1241         ret = sec_debugfs_init(qm);
1242         if (ret)
1243                 pci_warn(pdev, "Failed to init debugfs!\n");
1244
1245         hisi_qm_add_list(qm, &sec_devices);
1246         ret = hisi_qm_alg_register(qm, &sec_devices, ctx_q_num);
1247         if (ret < 0) {
1248                 pr_err("Failed to register driver to crypto.\n");
1249                 goto err_qm_del_list;
1250         }
1251
1252         if (qm->uacce) {
1253                 ret = uacce_register(qm->uacce);
1254                 if (ret) {
1255                         pci_err(pdev, "failed to register uacce (%d)!\n", ret);
1256                         goto err_alg_unregister;
1257                 }
1258         }
1259
1260         if (qm->fun_type == QM_HW_PF && vfs_num) {
1261                 ret = hisi_qm_sriov_enable(pdev, vfs_num);
1262                 if (ret < 0)
1263                         goto err_alg_unregister;
1264         }
1265
1266         hisi_qm_pm_init(qm);
1267
1268         return 0;
1269
1270 err_alg_unregister:
1271         hisi_qm_alg_unregister(qm, &sec_devices, ctx_q_num);
1272 err_qm_del_list:
1273         hisi_qm_del_list(qm, &sec_devices);
1274         sec_debugfs_exit(qm);
1275         hisi_qm_stop(qm, QM_NORMAL);
1276 err_probe_uninit:
1277         sec_show_last_regs_uninit(qm);
1278         sec_probe_uninit(qm);
1279 err_qm_uninit:
1280         sec_qm_uninit(qm);
1281         return ret;
1282 }
1283
1284 static void sec_remove(struct pci_dev *pdev)
1285 {
1286         struct hisi_qm *qm = pci_get_drvdata(pdev);
1287
1288         hisi_qm_pm_uninit(qm);
1289         hisi_qm_wait_task_finish(qm, &sec_devices);
1290         hisi_qm_alg_unregister(qm, &sec_devices, ctx_q_num);
1291         hisi_qm_del_list(qm, &sec_devices);
1292
1293         if (qm->fun_type == QM_HW_PF && qm->vfs_num)
1294                 hisi_qm_sriov_disable(pdev, true);
1295
1296         sec_debugfs_exit(qm);
1297
1298         (void)hisi_qm_stop(qm, QM_NORMAL);
1299
1300         if (qm->fun_type == QM_HW_PF)
1301                 sec_debug_regs_clear(qm);
1302         sec_show_last_regs_uninit(qm);
1303
1304         sec_probe_uninit(qm);
1305
1306         sec_qm_uninit(qm);
1307 }
1308
1309 static const struct dev_pm_ops sec_pm_ops = {
1310         SET_RUNTIME_PM_OPS(hisi_qm_suspend, hisi_qm_resume, NULL)
1311 };
1312
1313 static const struct pci_error_handlers sec_err_handler = {
1314         .error_detected = hisi_qm_dev_err_detected,
1315         .slot_reset     = hisi_qm_dev_slot_reset,
1316         .reset_prepare  = hisi_qm_reset_prepare,
1317         .reset_done     = hisi_qm_reset_done,
1318 };
1319
1320 static struct pci_driver sec_pci_driver = {
1321         .name = "hisi_sec2",
1322         .id_table = sec_dev_ids,
1323         .probe = sec_probe,
1324         .remove = sec_remove,
1325         .err_handler = &sec_err_handler,
1326         .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ?
1327                                 hisi_qm_sriov_configure : NULL,
1328         .shutdown = hisi_qm_dev_shutdown,
1329         .driver.pm = &sec_pm_ops,
1330 };
1331
1332 struct pci_driver *hisi_sec_get_pf_driver(void)
1333 {
1334         return &sec_pci_driver;
1335 }
1336 EXPORT_SYMBOL_GPL(hisi_sec_get_pf_driver);
1337
1338 static void sec_register_debugfs(void)
1339 {
1340         if (!debugfs_initialized())
1341                 return;
1342
1343         sec_debugfs_root = debugfs_create_dir("hisi_sec2", NULL);
1344 }
1345
1346 static void sec_unregister_debugfs(void)
1347 {
1348         debugfs_remove_recursive(sec_debugfs_root);
1349 }
1350
1351 static int __init sec_init(void)
1352 {
1353         int ret;
1354
1355         hisi_qm_init_list(&sec_devices);
1356         sec_register_debugfs();
1357
1358         ret = pci_register_driver(&sec_pci_driver);
1359         if (ret < 0) {
1360                 sec_unregister_debugfs();
1361                 pr_err("Failed to register pci driver.\n");
1362                 return ret;
1363         }
1364
1365         return 0;
1366 }
1367
1368 static void __exit sec_exit(void)
1369 {
1370         pci_unregister_driver(&sec_pci_driver);
1371         sec_unregister_debugfs();
1372 }
1373
1374 module_init(sec_init);
1375 module_exit(sec_exit);
1376
1377 MODULE_LICENSE("GPL v2");
1378 MODULE_AUTHOR("Zaibo Xu <[email protected]>");
1379 MODULE_AUTHOR("Longfang Liu <[email protected]>");
1380 MODULE_AUTHOR("Kai Ye <[email protected]>");
1381 MODULE_AUTHOR("Wei Zhang <[email protected]>");
1382 MODULE_DESCRIPTION("Driver for HiSilicon SEC accelerator");